diff --git a/.github/workflows/llms_txt_snippet_sync.yml b/.github/workflows/llms_txt_snippet_sync.yml new file mode 100644 index 000000000..d64b54b6d --- /dev/null +++ b/.github/workflows/llms_txt_snippet_sync.yml @@ -0,0 +1,57 @@ +name: llms.txt Snippet Sync + +# llms.txt is hand-maintained in the weaviate-io repo, so a snippet change here can +# strand a block in the published file. Only the weekly llms_txt_tests.yml job notices, +# which means the break surfaces days later. This gives the author the signal at PR time. +# +# It is advisory by design and never fails: when a snippet PR is opened, weaviate-io has +# not merged or deployed yet, so the live llms.txt legitimately cannot match. A blocking +# check would fire on every honest PR and would just be overridden. + +permissions: + contents: read + +on: + pull_request: + paths: + # Kept in step with SNIPPET_GLOBS in tests/test_llms_txt_code.py. Java and C# + # snippets live with their language suites, not under _includes/code/llms-txt/. + - "_includes/code/llms-txt/**" + - "_includes/code/java-v6/src/test/java/LlmsTxtTest.java" + - "_includes/code/csharp/LlmsTxtTest.cs" + - "tests/test_llms_txt_code.py" + - "tests/check_llms_txt_drift.py" + +env: + PYTHON_VERSION: "3.11" + +jobs: + check-snippet-sync: + name: Check llms.txt snippet sync + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Fetch the PR base commit + # Shallow single-commit fetch: the check only needs the base tree to read the + # pre-change snippet files. If it fails the check reports a degraded warning + # rather than blocking. + continue-on-error: true + run: git fetch --no-tags --depth=1 origin ${{ github.event.pull_request.base.sha }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install pytest + # The check imports tests/test_llms_txt_code.py to reuse its matching logic, and + # that module imports pytest. Nothing else from the test suite is needed, so this + # stays far cheaper than the full setup-test-env composite. + run: python -m pip install --quiet "pytest>=8.3.5" + + - name: Check llms.txt snippet sync + run: python tests/check_llms_txt_drift.py --base "${{ github.event.pull_request.base.sha }}" diff --git a/_includes/code/client-libraries/python_v4.py b/_includes/code/client-libraries/python_v4.py index 083c9d388..b647c8aca 100644 --- a/_includes/code/client-libraries/python_v4.py +++ b/_includes/code/client-libraries/python_v4.py @@ -360,6 +360,33 @@ # END BatchRateLimit +# START BatchIngestGenerator +import weaviate + +client = weaviate.connect_to_local() + +# A generator produces objects one at a time instead of building a list +def article_titles(): + for title in ["Multitenancy", "Database schema"]: + yield {"title": title} + +try: + articles = client.collections.use("WikiArticle") + # `ingest` accepts any iterable, including a generator + result = articles.data.ingest(article_titles()) + + if result.errors: + print(f"Number of failed imports: {len(result.errors)}") + +finally: + client.close() +# END BatchIngestGenerator + +# Tests +assert len(result.errors) == 0 +assert len(result.uuids) == 2 + + import weaviate client = weaviate.connect_to_local() diff --git a/_includes/code/csharp/ManageObjectsImportTest.cs b/_includes/code/csharp/ManageObjectsImportTest.cs index c556d141e..822c8e5dd 100644 --- a/_includes/code/csharp/ManageObjectsImportTest.cs +++ b/_includes/code/csharp/ManageObjectsImportTest.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using CsvHelper; using Weaviate.Client; +using Weaviate.Client.Batch; using Weaviate.Client.Models; using Xunit; @@ -146,8 +147,55 @@ await client.Collections.Create( var collection = client.Collections.Use("MyCollection"); - // Use `Batch.InsertMany` for server-side batching. The client sends - // data in batches at a rate controlled by the server. + // Use `Batch.StartBatch` for server-side batching. The client streams + // objects to the server, which paces the import based on its own load. + // highlight-start + await using var batch = await collection.Batch.StartBatch(); + + var handles = new List(); + foreach (var dataRow in dataRows) + { + handles.Add(await batch.Add(dataRow)); + } + + await batch.Close(); + // highlight-end + + var results = await Task.WhenAll(handles.Select(h => h.Result)); + var failedObjects = results.Where(r => !r.Success).ToList(); + if (failedObjects.Any()) + { + Console.WriteLine($"Number of failed imports: {failedObjects.Count}"); + } + // END ServerSideBatchImportExample + + var result = await collection.Aggregate.OverAll(totalCount: true); + Assert.Equal(5, result.TotalCount); + } + + [Fact] + public async Task TestServerSideIngest() + { + await BeforeEach(); + await client.Collections.Create( + new CollectionCreateParams + { + Name = "MyCollection", + VectorConfig = Configure.Vector("default", v => v.SelfProvided()), + } + ); + + // START ServerSideIngestExample + var dataRows = Enumerable + .Range(0, 5) + .Select(i => new { title = $"Object {i + 1}" }) + .ToList(); + + var collection = client.Collections.Use("MyCollection"); + + // `Batch.InsertMany` is the one-shot server-side ingest of an + // in-memory list. The client streams the list to the server + // using server-side batching under the hood. // highlight-start var response = await collection.Batch.InsertMany(dataRows); // highlight-end @@ -158,7 +206,7 @@ await client.Collections.Create( Console.WriteLine($"Number of failed imports: {failedObjects.Count}"); Console.WriteLine($"First failed object: {failedObjects.First().Error}"); } - // END ServerSideBatchImportExample + // END ServerSideIngestExample var result = await collection.Aggregate.OverAll(totalCount: true); Assert.Equal(5, result.TotalCount); @@ -194,8 +242,9 @@ await client.Collections.Create( var collection = client.Collections.Use("MyCollection"); + // `Batch.InsertMany` imports the list using server-side batching. // highlight-start - var response = await collection.Data.InsertMany(dataToInsert); + var response = await collection.Batch.InsertMany(dataToInsert); // highlight-end var failedObjects = response.Where(r => r.Error != null).ToList(); @@ -245,13 +294,17 @@ await client.Collections.Create( var collection = client.Collections.Use("MyCollection"); - var response = await collection.Data.InsertMany(dataToInsert); + // `Batch.InsertMany` imports the list using server-side batching. + // highlight-start + var response = await collection.Batch.InsertMany(dataToInsert); + // highlight-end // Handle errors - if (response.HasErrors) + var failedObjects = response.Where(r => r.Error != null).ToList(); + if (failedObjects.Any()) { - Console.WriteLine($"Number of failed imports: {response.Errors.Count()}"); - Console.WriteLine($"First failed object: {response.Errors.First().Message}"); + Console.WriteLine($"Number of failed imports: {failedObjects.Count}"); + Console.WriteLine($"First failed object: {failedObjects.First().Error}"); } // END BatchImportWithVectorExample @@ -342,16 +395,17 @@ await client.Collections.Create( var collection = client.Collections.Use("MyCollection"); - // Insert the data using InsertMany + // `Batch.InsertMany` imports the list using server-side batching. // highlight-start - var response = await collection.Data.InsertMany(dataToInsert); + var response = await collection.Batch.InsertMany(dataToInsert); // highlight-end // Handle errors - if (response.HasErrors) + var failedObjects = response.Where(r => r.Error != null).ToList(); + if (failedObjects.Any()) { - Console.WriteLine($"Number of failed imports: {response.Errors.Count()}"); - Console.WriteLine($"First failed object error: {response.Errors.First().Message}"); + Console.WriteLine($"Number of failed imports: {failedObjects.Count}"); + Console.WriteLine($"First failed object error: {failedObjects.First().Error}"); } // END BatchImportWithNamedVectors } diff --git a/_includes/code/csharp/QuickstartLocalTest.cs b/_includes/code/csharp/QuickstartLocalTest.cs index d6fc27dfc..6f9e71eb5 100644 --- a/_includes/code/csharp/QuickstartLocalTest.cs +++ b/_includes/code/csharp/QuickstartLocalTest.cs @@ -87,21 +87,25 @@ public async Task FullQuickstartWorkflowTest() ); } - // Call InsertMany with the list of objects converted to an array - var insertResponse = await questions.Data.InsertMany(questionsToInsert.ToArray()); + // `Batch.InsertMany` imports the list using server-side batching + var insertResponse = await questions.Batch.InsertMany(questionsToInsert); // highlight-end - // END Import // Check for errors if (insertResponse.HasErrors) { Console.WriteLine($"Number of failed imports: {insertResponse.Errors.Count()}"); - Console.WriteLine($"First failed object error: {insertResponse.Errors.First()}"); + // `Objects` holds one entry per object; `Index` is the position of the object in the input + foreach (var entry in insertResponse.Objects.Where(o => o.Error is not null)) + { + Console.WriteLine($"Failed object at index {entry.Index}: {entry.Error.Message}"); + } } else { - Console.WriteLine($"Successfully inserted {insertResponse.Objects.Count()} objects."); + Console.WriteLine($"Successfully inserted {insertResponse.Count} objects."); } + // END Import // START NearText // highlight-start diff --git a/_includes/code/csharp/QuickstartTest.cs b/_includes/code/csharp/QuickstartTest.cs index 894f41c68..f36f81802 100644 --- a/_includes/code/csharp/QuickstartTest.cs +++ b/_includes/code/csharp/QuickstartTest.cs @@ -63,7 +63,7 @@ public static async Task FullQuickstartWorkflowTest() Property.Text("category"), ], VectorConfig = Configure.Vector("default", v => v.Text2VecWeaviate()), // Configure the Weaviate Embeddings integration - GenerativeConfig = Configure.Generative.Cohere(), // Configure the Cohere generative AI integration + GenerativeConfig = Configure.Generative.OpenAI(), // Configure the OpenAI generative AI integration } ); // highlight-end @@ -95,21 +95,25 @@ public static async Task FullQuickstartWorkflowTest() ); } - // Call InsertMany with the list of objects converted to an array - var insertResponse = await questions.Data.InsertMany(questionsToInsert.ToArray()); + // `Batch.InsertMany` imports the list using server-side batching + var insertResponse = await questions.Batch.InsertMany(questionsToInsert); // highlight-end - // END Import // Check for errors if (insertResponse.HasErrors) { Console.WriteLine($"Number of failed imports: {insertResponse.Errors.Count()}"); - Console.WriteLine($"First failed object error: {insertResponse.Errors.First()}"); + // `Objects` holds one entry per object; `Index` is the position of the object in the input + foreach (var entry in insertResponse.Objects.Where(o => o.Error is not null)) + { + Console.WriteLine($"Failed object at index {entry.Index}: {entry.Error.Message}"); + } } else { - Console.WriteLine($"Successfully inserted {insertResponse.Objects.Count()} objects."); + Console.WriteLine($"Successfully inserted {insertResponse.Count} objects."); } + // END Import // START NearText // highlight-start diff --git a/_includes/code/csharp/RBACTest.cs b/_includes/code/csharp/RBACTest.cs index a83b85b50..969442883 100644 --- a/_includes/code/csharp/RBACTest.cs +++ b/_includes/code/csharp/RBACTest.cs @@ -475,4 +475,112 @@ public async Task TestUserLifecycle() var usersAfterDelete = await client.Users.Db.List(); Assert.DoesNotContain(usersAfterDelete, u => u.UserId == testUser); } + + [Fact] + public async Task TestOidcUserLifecycle() + { + // An OIDC user is authenticated by the identity provider, so it is never + // created in Weaviate. Only its role assignments are managed here. + string testUser = "custom-user"; + string testRole = "testRole"; + + var permissions = new PermissionScope[] + { + new Permissions.Collections("TargetCollection*") { Read = true }, + }; + await client.Roles.Create(testRole, permissions); + + // START AssignOidcUserRole + await client.Users.Oidc.AssignRoles(testUser, new[] { testRole, "viewer" }); + // END AssignOidcUserRole + + // START ListOidcUserRoles + var oidcUserRoles = await client.Users.Oidc.GetRoles(testUser); + foreach (var role in oidcUserRoles) + { + Console.WriteLine(role.Name); + } + // END ListOidcUserRoles + + var oidcRoleNames = oidcUserRoles.Select(r => r.Name).ToList(); + Assert.Contains(testRole, oidcRoleNames); + Assert.Contains("viewer", oidcRoleNames); + + // START RevokeOidcUserRoles + await client.Users.Oidc.RevokeRoles(testUser, new[] { testRole }); + // END RevokeOidcUserRoles + + var rolesAfterRevoke = await client.Users.Oidc.GetRoles(testUser); + var namesAfterRevoke = rolesAfterRevoke.Select(r => r.Name).ToList(); + Assert.DoesNotContain(testRole, namesAfterRevoke); + Assert.Contains("viewer", namesAfterRevoke); + + // Leave no assignment behind for the next test run + await client.Users.Oidc.RevokeRoles(testUser, new[] { "viewer" }); + } + + [Fact] + public async Task TestOidcGroupLifecycle() + { + string testGroup = "/admin-group"; + string testRole = "testRole"; + + var permissions = new PermissionScope[] + { + new Permissions.Collections("TargetCollection*") { Read = true }, + }; + await client.Roles.Create(testRole, permissions); + + // START AssignOidcGroupRoles + await client.Groups.Oidc.AssignRoles(testGroup, new[] { testRole, "viewer" }); + // END AssignOidcGroupRoles + + // START GetOidcGroupRoles + var groupRoles = await client.Groups.Oidc.GetRoles(testGroup, includeFullRoles: true); + foreach (var role in groupRoles) + { + Console.WriteLine(role.Name); + } + // END GetOidcGroupRoles + + var groupRoleNames = groupRoles.Select(r => r.Name).ToList(); + Assert.Contains(testRole, groupRoleNames); + Assert.Contains("viewer", groupRoleNames); + + // START GetKnownOidcGroups + var knownGroups = await client.Groups.Oidc.GetKnownGroupNames(); + Console.WriteLine($"Known OIDC groups ({knownGroups.Count()}): {string.Join(", ", knownGroups)}"); + // END GetKnownOidcGroups + + // The C# client writes the group ID straight into the request path without + // URL-encoding it, so a leading slash collapses and Weaviate registers + // "/admin-group" as "admin-group". Compare the normalized form so this + // assertion keeps holding once the client encodes the path segment. + Assert.Contains(testGroup.TrimStart('/'), knownGroups.Select(g => g.TrimStart('/'))); + + // Known client defect: the DTO behind GetGroupAssignments declares its + // groupType as an enum without a string-enum converter, so decoding a + // non-empty response throws. The call below is the correct usage and + // starts passing as soon as the client adds the converter. + // START GetGroupAssignments + var groupAssignments = await client.Roles.GetGroupAssignments(testRole); + Console.WriteLine($"Groups assigned to role '{testRole}':"); + foreach (var assignment in groupAssignments) + { + Console.WriteLine($" - Group ID: {assignment.GroupId}, Type: {assignment.GroupType}"); + } + // END GetGroupAssignments + Assert.Contains( + groupAssignments, + a => a.GroupId.TrimStart('/') == testGroup.TrimStart('/') + ); + Assert.Contains(groupAssignments, a => a.GroupType == RbacGroupType.Oidc); + + // START RevokeOidcGroupRoles + await client.Groups.Oidc.RevokeRoles(testGroup, new[] { testRole, "viewer" }); + // END RevokeOidcGroupRoles + + var rolesAfterRevoke = await client.Groups.Oidc.GetRoles(testGroup); + Assert.Empty(rolesAfterRevoke); + } } diff --git a/_includes/code/csharp/SearchGenerativeTest.cs b/_includes/code/csharp/SearchGenerativeTest.cs index 236ab788b..c9b783472 100644 --- a/_includes/code/csharp/SearchGenerativeTest.cs +++ b/_includes/code/csharp/SearchGenerativeTest.cs @@ -188,7 +188,7 @@ public async Task TestSingleGenerativeParameters() [Fact] public async Task TestGroupedGenerative() { - // START GroupedGenerative + // START GroupedGenerativeBasic // highlight-start var task = "What do these animals have in common, if anything?"; // highlight-end @@ -204,7 +204,7 @@ public async Task TestGroupedGenerative() // print the generated response Console.WriteLine($"Grouped task result: {response.Generative?.Values.First()}"); - // END GroupedGenerative + // END GroupedGenerativeBasic } [Fact] diff --git a/_includes/code/csharp/quickstart/QuickstartCreate.cs b/_includes/code/csharp/quickstart/QuickstartCreate.cs index e1a4f4c6f..256d8ff87 100644 --- a/_includes/code/csharp/quickstart/QuickstartCreate.cs +++ b/_includes/code/csharp/quickstart/QuickstartCreate.cs @@ -65,8 +65,8 @@ public static async Task Run() }, }; - // Insert objects using InsertMany - var insertResponse = await movies.Data.InsertMany(dataObjects.ToArray()); + // Insert the objects using server-side batching + var insertResponse = await movies.Batch.InsertMany(dataObjects); if (insertResponse.HasErrors) { diff --git a/_includes/code/csharp/quickstart/QuickstartCreateVectors.cs b/_includes/code/csharp/quickstart/QuickstartCreateVectors.cs index 9ebb7f3d1..330ccd720 100644 --- a/_includes/code/csharp/quickstart/QuickstartCreateVectors.cs +++ b/_includes/code/csharp/quickstart/QuickstartCreateVectors.cs @@ -92,8 +92,8 @@ public static async Task Run() ), }; - // Insert the objects with vectors - var insertResponse = await movies.Data.InsertMany(dataToInsert); + // Insert the objects with vectors using server-side batching + var insertResponse = await movies.Batch.InsertMany(dataToInsert); if (insertResponse.HasErrors) { diff --git a/_includes/code/csharp/quickstart/QuickstartLocalCreate.cs b/_includes/code/csharp/quickstart/QuickstartLocalCreate.cs index 6d00abfb2..cf0f4e75c 100644 --- a/_includes/code/csharp/quickstart/QuickstartLocalCreate.cs +++ b/_includes/code/csharp/quickstart/QuickstartLocalCreate.cs @@ -70,8 +70,8 @@ public static async Task Run() }, }; - // Insert objects using InsertMany - var insertResponse = await movies.Data.InsertMany(dataObjects.ToArray()); + // Insert the objects using server-side batching + var insertResponse = await movies.Batch.InsertMany(dataObjects); if (insertResponse.HasErrors) { diff --git a/_includes/code/csharp/quickstart/QuickstartLocalCreateVectors.cs b/_includes/code/csharp/quickstart/QuickstartLocalCreateVectors.cs index 1e08c3d1b..93cbf8b53 100644 --- a/_includes/code/csharp/quickstart/QuickstartLocalCreateVectors.cs +++ b/_includes/code/csharp/quickstart/QuickstartLocalCreateVectors.cs @@ -76,8 +76,8 @@ public static async Task Run() ), }; - // Insert the objects with vectors - var insertResponse = await movies.Data.InsertMany(dataToInsert); + // Insert the objects with vectors using server-side batching + var insertResponse = await movies.Batch.InsertMany(dataToInsert); if (insertResponse.HasErrors) { Console.WriteLine($"Errors during import: {insertResponse.Errors}"); diff --git a/_includes/code/howto/configure-sq/sq-compression-v3.ts b/_includes/code/howto/configure-sq/sq-compression-v3.ts index f5af4099f..0656559f2 100644 --- a/_includes/code/howto/configure-sq/sq-compression-v3.ts +++ b/_includes/code/howto/configure-sq/sq-compression-v3.ts @@ -1,5 +1,3 @@ -// not yet supported in client 3.0.8 - import assert from 'assert'; import weaviate from 'weaviate-client'; // START-ANY @@ -27,7 +25,7 @@ const collection = await client.collections.create({ let collectionConfig = await collection.config.get(); -assert.equal(collectionConfig.vectorizers.default.indexConfig.quantizer.type, "SQ") +assert.equal(collectionConfig.vectorizers.default.indexConfig.quantizer.type, "sq") // Clean-up await client.collections.delete(collectionName); @@ -49,10 +47,10 @@ const collection = await client.collections.create({ vectorizers: weaviate.configure.vectors.selfProvided({ vectorIndexConfig: weaviate.configure.vectorIndex.hnsw({ quantizer: weaviate.configure.vectorIndex.quantizer.sq({ - cache: true, // Enable caching - rescoreLimit: 200, // The minimum number of candidates to fetch before rescoring + rescoreLimit: 200, // The minimum number of candidates to fetch before rescoring + trainingLimit: 50000, // The size of the training set used to determine the bucket boundaries }), - vectorCacheMaxObjects: 10000 // Cache size (used if `cache` enabled) + vectorCacheMaxObjects: 100000 // Maximum number of objects in the vector cache }) }) }) @@ -60,9 +58,54 @@ const collection = await client.collections.create({ let collectionConfig = await collection.config.get(); -assert.equal(collectionConfig.vectorizers.default.indexConfig.quantizer.type, "SQ") +assert.equal(collectionConfig.vectorizers.default.indexConfig.quantizer.type, "sq") + +// Clean-up +await client.collections.delete(collectionName); + +client.close(); + + +// UPDATE SCHEMA +{ +const client = await weaviate.connectToLocal(); + +const collectionName = 'MyCollection'; + +// Prep +await client.collections.delete(collectionName); +await client.collections.create({ + name: collectionName, + vectorizers: weaviate.configure.vectors.selfProvided({ + vectorIndexConfig: weaviate.configure.vectorIndex.hnsw({ + quantizer: weaviate.configure.vectorIndex.quantizer.none(), + }) + }) +}) + +// START UpdateSchema +const collection = client.collections.use('MyCollection'); + +await collection.config.update({ + vectorizers: [ + weaviate.reconfigure.vectors.update({ + name: 'default', + vectorIndexConfig: weaviate.reconfigure.vectorIndex.hnsw({ + quantizer: weaviate.reconfigure.vectorIndex.quantizer.sq({ + rescoreLimit: 20, + }), + }), + }), + ], +}) +// END UpdateSchema + +let collectionConfig = await collection.config.get(); + +assert.equal(collectionConfig.vectorizers.default.indexConfig.quantizer.type, "sq") // Clean-up await client.collections.delete(collectionName); client.close(); +} diff --git a/_includes/code/howto/configure-sq/sq-compression.options-v3.ts b/_includes/code/howto/configure-sq/sq-compression.options-v3.ts index 8af478c8e..34422eb12 100644 --- a/_includes/code/howto/configure-sq/sq-compression.options-v3.ts +++ b/_includes/code/howto/configure-sq/sq-compression.options-v3.ts @@ -11,24 +11,24 @@ const collectionName = 'MyCollection'; // Prep await client.collections.delete(collectionName); -// START BQWithOptions +// START SQWithOptions const collection = await client.collections.create({ name: 'MyCollection', vectorizers: weaviate.configure.vectors.selfProvided({ vectorIndexConfig: weaviate.configure.vectorIndex.hnsw({ - quantizer: weaviate.configure.vectorIndex.quantizer.bq({ - cache: true, // Enable caching - rescoreLimit: 200, // The minimum number of candidates to fetch before rescoring + quantizer: weaviate.configure.vectorIndex.quantizer.sq({ + rescoreLimit: 200, // The minimum number of candidates to fetch before rescoring + trainingLimit: 50000, // The size of the training set used to determine the bucket boundaries }), - vectorCacheMaxObjects: 10000 // Cache size (used if `cache` enabled) + vectorCacheMaxObjects: 100000 // Maximum number of objects in the vector cache }) }) }) -// END BQWithOptions +// END SQWithOptions let collectionConfig = await collection.config.get(); -assert.equal(collectionConfig.vectorizers.default.indexConfig.quantizer.type, "bq") +assert.equal(collectionConfig.vectorizers.default.indexConfig.quantizer.type, "sq") // Clean-up await client.collections.delete(collectionName); diff --git a/_includes/code/howto/configure.backups.py b/_includes/code/howto/configure.backups.py index f143c91d7..e416fb5ba 100644 --- a/_includes/code/howto/configure.backups.py +++ b/_includes/code/howto/configure.backups.py @@ -191,6 +191,24 @@ assert client.collections.exists("Article") assert client.collections.exists("Publication") +# START ListBackups +backups = client.backup.list_backups( + backend="filesystem", + sort_by_starting_time_asc=True, +) + +for backup in backups: + print(backup.backup_id, backup.status, backup.incremental_base_backup_id) +# END ListBackups + +# Test +backup_ids = [backup.backup_id for backup in backups] +assert {"base-backup", "incremental-backup-1", "incremental-backup-2"}.issubset(backup_ids) +bases = {backup.backup_id: backup.incremental_base_backup_id for backup in backups} +assert bases["base-backup"] is None +assert bases["incremental-backup-1"] == "base-backup" +assert bases["incremental-backup-2"] == "incremental-backup-1" + # Clean up client.collections.delete(["Article", "Publication"]) diff --git a/_includes/code/howto/go/docs/mainpkg/search-generative_test.go b/_includes/code/howto/go/docs/mainpkg/search-generative_test.go index f9aa1a4e0..807fd1d5d 100644 --- a/_includes/code/howto/go/docs/mainpkg/search-generative_test.go +++ b/_includes/code/howto/go/docs/mainpkg/search-generative_test.go @@ -21,7 +21,7 @@ func TestSingleGenerative(t *testing.T) { client := setupClient() ctx := context.Background() - // START SingleGenerativeProperties Go + // START SingleGenerativeBasic Go generatePrompt := "Convert the following into a question for twitter. Include emojis for fun, but do not include the answer: {question}." gs := graphql.NewGenerativeSearch().SingleResult(generatePrompt) @@ -36,7 +36,7 @@ func TestSingleGenerative(t *testing.T) { WithConcepts([]string{"World history"})). WithLimit(2). Do(ctx) - // END SingleGenerativeProperties Go + // END SingleGenerativeBasic Go require.NoError(t, err) @@ -109,7 +109,7 @@ func TestGroupedGenerative(t *testing.T) { client := setupClient() ctx := context.Background() - // START GroupedGenerative Go + // START GroupedGenerativeBasic Go generatePrompt := "What do these animals have in common, if anything?" gs := graphql.NewGenerativeSearch().GroupedResult(generatePrompt) @@ -124,7 +124,7 @@ func TestGroupedGenerative(t *testing.T) { WithConcepts([]string{"Cute animals"})). WithLimit(3). Do(ctx) - // END GroupedGenerative Go + // END GroupedGenerativeBasic Go require.NoError(t, err) diff --git a/_includes/code/howto/go/docs/manage-data.read-all-objects_test.go b/_includes/code/howto/go/docs/manage-data.read-all-objects_test.go index f571d4edb..37b12e7cd 100644 --- a/_includes/code/howto/go/docs/manage-data.read-all-objects_test.go +++ b/_includes/code/howto/go/docs/manage-data.read-all-objects_test.go @@ -22,6 +22,7 @@ func Test_ManageDataReadAllObjects(t *testing.T) { t.Run("Read all objects", func(t *testing.T) { // CursorExample // Retrieve data + // START ReadAllProps sourceClient, err := weaviate.NewClient(weaviate.Config{ Scheme: "https", Host: "WEAVIATE_INSTANCE_URL", // Replace WEAVIATE_INSTANCE_URL with your instance URL @@ -59,6 +60,7 @@ func Test_ManageDataReadAllObjects(t *testing.T) { } return get.Do(context.Background()) } + // END ReadAllProps // Use this function to retrieve data // START FetchClassDefinition diff --git a/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go b/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go index f7a44f959..400ddbc45 100644 --- a/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go +++ b/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go @@ -288,8 +288,8 @@ func main() { "title_vector": { Vectorizer: map[string]interface{}{ "text2vec-google": map[string]interface{}{ - "project_id": "", - "model_id": "gemini-embedding-001", // (Optional) To manually set the model ID + "projectId": "", + "modelId": "gemini-embedding-001", // (Optional) To manually set the model ID }, }, }, @@ -321,8 +321,9 @@ func main() { "title_vector": { Vectorizer: map[string]interface{}{ "text2vec-google": map[string]interface{}{ - "properties": []string{"title"}, - "model_id": "gemini-embedding-001", // (Optional) To manually set the model ID + "properties": []string{"title"}, + "apiEndpoint": "generativelanguage.googleapis.com", + "modelId": "gemini-embedding-001", // (Optional) To manually set the model ID }, }, }, @@ -353,11 +354,11 @@ func main() { VectorConfig: map[string]models.VectorConfig{ "title_vector": { Vectorizer: map[string]interface{}{ - "text2vec-aws": map[string]interface{}{ - "properties": []string{"title"}, - "project_id": "", // Required for Vertex AU - "model_id": "textembedding-gecko@latest", // (Optional) To manually set the model ID - "api_endpoint": "", // (Optional) To manually set the API endpoint + "text2vec-google": map[string]interface{}{ + "properties": []string{"title"}, + "projectId": "", // Required for Vertex AI + "modelId": "textembedding-gecko@latest", // (Optional) To manually set the model ID + "apiEndpoint": "", // (Optional) To manually set the API endpoint }, }, }, @@ -423,10 +424,9 @@ func main() { Vectorizer: map[string]interface{}{ "text2vec-huggingface": map[string]interface{}{ "properties": []string{"title"}, - // Note: Use only one of (`model`), (`passage_model` and `query_model`), or (`endpoint_url`) + // Note: Use only one of (`model`), (`passage_model`), or (`endpoint_url`) "model": "sentence-transformers/all-MiniLM-L6-v2", - // "passage_model": "sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base", // Required if using `query_model` - // "query_model": "sentence-transformers/facebook-dpr-question_encoder-single-nq-base", // Required if using `passage_model` + // "passage_model": "sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base", // "endpoint_url": "", // // Optional parameters // "wait_for_model": true, diff --git a/_includes/code/howto/manage-data.import.py b/_includes/code/howto/manage-data.import.py index fa191b5b7..ff0646011 100644 --- a/_includes/code/howto/manage-data.import.py +++ b/_includes/code/howto/manage-data.import.py @@ -157,38 +157,43 @@ # ===== Insert many with custom ID ===== # ======================================= +# Re-create the collection +client.collections.delete("MyCollection") +client.collections.create( + "MyCollection", + vector_config=Configure.Vectors.self_provided() +) + # START BatchImportWithIDExample # highlight-start from weaviate.util import generate_uuid5 # Generate a deterministic ID +from weaviate.classes.data import DataObject # highlight-end -# START BatchImportWithIDExample data_rows = [{"title": f"Object {i+1}"} for i in range(5)] collection = client.collections.use("MyCollection") # highlight-start -with collection.batch.fixed_size(batch_size=200) as batch: - for data_row in data_rows: - obj_uuid = generate_uuid5(data_row) - batch.add_object( - properties=data_row, - uuid=obj_uuid - ) +data_objects = [ + DataObject( + properties=data_row, + uuid=generate_uuid5(data_row) + ) + for data_row in data_rows +] + +result = collection.data.ingest(data_objects) # highlight-end - if batch.number_errors > 10: - print("Batch import stopped due to excessive errors.") - break -failed_objects = collection.batch.failed_objects -if failed_objects: - print(f"Number of failed imports: {len(failed_objects)}") - print(f"First failed object: {failed_objects[0]}") +if result.errors: + print(f"Number of failed imports: {len(result.errors)}") # END BatchImportWithIDExample -result = collection.aggregate.over_all(total_count=True) -assert result.total_count == 5 -resp_obj = collection.query.fetch_object_by_id(obj_uuid) +# Tests +agg_result = collection.aggregate.over_all(total_count=True) +assert agg_result.total_count == 5 +resp_obj = collection.query.fetch_object_by_id(generate_uuid5(data_rows[-1])) assert resp_obj != None # Clean up client.collections.delete(collection.name) @@ -197,32 +202,40 @@ # ===== Batch import with custom vector ===== # =========================================== +# Re-create the collection +client.collections.delete("MyCollection") +client.collections.create( + "MyCollection", + vector_config=Configure.Vectors.self_provided() +) + # START BatchImportWithVectorExample +from weaviate.classes.data import DataObject + data_rows = [{"title": f"Object {i+1}"} for i in range(5)] vectors = [[0.1] * 1536 for i in range(5)] collection = client.collections.use("MyCollection") # highlight-start -with collection.batch.fixed_size(batch_size=200) as batch: - for i, data_row in enumerate(data_rows): - batch.add_object( - properties=data_row, - vector=vectors[i] - ) +data_objects = [ + DataObject( + properties=data_row, + vector=vectors[i] + ) + for i, data_row in enumerate(data_rows) +] + +result = collection.data.ingest(data_objects) # highlight-end - if batch.number_errors > 10: - print("Batch import stopped due to excessive errors.") - break -failed_objects = collection.batch.failed_objects -if failed_objects: - print(f"Number of failed imports: {len(failed_objects)}") - print(f"First failed object: {failed_objects[0]}") +if result.errors: + print(f"Number of failed imports: {len(result.errors)}") # END BatchImportWithVectorExample -result = collection.aggregate.over_all(total_count=True) -assert result.total_count == 5 +# Tests +agg_result = collection.aggregate.over_all(total_count=True) +assert agg_result.total_count == 5 # Clean up client.collections.delete(collection.name) @@ -255,6 +268,8 @@ ) # START BatchImportWithNamedVectors +from weaviate.classes.data import DataObject + data_rows = [{ "title": f"Object {i+1}", "body": f"Body {i+1}" @@ -266,24 +281,22 @@ collection = client.collections.use("MyCollection") # highlight-start -with collection.batch.fixed_size(batch_size=200) as batch: - for i, data_row in enumerate(data_rows): - batch.add_object( - properties=data_row, - vector={ - "title": title_vectors[i], - "body": body_vectors[i], - } - ) +data_objects = [ + DataObject( + properties=data_row, + vector={ + "title": title_vectors[i], + "body": body_vectors[i], + } + ) + for i, data_row in enumerate(data_rows) +] + +result = collection.data.ingest(data_objects) # highlight-end - if batch.number_errors > 10: - print("Batch import stopped due to excessive errors.") - break -failed_objects = collection.batch.failed_objects -if failed_objects: - print(f"Number of failed imports: {len(failed_objects)}") - print(f"First failed object: {failed_objects[0]}") +if result.errors: + print(f"Number of failed imports: {len(result.errors)}") # END BatchImportWithNamedVectors response = collection.query.fetch_objects(include_vector=True) @@ -318,36 +331,37 @@ ] ) -from_uuid = authors.data.insert( - properties={"name": "Jane Austen"} -) - publications.data.insert( {"title": "Ye Olde Times"} ) target_uuid = publications.query.fetch_objects(limit=1).objects[0].uuid -# BatchImportWithRefExample +# START BatchImportWithRefExample +from weaviate.classes.data import DataObject + collection = client.collections.use("Author") -with collection.batch.fixed_size(batch_size=100) as batch: - batch.add_reference( - from_property="writesFor", - from_uuid=from_uuid, - to=target_uuid, - ) +# highlight-start +data_objects = [ + DataObject( + properties={"name": "Jane Austen"}, + references={"writesFor": target_uuid}, + ), +] + +result = collection.data.ingest(data_objects) +# highlight-end -failed_references = collection.batch.failed_references -if failed_references: - print(f"Number of failed imports: {len(failed_references)}") - print(f"First failed reference: {failed_references[0]}") +if result.errors: + print(f"Number of failed imports: {len(result.errors)}") # END BatchImportWithRefExample # Tests from weaviate.classes.query import QueryReference +new_uuid = result.uuids[0] response = collection.query.fetch_object_by_id( - from_uuid, + new_uuid, return_references=QueryReference(link_on="writesFor", return_properties=["title"]) ) @@ -574,7 +588,7 @@ def add_object(obj) -> None: {"title": f"Object {i+1}"} for i in range(5) ] -collection = client.collections.get("MyCollection") +collection = client.collections.use("MyCollection") # highlight-start # Use `stream` for server-side batching. The client will send data @@ -602,4 +616,99 @@ def add_object(obj) -> None: client.collections.delete(collection.name) +# ================================================== +# ===== Server-side ingest from a generator ===== +# ================================================== + +# Re-create the collection +client.collections.delete("MyCollection") +client.collections.create( + "MyCollection", + vector_config=Configure.Vectors.self_provided() +) + +# Create the source file used by the example below +with open("my-data.jsonl", "w") as f: + for i in range(5): + f.write(json.dumps({"title": f"Object {i+1}"}) + "\n") + f.write("\n") # A blank line, to show that the generator skips it + +# START ServerSideIngestGeneratorExample +import json + +# Each line of the source file holds one JSON object +def read_objects(path): + with open(path) as f: + for line in f: + line = line.strip() + if not line: # Skip blank lines + continue + record = json.loads(line) + yield {"title": record["title"]} + +collection = client.collections.use("MyCollection") + +# highlight-start +# `ingest` pulls objects from the generator as it goes +result = collection.data.ingest(read_objects("my-data.jsonl")) +# highlight-end + +if result.errors: + print(f"Number of failed imports: {len(result.errors)}") +# END ServerSideIngestGeneratorExample + +# Tests +assert len(result.errors) == 0 +assert len(result.uuids) == 5 + +agg_result = collection.aggregate.over_all(total_count=True) +assert agg_result.total_count == 5 + +# Clean up +client.collections.delete(collection.name) +os.remove("my-data.jsonl") + + +# ================================================== +# ===== Server-side one-shot ingest ===== +# ================================================== + +# Re-create the collection +client.collections.delete("MyCollection") +client.collections.create( + "MyCollection", + vector_config=Configure.Vectors.self_provided() +) + +# START ServerSideIngestExample +data_rows = [ + {"title": f"Object {i+1}"} for i in range(5) +] + +collection = client.collections.use("MyCollection") + +# highlight-start +# `ingest` imports the whole list with server-side batching in a single call +result = collection.data.ingest(data_rows) +# highlight-end + +# The return object is the same as for `insert_many` +if result.errors: + print(f"Number of failed imports: {len(result.errors)}") + # `errors` is a dict keyed by the index of the failed object + for index, error in result.errors.items(): + print(f"Failed object at index {index}: {error.message}") +# END ServerSideIngestExample + +# Tests +assert len(result.errors) == 0 +assert len(result.uuids) == 5 + +agg_result = collection.aggregate.over_all(total_count=True) +assert agg_result.total_count == 5 + +# Clean up +client.collections.delete(collection.name) + + client.close() diff --git a/_includes/code/howto/manage-data.import.ts b/_includes/code/howto/manage-data.import.ts index d5b438d2d..04c56fc55 100644 --- a/_includes/code/howto/manage-data.import.ts +++ b/_includes/code/howto/manage-data.import.ts @@ -111,7 +111,10 @@ let dataObjects = [ ] const myCollection = client.collections.use('MyCollection') -await myCollection.data.insertMany(dataObject) +// highlight-start +// `ingest` imports the list using server-side batching +await myCollection.data.ingest(dataObjects) +// highlight-end // END BatchImportWithIDExample // result = await client.graphql.aggregate().withClassName(className).withFields('meta { count }').do(); @@ -146,7 +149,10 @@ let dataObjects = [ // ... ] -await jeopardy.data.insertMany(dataObjects) +// highlight-start +// `ingest` imports the list using server-side batching +await myCollection.data.ingest(dataObjects) +// highlight-end // END BatchImportWithVectorExample // result = await client.graphql.aggregate().withClassName(className).withFields('meta { count }').do(); @@ -204,7 +210,10 @@ let dataObjects = [ // ... ] -await myCollection.data.insertMany(dataObjects) +// highlight-start +// `ingest` imports the list using server-side batching +await myCollection.data.ingest(dataObjects) +// highlight-end } // END BatchImportWithNamedVectors @@ -334,6 +343,53 @@ try { { // START ServerSideBatchImportExample +const myCollection = client.collections.use('MyCollection') + +// highlight-start +// `ingest` is the TypeScript server-side batching API. It accepts any +// Iterable, so passing a generator streams objects to the server +// without building the full list in memory. +function* generateData(): Generator<{ properties: { title: string } }> { + for (let i = 1; i <= 5; i++) { + yield { properties: { title: `Object ${i}` } } + } +} + +const result = await myCollection.data.ingest(generateData()) +// highlight-end + +console.log(result) +// END ServerSideBatchImportExample + +// Verify the import (not shown in the docs snippet): all 5 objects and +// their `title` property must have persisted. +const check = await myCollection.query.fetchObjects({ limit: 5 }) +if (check.objects.length !== 5) + throw new Error(`SSB import: expected 5 objects, got ${check.objects.length}`) +if (!check.objects.every((o) => typeof o.properties.title === 'string' && o.properties.title.length > 0)) + throw new Error('SSB import did not persist the title property') +} + +await client.collections.delete('MyCollection'); + +// ================================================== +// ===== Server-side one-shot ingest ===== +// ================================================== + +// Clean slate +try { + await client.collections.delete('MyCollection'); +} catch (e) { + // ignore error if class doesn't exist +} finally { + await client.collections.create({ + name: 'MyCollection', + vectorizers: weaviate.configure.vectors.selfProvided(), + }) +} + +{ +// START ServerSideIngestExample const dataObjects = [ { properties: { title: 'Object 1' } }, { properties: { title: 'Object 2' } }, @@ -345,21 +401,20 @@ const dataObjects = [ const myCollection = client.collections.use('MyCollection') // highlight-start -// Use `ingest` for server-side batching. The client sends data -// in batches at a rate controlled by the server. +// `ingest` imports the whole list with server-side batching in a single call const result = await myCollection.data.ingest(dataObjects) // highlight-end console.log(result) -// END ServerSideBatchImportExample +// END ServerSideIngestExample // Verify the import (not shown in the docs snippet): all 5 objects and // their `title` property must have persisted. const check = await myCollection.query.fetchObjects({ limit: 5 }) if (check.objects.length !== 5) - throw new Error(`SSB import: expected 5 objects, got ${check.objects.length}`) + throw new Error(`Ingest import: expected 5 objects, got ${check.objects.length}`) if (!check.objects.every((o) => typeof o.properties.title === 'string' && o.properties.title.length > 0)) - throw new Error('SSB import did not persist the title property') + throw new Error('Ingest import did not persist the title property') } await client.collections.delete('MyCollection'); diff --git a/_includes/code/howto/search.bm25.gql.py b/_includes/code/howto/search.bm25.gql.py index 18a69fd0a..328170f0b 100644 --- a/_includes/code/howto/search.bm25.gql.py +++ b/_includes/code/howto/search.bm25.gql.py @@ -72,4 +72,29 @@ gqlresponse = client.graphql_raw_query(gql_query) +gql_query = """ +# START BM25OperatorCrossPropertyAnd +{ + Get { + JeopardyQuestion( + limit: 3 + bm25: { + query: "Australian mammal cute" + # highlight-start + searchOperator: { + operator: AndCross, + } + # highlight-end + } + ) { + question + answer + } + } +} +# END BM25OperatorCrossPropertyAnd +""" + +gqlresponse = client.graphql_raw_query(gql_query) + client.close() diff --git a/_includes/code/howto/search.bm25.py b/_includes/code/howto/search.bm25.py index 696fae0b3..36ad62f4d 100644 --- a/_includes/code/howto/search.bm25.py +++ b/_includes/code/howto/search.bm25.py @@ -107,6 +107,36 @@ # End test +# ============================ +# ===== BM25 w/ AND_CROSS ===== +# ============================ + +# START BM25OperatorCrossPropertyAnd +# highlight-start +from weaviate.classes.query import BM25Operator +# highlight-end + +jeopardy = client.collections.use("JeopardyQuestion") +response = jeopardy.query.bm25( + # highlight-start + query="African desert wind", + # Each token must be matched by at least one searched property, + # but not necessarily all by the same property + operator=BM25Operator.and_cross(), + # highlight-end + limit=3, +) + +for o in response.objects: + print(o.properties) +# END BM25OperatorCrossPropertyAnd + + +# Tests +assert response.objects[0].collection == "JeopardyQuestion" +# End test + + # ================================================ # ===== BM25 Query with score / explainScore ===== # ================================================ diff --git a/_includes/code/howto/search.filters.py b/_includes/code/howto/search.filters.py index 10e924e2c..ec249425a 100644 --- a/_includes/code/howto/search.filters.py +++ b/_includes/code/howto/search.filters.py @@ -538,9 +538,8 @@ filters=Filter.by_property("country").is_none(True) # Find objects where the `country` property is null # highlight-end ) -print("despot. othing") for o in response.objects: - print("despot"+o.properties) # Inspect returned objects + print(o.properties) # Inspect returned objects # END FilterByPropertyNullState @@ -573,6 +572,24 @@ }, ) +# Test scaffolding below; not rendered in the docs. +# This block used to run against a local instance, where a write is queryable +# the moment `insert` returns. It now runs against a remote cluster, where it is +# not, so the example could query the collection before the object was visible +# and get back nothing. Wait (bounded) for the write to land, so the assertion +# tests the geo filter and not write visibility. The assertion stays exact. +import time +from weaviate.classes.query import Filter, GeoCoordinate + +readiness_filter = Filter.by_property("headquartersGeoLocation").within_geo_range( + coordinate=GeoCoordinate(latitude=52.39, longitude=4.84), + distance=1000 +) +for _ in range(30): + if len(publications.query.fetch_objects(filters=readiness_filter).objects) >= 1: + break + time.sleep(1) + # START FilterbyGeolocation from weaviate.classes.query import Filter from weaviate.classes.query import GeoCoordinate diff --git a/_includes/code/howto/search.similarity.mmr.py b/_includes/code/howto/search.similarity.mmr.py index b860bc6ca..feb486f2b 100644 --- a/_includes/code/howto/search.similarity.mmr.py +++ b/_includes/code/howto/search.similarity.mmr.py @@ -37,7 +37,7 @@ response = collection.query.near_vector( near_vector=base_vec, limit=20, - selection=Diversity.MMR( + diversity_selection=Diversity.mmr( limit=5, balance=0.5, ), @@ -75,7 +75,7 @@ response = collection.query.near_vector( near_vector=query_vector, limit=20, - selection=Diversity.MMR( + diversity_selection=Diversity.mmr( limit=5, balance=0.5, ), @@ -98,21 +98,21 @@ response_diverse = collection.query.near_vector( near_vector=base_vec, limit=20, - selection=Diversity.MMR(limit=5, balance=0.0), + diversity_selection=Diversity.mmr(limit=5, balance=0.0), ) # Balanced — equal weight on relevance and diversity response_balanced = collection.query.near_vector( near_vector=base_vec, limit=20, - selection=Diversity.MMR(limit=5, balance=0.5), + diversity_selection=Diversity.mmr(limit=5, balance=0.5), ) # Pure relevance — equivalent to standard vector search response_relevant = collection.query.near_vector( near_vector=base_vec, limit=20, - selection=Diversity.MMR(limit=5, balance=1.0), + diversity_selection=Diversity.mmr(limit=5, balance=1.0), ) # END MMRBalanceExamples @@ -126,6 +126,83 @@ relevant_questions = [o.properties["question"] for o in response_relevant.objects] assert diverse_questions != relevant_questions, "Pure diversity and pure relevance should differ" +# START MMRHybridExample +from weaviate.classes.query import Diversity + +collection = client.collections.get("MMRDemo") + +# Fuse the keyword and vector results into 20 candidates, then select 5 diverse results +response = collection.query.hybrid( + query="Question", + vector=base_vec, + limit=20, + # highlight-start + diversity_selection=Diversity.mmr( + limit=5, + balance=0.5, + ), + # highlight-end +) + +for o in response.objects: + print(o.properties["question"]) +# END MMRHybridExample + +# Test +assert len(response.objects) == 5 +hybrid_mmr_questions = [o.properties["question"] for o in response.objects] + +hybrid_standard = collection.query.hybrid( + query="Question", + vector=base_vec, + limit=5, +) +hybrid_standard_questions = [o.properties["question"] for o in hybrid_standard.objects] +assert set(hybrid_standard_questions) != set( + hybrid_mmr_questions +), "MMR should select different items than standard hybrid search" + +# START MMRPagination +from weaviate.classes.query import Diversity + +collection = client.collections.get("MMRDemo") + +# The query limit is the size of the window that gets diversified. +# The diversity limit is the page size. +query_limit = 10 +page_size = 3 + +# Advance offset by the query limit, NOT by the number of returned objects +for offset in range(0, 30, query_limit): + page = collection.query.near_vector( + near_vector=base_vec, + limit=query_limit, + # highlight-start + offset=offset, + diversity_selection=Diversity.mmr(limit=page_size, balance=0.5), + # highlight-end + ) + + for o in page.objects: + print(offset, o.properties["question"]) +# END MMRPagination + +# Test +paged_questions = [] +for offset in range(0, 30, query_limit): + page = collection.query.near_vector( + near_vector=base_vec, + limit=query_limit, + offset=offset, + diversity_selection=Diversity.mmr(limit=page_size, balance=0.5), + ) + paged_questions += [o.properties["question"] for o in page.objects] + +assert len(paged_questions) == 9 +assert len(set(paged_questions)) == len( + paged_questions +), "Advancing offset by the query limit must not repeat objects across pages" + # Cleanup client.collections.delete("MMRDemo") client.close() diff --git a/_includes/code/java-v6/src/test/java/ConnectionTest.java b/_includes/code/java-v6/src/test/java/ConnectionTest.java index c44ee6e37..987b62b3d 100644 --- a/_includes/code/java-v6/src/test/java/ConnectionTest.java +++ b/_includes/code/java-v6/src/test/java/ConnectionTest.java @@ -151,7 +151,7 @@ void testCustomApiKeyConnection() throws Exception { System.out.println(client.isReady()); // Should print: `True` client.close(); // Free up resources - // // END ConnectWithApiKeyExample + // END ConnectWithApiKeyExample } @Test diff --git a/_includes/code/java-v6/src/test/java/GetStartedTest.java b/_includes/code/java-v6/src/test/java/GetStartedTest.java index fd71c41fb..439838e2a 100644 --- a/_includes/code/java-v6/src/test/java/GetStartedTest.java +++ b/_includes/code/java-v6/src/test/java/GetStartedTest.java @@ -39,7 +39,11 @@ void testGetStartedWorkflow() throws Exception { col -> col.properties(Property.text("answer"), Property.text("question"), Property.text("category")) - .vectorConfig(VectorConfig.text2vecTransformers()) // Configure the Contextionary embedding model + .vectorConfig(VectorConfig.text2vecOllama(v -> v + // If using Docker you might need: http://host.docker.internal:11434 + .apiEndpoint("http://ollama:11434") + .model("nomic-embed-text") // The model to use + )) // Configure the Ollama embedding model ); CollectionHandle> questions = client.collections.use(collectionName); // highlight-end diff --git a/_includes/code/java-v6/src/test/java/ManageObjectsImportTest.java b/_includes/code/java-v6/src/test/java/ManageObjectsImportTest.java index 4e0c5140f..0211cb63a 100644 --- a/_includes/code/java-v6/src/test/java/ManageObjectsImportTest.java +++ b/_includes/code/java-v6/src/test/java/ManageObjectsImportTest.java @@ -7,7 +7,6 @@ import io.weaviate.client6.v1.api.collections.data.BatchReference; import io.weaviate.client6.v1.api.collections.Vectors; import io.weaviate.client6.v1.api.collections.WeaviateObject; -import io.weaviate.client6.v1.api.collections.data.InsertManyResponse; import io.weaviate.client6.v1.api.collections.query.QueryReference; import org.junit.jupiter.api.AfterAll; @@ -158,32 +157,35 @@ void testBatchImportWithID() throws IOException { col -> col.vectorConfig(VectorConfig.selfProvided())); // START BatchImportWithIDExample - List>> dataObjects = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - Map dataRow = Map.of("title", "Object " + (i + 1)); - UUID objUuid = generateUuid5(dataRow.toString()); - - dataObjects.add(WeaviateObject.>of( - obj -> obj.properties(dataRow).uuid(objUuid.toString()))); - } - var collection = client.collections.use("MyCollection"); + // Add objects with custom IDs to a server-side batch import. // highlight-start - var response = collection.data.insertMany(dataObjects); + BatchContext> batch = collection.batch.start(); + try (batch) { + for (int i = 0; i < 5; i++) { + Map dataRow = Map.of("title", "Object " + (i + 1)); + UUID objUuid = generateUuid5(dataRow.toString()); + + batch.add(WeaviateObject.>of( + obj -> obj.properties(dataRow).uuid(objUuid.toString()))); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } // highlight-end - if (!response.errors().isEmpty()) { + if (batch.numberOfErrors() > 0) { System.err - .println("Number of failed imports: " + response.errors().size()); - System.err.println("First failed object: " + response.errors().get(0)); + .println("Number of failed imports: " + batch.numberOfErrors()); } // END BatchImportWithIDExample var result = collection.aggregate.overAll(agg -> agg.includeTotalCount(true)); assertThat(result.totalCount()).isEqualTo(5); - String lastUuid = dataObjects.get(4).uuid(); + String lastUuid = + generateUuid5(Map.of("title", "Object 5").toString()).toString(); assertThat(collection.data.exists(lastUuid)).isTrue(); client.collections.delete("MyCollection"); @@ -195,30 +197,29 @@ void testBatchImportWithVector() throws IOException { col -> col.vectorConfig(VectorConfig.selfProvided())); // START BatchImportWithVectorExample - List>> dataObjects = new ArrayList<>(); float[] vector = new float[10]; // Using a small vector for demonstration Arrays.fill(vector, 0.1f); - for (int i = 0; i < 5; i++) { - Map dataRow = Map.of("title", "Object " + (i + 1)); - UUID objUuid = generateUuid5(dataRow.toString()); - - dataObjects.add( - WeaviateObject.>of(obj -> obj.properties(dataRow) - .uuid(objUuid.toString()) - .vectors(Vectors.of(vector)))); - } - var collection = client.collections.use("MyCollection"); + // Add objects with custom vectors to a server-side batch import. // highlight-start - var response = collection.data.insertMany(dataObjects); + BatchContext> batch = collection.batch.start(); + try (batch) { + for (int i = 0; i < 5; i++) { + Map dataRow = Map.of("title", "Object " + (i + 1)); + + batch.add(WeaviateObject.>of( + obj -> obj.properties(dataRow).vectors(Vectors.of(vector)))); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } // highlight-end - if (!response.errors().isEmpty()) { + if (batch.numberOfErrors() > 0) { System.err - .println("Number of failed imports: " + response.errors().size()); - System.err.println("First failed object: " + response.errors().get(0)); + .println("Number of failed imports: " + batch.numberOfErrors()); } // END BatchImportWithVectorExample @@ -297,32 +298,26 @@ void testImportWithNamedVectors() throws IOException { CollectionHandle> collection = client.collections.use("MyCollection"); - List>> objectsToInsert = - new ArrayList<>(); - for (int i = 0; i < dataRows.size(); i++) { - int index = i; - objectsToInsert.add( - // highlight-start - // Use the Builder with the EXACT matching generic types - WeaviateObject - .>of(v -> v.properties(dataRows.get(index)) - .vectors(Vectors.of("title", titleVectors.get(index))) - .vectors(Vectors.of("body", bodyVectors.get(index))))); - // highlight-end - - } - - // Insert the data using insertMany with the List + // Add objects with named vectors to a server-side batch import. // highlight-start - InsertManyResponse response = collection.data.insertMany(objectsToInsert); + BatchContext> batch = collection.batch.start(); + try (batch) { + for (int i = 0; i < dataRows.size(); i++) { + int index = i; + batch.add(WeaviateObject + .>of(v -> v.properties(dataRows.get(index)) + .vectors(Vectors.of("title", titleVectors.get(index))) + .vectors(Vectors.of("body", bodyVectors.get(index))))); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } // highlight-end // Check for errors - if (!response.errors().isEmpty()) { + if (batch.numberOfErrors() > 0) { System.err.printf("Number of failed imports: %d\n", - response.errors().size()); - System.err.printf("First failed object error: %s\n", - response.errors().get(0)); + batch.numberOfErrors()); } // END BatchImportWithNamedVectors } diff --git a/_includes/code/java-v6/src/test/java/QuickstartLocalTest.java b/_includes/code/java-v6/src/test/java/QuickstartLocalTest.java index c067883b9..0a80c19ca 100644 --- a/_includes/code/java-v6/src/test/java/QuickstartLocalTest.java +++ b/_includes/code/java-v6/src/test/java/QuickstartLocalTest.java @@ -3,7 +3,8 @@ import io.weaviate.client6.v1.api.collections.Generative; import io.weaviate.client6.v1.api.collections.Property; import io.weaviate.client6.v1.api.collections.VectorConfig; -import io.weaviate.client6.v1.api.collections.data.InsertManyResponse; +import io.weaviate.client6.v1.api.collections.WeaviateObject; +import io.weaviate.client6.v1.api.collections.batch.BatchContext; import org.json.JSONArray; import org.json.JSONObject; @@ -94,16 +95,21 @@ void testImportDataWorkflow() throws Exception { questionsToInsert.add(properties); }); - // Call insertMany with the list of objects - InsertManyResponse insertResponse = questions.data.insertMany(questionsToInsert.toArray(new Map[0])); + // `batch.start()` opens a server-side batch + BatchContext> batch = questions.batch.start(); + // Closing the batch sends the remaining objects and waits for the results + try (batch) { + for (Map properties : questionsToInsert) { + batch.add(WeaviateObject.>of(o -> o.properties(properties))); + } + } // highlight-end // Check for errors - if (!insertResponse.errors().isEmpty()) { - System.err.printf("Number of failed imports: %d\n", insertResponse.errors().size()); - System.err.printf("First failed object error: %s\n", insertResponse.errors().get(0)); + if (batch.numberOfErrors() > 0) { + System.err.printf("Number of failed imports: %d\n", batch.numberOfErrors()); } else { - System.out.printf("Successfully inserted %d objects.\n", insertResponse.uuids().size()); + System.out.printf("Successfully inserted %d objects.\n", questionsToInsert.size()); } // END Import // client.collections.delete(collectionName); diff --git a/_includes/code/java-v6/src/test/java/QuickstartTest.java b/_includes/code/java-v6/src/test/java/QuickstartTest.java index 3de3fa0b5..6732689b3 100644 --- a/_includes/code/java-v6/src/test/java/QuickstartTest.java +++ b/_includes/code/java-v6/src/test/java/QuickstartTest.java @@ -3,7 +3,9 @@ import io.weaviate.client6.v1.api.collections.Generative; import io.weaviate.client6.v1.api.collections.Property; import io.weaviate.client6.v1.api.collections.VectorConfig; -import io.weaviate.client6.v1.api.collections.data.InsertManyResponse; +import io.weaviate.client6.v1.api.collections.WeaviateObject; +import io.weaviate.client6.v1.api.collections.batch.BatchContext; +import io.weaviate.client6.v1.api.collections.generate.GenerativeProvider; import org.json.JSONArray; import org.json.JSONObject; @@ -59,7 +61,7 @@ void testCreateCollection() throws Exception { collectionName, col -> col .vectorConfig(VectorConfig.text2vecWeaviate()) // Configure the Weaviate Embeddings integration - .generativeModule(Generative.cohere()) // Configure the Cohere generative AI integration + .generativeModule(Generative.openai()) // Configure the OpenAI generative AI integration ); CollectionHandle> questions = client.collections.use(collectionName); // highlight-end @@ -115,16 +117,21 @@ void testImportDataWorkflow() throws Exception { questionsToInsert.add(properties); }); - // Call insertMany with the list of objects - InsertManyResponse insertResponse = questions.data.insertMany(questionsToInsert.toArray(new Map[0])); + // `batch.start()` opens a server-side batch + BatchContext> batch = questions.batch.start(); + // Closing the batch sends the remaining objects and waits for the results + try (batch) { + for (Map properties : questionsToInsert) { + batch.add(WeaviateObject.>of(o -> o.properties(properties))); + } + } // highlight-end // Check for errors - if (!insertResponse.errors().isEmpty()) { - System.err.printf("Number of failed imports: %d\n", insertResponse.errors().size()); - System.err.printf("First failed object error: %s\n", insertResponse.errors().get(0)); + if (batch.numberOfErrors() > 0) { + System.err.printf("Number of failed imports: %d\n", batch.numberOfErrors()); } else { - System.out.printf("Successfully inserted %d objects.\n", insertResponse.uuids().size()); + System.out.printf("Successfully inserted %d objects.\n", questionsToInsert.size()); } // END Import // client.collections.delete(collectionName); @@ -164,31 +171,67 @@ void testNearTextQuery() throws Exception { // END NearText } - // @Test - // void testRagQuery() { - // // Best practice: store your credentials in environment variables - // String weaviateUrl = System.getenv("WEAVIATE_URL"); - // String weaviateApiKey = System.getenv("WEAVIATE_API_KEY"); - - // WeaviateClient client = WeaviateClient.connectToWeaviateCloud( - // weaviateUrl, // Replace with your Weaviate Cloud URL - // weaviateApiKey // Replace with your Weaviate Cloud key - // ); - - // var questions = client.collections.use("Question"); - - // // highlight-start - // var response = questions.generate.nearText( - // q -> q - // .query("biology") - // .limit(2), - // g -> g.groupedTask("Write a tweet with emojis about these facts.")); - // // highlight-end - - // System.out.println(response.generative().text()); // Inspect the generated - // text - // } - // START RAG - // Coming soon - // END RAG + @Test + void testRagQuery() throws Exception { + // Setup, not shown in the docs: build the `Question` collection this example + // queries, so the test does not depend on the order the other tests run in. + // The collection is deliberately left in place: `testCreateCollection` and + // `testImportDataWorkflow` both delete it before they recreate it. + WeaviateClient setupClient = WeaviateClient.connectToWeaviateCloud( + System.getenv("WEAVIATE_URL"), + System.getenv("WEAVIATE_API_KEY")); + String setupCollectionName = "Question"; + if (setupClient.collections.exists(setupCollectionName)) { + setupClient.collections.delete(setupCollectionName); + } + setupClient.collections.create(setupCollectionName, col -> col + .properties( + Property.text("answer"), + Property.text("question"), + Property.text("category")) + .vectorConfig(VectorConfig.text2vecWeaviate())); + setupClient.collections.use(setupCollectionName).data.insertMany( + Map.of("answer", "DNA", + "question", "In 1953 Watson & Crick built a model of the molecular structure of this, the gene-carrying substance", + "category", "SCIENCE"), + Map.of("answer", "Liver", + "question", "This organ removes excess glucose from the blood & stores it as glycogen", + "category", "SCIENCE")); + Thread.sleep(3000); // Give the vectorizer time to index the new objects + setupClient.close(); + + // START RAG + // Best practice: store your credentials in environment variables + String weaviateUrl = System.getenv("WEAVIATE_URL"); + String weaviateApiKey = System.getenv("WEAVIATE_API_KEY"); + String openaiApiKey = System.getenv("OPENAI_API_KEY"); + + // highlight-start + WeaviateClient client = WeaviateClient.connectToWeaviateCloud( + weaviateUrl, // Replace with your Weaviate Cloud URL + weaviateApiKey, // Replace with your Weaviate Cloud key + config -> config.setHeaders( + Map.of("X-OpenAI-Api-Key", openaiApiKey)) // Replace with your OpenAI API key + ); + // highlight-end + + CollectionHandle> questions = client.collections.use("Question"); + + // highlight-start + var response = questions.generate.nearText( + "biology", + // Query configuration (nearText and limit) + q -> q.limit(2), + // Generative configuration (the RAG task) + g -> g.groupedTask( + "Write a tweet with emojis about these facts.", + c -> c.generativeProvider(GenerativeProvider.openai(o -> o)))); + // highlight-end + + // Use `.generative()` to access the generated text + System.out.println(response.generative().text()); + + client.close(); // Free up resources + // END RAG + } } \ No newline at end of file diff --git a/_includes/code/java-v6/src/test/java/RBACTest.java b/_includes/code/java-v6/src/test/java/RBACTest.java index 60acb2d01..5a20589b4 100644 --- a/_includes/code/java-v6/src/test/java/RBACTest.java +++ b/_includes/code/java-v6/src/test/java/RBACTest.java @@ -14,6 +14,7 @@ import io.weaviate.client6.v1.api.rbac.TenantsPermission; import io.weaviate.client6.v1.api.rbac.UsersPermission; import io.weaviate.client6.v1.api.rbac.groups.GroupType; +import io.weaviate.client6.v1.api.rbac.roles.GroupAssignment; import io.weaviate.client6.v1.api.rbac.users.DbUser; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -428,4 +429,85 @@ void testUserLifecycle() throws IOException { // END DeleteUser assertThat(client.users.db.byName(testUser)).isEmpty(); } + + @Test + void testOidcUserLifecycle() throws IOException { + // An OIDC user is authenticated by the identity provider, so it is never + // created in Weaviate. Only its role assignments are managed here. + String testUser = "custom-user"; + String testRole = "testRole"; + + Permission[] permissions = new Permission[] {Permission + .collections("TargetCollection*", CollectionsPermission.Action.READ),}; + client.roles.create(testRole, permissions); + + // START AssignOidcUserRole + client.users.oidc.assignRoles(testUser, testRole, "viewer"); + // END AssignOidcUserRole + + // START ListOidcUserRoles + var oidcUserRoles = client.users.oidc.assignedRoles(testUser); + for (Role role : oidcUserRoles) { + System.out.println(role.name()); + } + // END ListOidcUserRoles + assertThat(oidcUserRoles).extracting(Role::name).contains(testRole, "viewer"); + + // START RevokeOidcUserRoles + client.users.oidc.revokeRoles(testUser, testRole); + // END RevokeOidcUserRoles + assertThat(client.users.oidc.assignedRoles(testUser)).extracting(Role::name) + .doesNotContain(testRole) + .contains("viewer"); + + // Leave no assignment behind for the next test run + client.users.oidc.revokeRoles(testUser, "viewer"); + } + + @Test + void testOidcGroupLifecycle() throws IOException { + String testGroup = "/admin-group"; + String testRole = "testRole"; + + Permission[] permissions = new Permission[] {Permission + .collections("TargetCollection*", CollectionsPermission.Action.READ),}; + client.roles.create(testRole, permissions); + + // START AssignOidcGroupRoles + client.groups.assignRoles(testGroup, testRole, "viewer"); + // END AssignOidcGroupRoles + + // START GetOidcGroupRoles + List groupRoles = client.groups.assignedRoles(testGroup, + g -> g.includePermissions(true)); + for (Role role : groupRoles) { + System.out.println(role.name()); + } + // END GetOidcGroupRoles + assertThat(groupRoles).extracting(Role::name).contains(testRole, "viewer"); + + // START GetKnownOidcGroups + List knownGroups = client.groups.knownGroupNames(); + System.out.println("Known OIDC groups (" + knownGroups.size() + "): " + knownGroups); + // END GetKnownOidcGroups + assertThat(knownGroups).contains(testGroup); + + // START GetGroupAssignments + List groupAssignments = client.roles.groupAssignments(testRole); + System.out.println("Groups assigned to role '" + testRole + "':"); + for (GroupAssignment assignment : groupAssignments) { + System.out.println(" - Group ID: " + assignment.groupId() + ", Type: " + + assignment.groupType()); + } + // END GetGroupAssignments + assertThat(groupAssignments).extracting(GroupAssignment::groupId) + .contains(testGroup); + assertThat(groupAssignments).extracting(GroupAssignment::groupType) + .contains(GroupType.OIDC); + + // START RevokeOidcGroupRoles + client.groups.revokeRoles(testGroup, testRole, "viewer"); + // END RevokeOidcGroupRoles + assertThat(client.groups.assignedRoles(testGroup)).isEmpty(); + } } diff --git a/_includes/code/java-v6/src/test/java/SearchGenerativeTest.java b/_includes/code/java-v6/src/test/java/SearchGenerativeTest.java index 1fdf34f53..23086acb7 100644 --- a/_includes/code/java-v6/src/test/java/SearchGenerativeTest.java +++ b/_includes/code/java-v6/src/test/java/SearchGenerativeTest.java @@ -136,7 +136,7 @@ void testSingleGenerativeProperties() { @Test void testSingleGenerativeParameters() { - // START SingleGenerativeParametersPython + // START SingleGenerativeParameters CollectionHandle> jeopardy = client.collections.use("JeopardyQuestion"); var response = jeopardy.generate.nearText("World history", q -> q.limit(2), @@ -158,12 +158,12 @@ void testSingleGenerativeParameters() { System.out.printf("Debug: %s\n", o.generative().debug()); System.out.printf("Metadata: %s\n", o.generative().metadata()); } - // END SingleGenerativeParametersPython + // END SingleGenerativeParameters } @Test void testGroupedGenerative() { - // START GroupedGenerativePython + // START GroupedGenerativeBasic // highlight-start String task = "What do these animals have in common, if anything?"; // highlight-end @@ -179,7 +179,7 @@ void testGroupedGenerative() { // print the generated response System.out.printf("Grouped task result: %s\n", response.generative().text()); - // END GroupedGenerativePython + // END GroupedGenerativeBasic } @Test diff --git a/_includes/code/java-v6/src/test/java/StarterGuidesGenerativeTest.java b/_includes/code/java-v6/src/test/java/StarterGuidesGenerativeTest.java index d8da5bfa9..0d7bd0f7b 100644 --- a/_includes/code/java-v6/src/test/java/StarterGuidesGenerativeTest.java +++ b/_includes/code/java-v6/src/test/java/StarterGuidesGenerativeTest.java @@ -24,6 +24,7 @@ class StarterGuidesGenerativeTest { private final ObjectMapper objectMapper = new ObjectMapper(); + // START ChunkText private List downloadAndChunk(String srcUrl, int chunkSize, int overlapSize) throws Exception { // Retrieve source text @@ -64,12 +65,14 @@ void testFullWorkflow() throws Exception { // === Connect to Local and Setup GitBookChunk Collection ======== // ================================================================= - // Re-instantiate for writing data (Connect to Local) + // START Instantiation + // Pass the API key for your LLM provider, OpenAI in this case, as a header client = WeaviateClient.connectToLocal(config -> config.setHeaders( Map.of("X-OpenAI-Api-Key", System.getenv("OPENAI_API_KEY")))); + // END Instantiation assertThat(client.isReady()).isTrue(); - // ChunkText + // Download and chunk the source text String proGitChapterUrl = "https://raw.githubusercontent.com/progit/progit2/main/book/01-introduction/sections/what-is-git.asc"; List chunkedText = downloadAndChunk(proGitChapterUrl, 150, 25); diff --git a/_includes/code/java-v6/src/test/java/quickstart/QuickstartCreate.java b/_includes/code/java-v6/src/test/java/quickstart/QuickstartCreate.java index 745c1049f..d451389b1 100644 --- a/_includes/code/java-v6/src/test/java/quickstart/QuickstartCreate.java +++ b/_includes/code/java-v6/src/test/java/quickstart/QuickstartCreate.java @@ -5,7 +5,8 @@ import io.weaviate.client6.v1.api.collections.CollectionHandle; import io.weaviate.client6.v1.api.collections.Property; import io.weaviate.client6.v1.api.collections.VectorConfig; -import io.weaviate.client6.v1.api.collections.data.InsertManyResponse; +import io.weaviate.client6.v1.api.collections.WeaviateObject; +import io.weaviate.client6.v1.api.collections.batch.BatchContext; import java.util.List; import java.util.Map; @@ -54,18 +55,24 @@ public static void main(String[] args) throws Exception { "A meek Hobbit and his companions set out on a perilous journey to destroy a powerful ring and save Middle-earth.", "genre", "Fantasy")); - // Insert objects using insertMany + // Insert the objects using server-side batching CollectionHandle> movies = client.collections.use(collectionName); - InsertManyResponse insertResponse = - movies.data.insertMany(dataObjects.toArray(new Map[0])); + BatchContext> batch = movies.batch.start(); + // Closing the batch sends the remaining objects and waits for the results + try (batch) { + for (Map properties : dataObjects) { + batch.add( + WeaviateObject.>of(o -> o.properties(properties))); + } + } - if (!insertResponse.errors().isEmpty()) { - System.err.println("Errors during import: " + insertResponse.errors()); + if (batch.numberOfErrors() > 0) { + System.err + .println("Number of failed imports: " + batch.numberOfErrors()); } else { - System.out - .println("Imported & vectorized " + insertResponse.uuids().size() - + " objects into the Movie collection"); + System.out.println("Imported & vectorized " + dataObjects.size() + + " objects into the Movie collection"); } } finally { if (client != null) { diff --git a/_includes/code/java-v6/src/test/java/quickstart/QuickstartCreateVectors.java b/_includes/code/java-v6/src/test/java/quickstart/QuickstartCreateVectors.java index 6a6d930d8..e1f208d41 100644 --- a/_includes/code/java-v6/src/test/java/quickstart/QuickstartCreateVectors.java +++ b/_includes/code/java-v6/src/test/java/quickstart/QuickstartCreateVectors.java @@ -7,7 +7,8 @@ import io.weaviate.client6.v1.api.collections.VectorConfig; import io.weaviate.client6.v1.api.collections.Vectors; import io.weaviate.client6.v1.api.collections.WeaviateObject; -import io.weaviate.client6.v1.api.collections.data.InsertManyResponse; +import io.weaviate.client6.v1.api.collections.batch.BatchContext; +import java.util.List; import java.util.Map; public class QuickstartCreateVectors { @@ -63,20 +64,30 @@ public static void main(String[] args) throws Exception { float[] vector3 = new float[] {0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f, 0.9f, 1.0f}; - // Insert the objects with vectors + // Insert the objects with vectors using server-side batching CollectionHandle> movies = client.collections.use(collectionName); - InsertManyResponse insertResponse = movies.data.insertMany( + List>> objectsToInsert = List.of( WeaviateObject.of(v -> v.properties(props1) .vectors(Vectors.of(vector1))), WeaviateObject.of(v -> v.properties(props2) .vectors(Vectors.of(vector2))), WeaviateObject.of(v -> v.properties(props3) .vectors(Vectors.of(vector3)))); - if (!insertResponse.errors().isEmpty()) { - System.err.println("Errors during import: " + insertResponse.errors()); + + BatchContext> batch = movies.batch.start(); + // Closing the batch sends the remaining objects and waits for the results + try (batch) { + for (WeaviateObject> object : objectsToInsert) { + batch.add(object); + } + } + + if (batch.numberOfErrors() > 0) { + System.err + .println("Number of failed imports: " + batch.numberOfErrors()); } else { - System.out.println("Imported " + insertResponse.uuids().size() + System.out.println("Imported " + objectsToInsert.size() + " objects with vectors into the Movie collection"); } } finally { diff --git a/_includes/code/java-v6/src/test/java/quickstart/QuickstartLocalCreate.java b/_includes/code/java-v6/src/test/java/quickstart/QuickstartLocalCreate.java index 16e7149be..6b6b45477 100644 --- a/_includes/code/java-v6/src/test/java/quickstart/QuickstartLocalCreate.java +++ b/_includes/code/java-v6/src/test/java/quickstart/QuickstartLocalCreate.java @@ -5,7 +5,8 @@ import io.weaviate.client6.v1.api.collections.CollectionHandle; import io.weaviate.client6.v1.api.collections.Property; import io.weaviate.client6.v1.api.collections.VectorConfig; -import io.weaviate.client6.v1.api.collections.data.InsertManyResponse; +import io.weaviate.client6.v1.api.collections.WeaviateObject; +import io.weaviate.client6.v1.api.collections.batch.BatchContext; import java.util.List; import java.util.Map; @@ -53,18 +54,24 @@ public static void main(String[] args) throws Exception { "A meek Hobbit and his companions set out on a perilous journey to destroy a powerful ring and save Middle-earth.", "genre", "Fantasy")); - // Insert objects using insertMany + // Insert the objects using server-side batching CollectionHandle> movies = client.collections.use(collectionName); - InsertManyResponse insertResponse = - movies.data.insertMany(dataObjects.toArray(new Map[0])); + BatchContext> batch = movies.batch.start(); + // Closing the batch sends the remaining objects and waits for the results + try (batch) { + for (Map properties : dataObjects) { + batch.add( + WeaviateObject.>of(o -> o.properties(properties))); + } + } - if (!insertResponse.errors().isEmpty()) { - System.err.println("Errors during import: " + insertResponse.errors()); + if (batch.numberOfErrors() > 0) { + System.err + .println("Number of failed imports: " + batch.numberOfErrors()); } else { - System.out - .println("Imported & vectorized " + insertResponse.uuids().size() - + " objects into the Movie collection"); + System.out.println("Imported & vectorized " + dataObjects.size() + + " objects into the Movie collection"); } } finally { if (client != null) { diff --git a/_includes/code/java-v6/src/test/java/quickstart/QuickstartLocalCreateVectors.java b/_includes/code/java-v6/src/test/java/quickstart/QuickstartLocalCreateVectors.java index f31eaad3d..66c88691d 100644 --- a/_includes/code/java-v6/src/test/java/quickstart/QuickstartLocalCreateVectors.java +++ b/_includes/code/java-v6/src/test/java/quickstart/QuickstartLocalCreateVectors.java @@ -7,7 +7,8 @@ import io.weaviate.client6.v1.api.collections.VectorConfig; import io.weaviate.client6.v1.api.collections.Vectors; import io.weaviate.client6.v1.api.collections.WeaviateObject; -import io.weaviate.client6.v1.api.collections.data.InsertManyResponse; +import io.weaviate.client6.v1.api.collections.batch.BatchContext; +import java.util.List; import java.util.Map; public class QuickstartLocalCreateVectors { @@ -59,10 +60,10 @@ public static void main(String[] args) throws Exception { float[] vector3 = new float[] {0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f, 0.9f, 1.0f}; - // Insert the objects with vectors + // Insert the objects with vectors using server-side batching CollectionHandle> movies = client.collections.use(collectionName); - InsertManyResponse insertResponse = movies.data.insertMany( + List>> objectsToInsert = List.of( WeaviateObject.of(v -> v.properties(props1) .vectors(Vectors.of(vector1))), WeaviateObject.of(v -> v.properties(props2) @@ -70,10 +71,19 @@ public static void main(String[] args) throws Exception { WeaviateObject.of(v -> v.properties(props3) .vectors(Vectors.of(vector3)))); - if (!insertResponse.errors().isEmpty()) { - System.err.println("Errors during import: " + insertResponse.errors()); + BatchContext> batch = movies.batch.start(); + // Closing the batch sends the remaining objects and waits for the results + try (batch) { + for (WeaviateObject> object : objectsToInsert) { + batch.add(object); + } + } + + if (batch.numberOfErrors() > 0) { + System.err + .println("Number of failed imports: " + batch.numberOfErrors()); } else { - System.out.println("Imported " + insertResponse.uuids().size() + System.out.println("Imported " + objectsToInsert.size() + " objects with vectors into the Movie collection"); } } finally { diff --git a/_includes/code/llms-txt/python/quickstart.py b/_includes/code/llms-txt/python/quickstart.py index 30ad432ff..7f46cb2c1 100644 --- a/_includes/code/llms-txt/python/quickstart.py +++ b/_includes/code/llms-txt/python/quickstart.py @@ -46,9 +46,7 @@ movies = client.collections.use("Movie__QuickstartPy") # Import objects - with movies.batch.fixed_size(batch_size=200) as batch: - for obj in data_objects: - batch.add_object(properties=obj) + movies.data.ingest(data_objects) print(f"Imported & vectorized {len(data_objects)} objects into the Movie collection") diff --git a/_includes/code/llms-txt/typescript/quickstart.ts b/_includes/code/llms-txt/typescript/quickstart.ts index a83e17a1e..3270ee824 100644 --- a/_includes/code/llms-txt/typescript/quickstart.ts +++ b/_includes/code/llms-txt/typescript/quickstart.ts @@ -36,7 +36,7 @@ if (!(await client.collections.exists('Movie__QuickstartTs'))) { const movies = client.collections.use('Movie__QuickstartTs'); // Import objects -await movies.data.insertMany(dataObjects); +await movies.data.ingest(dataObjects.map((properties) => ({ properties }))); console.log(`Imported & vectorized ${dataObjects.length} objects into the Movie collection`); diff --git a/_includes/code/python/local.quickstart.import_objects.py b/_includes/code/python/local.quickstart.import_objects.py index 9f9c3d7b4..d46b4dd1c 100644 --- a/_includes/code/python/local.quickstart.import_objects.py +++ b/_includes/code/python/local.quickstart.import_objects.py @@ -9,27 +9,26 @@ ) data = json.loads(resp.text) -# highlight-start questions = client.collections.use("Question") -with questions.batch.fixed_size(batch_size=200) as batch: - for d in data: - batch.add_object( - { - "answer": d["Answer"], - "question": d["Question"], - "category": d["Category"], - } - ) - # highlight-end - if batch.number_errors > 10: - print("Batch import stopped due to excessive errors.") - break +# highlight-start +result = questions.data.ingest( + [ + { + "answer": d["Answer"], + "question": d["Question"], + "category": d["Category"], + } + for d in data + ] +) +# highlight-end -failed_objects = questions.batch.failed_objects -if failed_objects: - print(f"Number of failed imports: {len(failed_objects)}") - print(f"First failed object: {failed_objects[0]}") +# `errors` holds one entry per failed object, keyed by its position in the input +if result.errors: + print(f"Number of failed imports: {len(result.errors)}") + for index, error in result.errors.items(): + print(f"Failed object at index {index}: {error.message}") client.close() # Free up resources # END Import diff --git a/_includes/code/python/quickstart.import_objects.py b/_includes/code/python/quickstart.import_objects.py index fa89fdf71..0f4041af9 100644 --- a/_includes/code/python/quickstart.import_objects.py +++ b/_includes/code/python/quickstart.import_objects.py @@ -17,27 +17,26 @@ ) data = json.loads(resp.text) -# highlight-start questions = client.collections.use("Question") -with questions.batch.fixed_size(batch_size=200) as batch: - for d in data: - batch.add_object( - { - "answer": d["Answer"], - "question": d["Question"], - "category": d["Category"], - } - ) - # highlight-end - if batch.number_errors > 10: - print("Batch import stopped due to excessive errors.") - break +# highlight-start +result = questions.data.ingest( + [ + { + "answer": d["Answer"], + "question": d["Question"], + "category": d["Category"], + } + for d in data + ] +) +# highlight-end -failed_objects = questions.batch.failed_objects -if failed_objects: - print(f"Number of failed imports: {len(failed_objects)}") - print(f"First failed object: {failed_objects[0]}") +# `errors` holds one entry per failed object, keyed by its position in the input +if result.errors: + print(f"Number of failed imports: {len(result.errors)}") + for index, error in result.errors.items(): + print(f"Failed object at index {index}: {error.message}") client.close() # Free up resources # END Import diff --git a/_includes/code/python/quickstart.short.create_collection.py b/_includes/code/python/quickstart.short.create_collection.py index a4685294d..62becb3a9 100644 --- a/_includes/code/python/quickstart.short.create_collection.py +++ b/_includes/code/python/quickstart.short.create_collection.py @@ -42,9 +42,7 @@ # START CreateCollection movies = client.collections.use("Movie") - with movies.batch.fixed_size(batch_size=200) as batch: - for obj in data_objects: - batch.add_object(properties=obj) + movies.data.ingest(data_objects) print(f"Imported & vectorized {len(movies)} objects into the Movie collection") # END CreateCollection diff --git a/_includes/code/python/quickstart.short.import_vectors.create_collection.py b/_includes/code/python/quickstart.short.import_vectors.create_collection.py index 2211e39ae..76a15f1b8 100644 --- a/_includes/code/python/quickstart.short.import_vectors.create_collection.py +++ b/_includes/code/python/quickstart.short.import_vectors.create_collection.py @@ -1,6 +1,7 @@ # START CreateCollection import weaviate from weaviate.classes.config import Configure +from weaviate.classes.data import DataObject import os # Best practice: store your credentials in environment variables @@ -46,9 +47,10 @@ # Insert the objects with vectors movies = client.collections.get("Movie") - with movies.batch.fixed_size(batch_size=200) as batch: - for obj in data_objects: - batch.add_object(properties=obj["properties"], vector=obj["vector"]) + movies.data.ingest( + DataObject(properties=obj["properties"], vector=obj["vector"]) + for obj in data_objects + ) print( f"Imported {len(data_objects)} objects with vectors into the Movie collection" diff --git a/_includes/code/python/quickstart.short.local.create_collection.py b/_includes/code/python/quickstart.short.local.create_collection.py index 5d8cdfc56..ff8ce55a0 100644 --- a/_includes/code/python/quickstart.short.local.create_collection.py +++ b/_includes/code/python/quickstart.short.local.create_collection.py @@ -36,9 +36,7 @@ # START CreateCollection movies = client.collections.use("Movie") - with movies.batch.fixed_size(batch_size=200) as batch: - for obj in data_objects: - batch.add_object(properties=obj) + movies.data.ingest(data_objects) print(f"Imported & vectorized {len(movies)} objects into the Movie collection") # END CreateCollection diff --git a/_includes/code/python/quickstart.short.local.import_vectors.create_collection.py b/_includes/code/python/quickstart.short.local.import_vectors.create_collection.py index 208b7a31c..f5a874884 100644 --- a/_includes/code/python/quickstart.short.local.import_vectors.create_collection.py +++ b/_includes/code/python/quickstart.short.local.import_vectors.create_collection.py @@ -1,6 +1,7 @@ # START CreateCollection import weaviate from weaviate.classes.config import Configure +from weaviate.classes.data import DataObject # Step 1.1: Connect to your local Weaviate instance with weaviate.connect_to_local() as client: @@ -38,9 +39,10 @@ # Insert the objects with vectors movies = client.collections.get("Movie") - with movies.batch.fixed_size(batch_size=200) as batch: - for obj in data_objects: - batch.add_object(properties=obj["properties"], vector=obj["vector"]) + movies.data.ingest( + DataObject(properties=obj["properties"], vector=obj["vector"]) + for obj in data_objects + ) print( f"Imported {len(data_objects)} objects with vectors into the Movie collection" diff --git a/_includes/code/quickstart/local.quickstart.import_objects.mdx b/_includes/code/quickstart/local.quickstart.import_objects.mdx index 705df5b35..ad0c6222f 100644 --- a/_includes/code/quickstart/local.quickstart.import_objects.mdx +++ b/_includes/code/quickstart/local.quickstart.import_objects.mdx @@ -8,6 +8,8 @@ import JavaV6Code from "!!raw-loader!/_includes/code/java-v6/src/test/java/Quick import CSharpCode from "!!raw-loader!/_includes/code/csharp/QuickstartLocalTest.cs"; +Every import reports whether any objects failed. Check for failures in your own code to catch problems such as malformed data or a misconfigured model provider. + @@ -19,7 +21,7 @@ import CSharpCode from "!!raw-loader!/_includes/code/csharp/QuickstartLocalTest. title="quickstart_import.py" /> -During a batch import, any failed objects can be obtained through `batch.failed_objects`. Additionally, a running count of failed objects is maintained and can be accessed through `batch.number_errors` within the context manager. This counter can be used to stop the import process in order to investigate the failed objects or references. Find out more about error handling on the Python client [reference page](/weaviate/client-libraries/python/notes-best-practices#error-handling). +`data.ingest()` returns a `BatchObjectReturn`. Read `result.errors` to check for failures. It holds one entry per failed object, keyed by its position in the input. For more, see [error handling in the Python client reference](/weaviate/client-libraries/python/notes-best-practices#error-handling). @@ -33,6 +35,8 @@ During a batch import, any failed objects can be obtained through `batch.failed_ title="quickstart_import.ts" /> +`data.ingest()` returns a result object. Read `result.hasErrors` for a quick check, and `result.errors` for one entry per failed object, keyed by its position in the input. + @@ -52,6 +56,9 @@ During a batch import, any failed objects can be obtained through `batch.failed_ endMarker="// END Import" language="java" /> + +`batch.start()` opens a server-side batch. Closing the batch sends any remaining objects and waits for the server to report the results. Read `batch.numberOfErrors()` after the batch closes; before then, the tally is incomplete. + @@ -61,6 +68,9 @@ During a batch import, any failed objects can be obtained through `batch.failed_ endMarker="// END Import" language="csharp" /> + +`Batch.InsertMany()` returns a `BatchInsertResponse`. Read `HasErrors` for a quick check, `Errors` for the failures alone, and `Objects` for one entry per object. Each entry's `Index` is its position in the input, and failed entries carry an `Error`. + diff --git a/_includes/code/quickstart/quickstart.create_collection.mdx b/_includes/code/quickstart/quickstart.create_collection.mdx index 35925009c..1832d0339 100644 --- a/_includes/code/quickstart/quickstart.create_collection.mdx +++ b/_includes/code/quickstart/quickstart.create_collection.mdx @@ -88,7 +88,7 @@ curl -X POST \ "vectorizer": "text2vec-weaviate", "moduleConfig": { "text2vec-weaviate": {}, - "generative-cohere": {} + "generative-openai": {} } }' \ "$WEAVIATE_URL/v1/schema" diff --git a/_includes/code/quickstart/quickstart.import_objects.mdx b/_includes/code/quickstart/quickstart.import_objects.mdx index 35ef8a581..5ca38bfca 100644 --- a/_includes/code/quickstart/quickstart.import_objects.mdx +++ b/_includes/code/quickstart/quickstart.import_objects.mdx @@ -8,6 +8,8 @@ import JavaV6Code from "!!raw-loader!/_includes/code/java-v6/src/test/java/Quick import CSharpCode from "!!raw-loader!/_includes/code/csharp/QuickstartTest.cs"; +Every import reports whether any objects failed. Check for failures in your own code to catch problems such as malformed data or a misconfigured model provider. + @@ -19,8 +21,7 @@ import CSharpCode from "!!raw-loader!/_includes/code/csharp/QuickstartTest.cs"; title="quickstart_import.py" /> - -During a batch import, any failed objects can be obtained through `batch.failed_objects`. Additionally, a running count of failed objects is maintained and can be accessed through `batch.number_errors` within the context manager. This counter can be used to stop the import process in order to investigate the failed objects or references. Find out more about error handling on the Python client [reference page](/weaviate/client-libraries/python/notes-best-practices#error-handling). +`data.ingest()` returns a `BatchObjectReturn`. Read `result.errors` to check for failures. It holds one entry per failed object, keyed by its position in the input. For more, see [error handling in the Python client reference](/weaviate/client-libraries/python/notes-best-practices#error-handling). @@ -34,6 +35,8 @@ During a batch import, any failed objects can be obtained through `batch.failed_ title="quickstart_import.ts" /> +`data.ingest()` returns a result object. Read `result.hasErrors` for a quick check, and `result.errors` for one entry per failed object, keyed by its position in the input. + @@ -53,6 +56,9 @@ During a batch import, any failed objects can be obtained through `batch.failed_ endMarker="// END Import" language="java" /> + +`batch.start()` opens a server-side batch. Closing the batch sends any remaining objects and waits for the server to report the results. Read `batch.numberOfErrors()` after the batch closes; before then, the tally is incomplete. + @@ -62,6 +68,9 @@ During a batch import, any failed objects can be obtained through `batch.failed_ endMarker="// END Import" language="csharp" /> + +`Batch.InsertMany()` returns a `BatchInsertResponse`. Read `HasErrors` for a quick check, `Errors` for the failures alone, and `Objects` for one entry per object. Each entry's `Index` is its position in the input, and failed entries carry an `Error`. + diff --git a/_includes/code/quickstart/quickstart.query.rag.mdx b/_includes/code/quickstart/quickstart.query.rag.mdx index 63cf292cd..7f9b711b7 100644 --- a/_includes/code/quickstart/quickstart.query.rag.mdx +++ b/_includes/code/quickstart/quickstart.query.rag.mdx @@ -59,7 +59,7 @@ import JavaV6Code from "!!raw-loader!/_includes/code/java-v6/src/test/java/Quick # Best practice: store your credentials in environment variables # export WEAVIATE_URL="YOUR_INSTANCE_URL" # Your Weaviate instance URL # export WEAVIATE_API_KEY="YOUR_API_KEY" # Your Weaviate instance API key -# export COHERE_API_KEY="YOUR_API_KEY" # Your Cohere API key +# export OPENAI_API_KEY="YOUR_API_KEY" # Your OpenAI API key echo '{ "query": "{ @@ -92,7 +92,7 @@ echo '{ -X POST \ -H 'Content-Type: application/json' \ -H "Authorization: Bearer $WEAVIATE_API_KEY" \ - -H "X-Cohere-Api-Key: $COHERE_API_KEY" \ + -H "X-OpenAI-Api-Key: $OPENAI_API_KEY" \ -d @- \ $WEAVIATE_URL/v1/graphql ``` diff --git a/_includes/code/typescript/local.quickstart.import_objects.ts b/_includes/code/typescript/local.quickstart.import_objects.ts index ffe894279..dfa4a0861 100644 --- a/_includes/code/typescript/local.quickstart.import_objects.ts +++ b/_includes/code/typescript/local.quickstart.import_objects.ts @@ -11,18 +11,27 @@ async function getJsonData() { return file.json(); } -// highlight-start -// Note: The TS client does not have a `batch` method yet -// We use `insertMany` instead, which sends all of the data in one request async function importQuestions() { const questions = client.collections.use('Question'); const data = await getJsonData(); - const result = await questions.data.insertMany(data); - console.log('Insertion response: ', result); + + // highlight-start + // `ingest` imports the list using server-side batching + const result = await questions.data.ingest( + data.map((properties) => ({ properties })) + ); + // highlight-end + + if (result.hasErrors) { + console.log(`Number of failed imports: ${Object.keys(result.errors).length}`); + // `errors` is keyed by the position of the object in the input + for (const [index, error] of Object.entries(result.errors)) { + console.log(`Failed object at index ${index}: ${error.message}`); + } + } } await importQuestions(); -// highlight-end client.close(); // Close the client connection // END Import diff --git a/_includes/code/typescript/quickstart.import_objects.ts b/_includes/code/typescript/quickstart.import_objects.ts index b04257fd7..43aebb365 100644 --- a/_includes/code/typescript/quickstart.import_objects.ts +++ b/_includes/code/typescript/quickstart.import_objects.ts @@ -20,18 +20,27 @@ async function getJsonData() { return file.json(); } -// highlight-start -// Note: The TS client does not have a `batch` method yet -// We use `insertMany` instead, which sends all of the data in one request async function importQuestions() { const questions = client.collections.use('Question'); const data = await getJsonData(); - const result = await questions.data.insertMany(data); - console.log('Insertion response: ', result); + + // highlight-start + // `ingest` imports the list using server-side batching + const result = await questions.data.ingest( + data.map((properties) => ({ properties })) + ); + // highlight-end + + if (result.hasErrors) { + console.log(`Number of failed imports: ${Object.keys(result.errors).length}`); + // `errors` is keyed by the position of the object in the input + for (const [index, error] of Object.entries(result.errors)) { + console.log(`Failed object at index ${index}: ${error.message}`); + } + } } await importQuestions(); -// highlight-end client.close(); // Close the client connection // END Import diff --git a/_includes/code/typescript/quickstart.short.create_collection.ts b/_includes/code/typescript/quickstart.short.create_collection.ts index 3c7e5fb88..167ff7734 100644 --- a/_includes/code/typescript/quickstart.short.create_collection.ts +++ b/_includes/code/typescript/quickstart.short.create_collection.ts @@ -41,7 +41,9 @@ const dataObjects = [ // START CreateCollection const movieCollection = client.collections.get('Movie'); -const response = await movieCollection.data.insertMany(dataObjects); +await movieCollection.data.ingest( + dataObjects.map((properties) => ({ properties })) +); console.log(`Imported & vectorized ${dataObjects.length} objects into the Movie collection`); diff --git a/_includes/code/typescript/quickstart.short.import_vectors.create_collection.ts b/_includes/code/typescript/quickstart.short.import_vectors.create_collection.ts index a8f0cced0..82e4a8aae 100644 --- a/_includes/code/typescript/quickstart.short.import_vectors.create_collection.ts +++ b/_includes/code/typescript/quickstart.short.import_vectors.create_collection.ts @@ -51,7 +51,7 @@ const dataObjects = [ // START CreateCollection // Insert the objects with vectors const movieCollection = client.collections.get('Movie'); -const response = await movieCollection.data.insertMany(dataObjects); +await movieCollection.data.ingest(dataObjects); console.log(`Imported ${dataObjects.length} objects with vectors into the Movie collection`); diff --git a/_includes/code/typescript/quickstart.short.local.create_collection.ts b/_includes/code/typescript/quickstart.short.local.create_collection.ts index dabb209d9..239cae8fa 100644 --- a/_includes/code/typescript/quickstart.short.local.create_collection.ts +++ b/_includes/code/typescript/quickstart.short.local.create_collection.ts @@ -35,7 +35,9 @@ const dataObjects = [ // START CreateCollection const movieCollection = client.collections.get('Movie'); -const response = await movieCollection.data.insertMany(dataObjects); +await movieCollection.data.ingest( + dataObjects.map((properties) => ({ properties })) +); console.log(`Imported & vectorized ${dataObjects.length} objects into the Movie collection`); diff --git a/_includes/code/typescript/quickstart.short.local.import_vectors.create_collection.ts b/_includes/code/typescript/quickstart.short.local.import_vectors.create_collection.ts index 871ffe26c..6c667bcba 100644 --- a/_includes/code/typescript/quickstart.short.local.import_vectors.create_collection.ts +++ b/_includes/code/typescript/quickstart.short.local.import_vectors.create_collection.ts @@ -36,7 +36,7 @@ const dataObjects = [ // Insert the objects with vectors const movieCollection = client.collections.get('Movie'); -const response = await movieCollection.data.insertMany(dataObjects); +await movieCollection.data.ingest(dataObjects); console.log(`Imported ${dataObjects.length} objects with vectors into the Movie collection`); diff --git a/_includes/configuration/bq-compression-parameters.mdx b/_includes/configuration/bq-compression-parameters.mdx index 9552fb9b1..8e4773eb7 100644 --- a/_includes/configuration/bq-compression-parameters.mdx +++ b/_includes/configuration/bq-compression-parameters.mdx @@ -1,6 +1,6 @@ | Parameter | Type | Default | Details | | :---------------------- | :------ | :------ | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `bq` : `enabled` | boolean | `false` | Enable BQ. Weaviate uses binary quantization (BQ) compression when `true`.

The Python client does not use the `enabled` parameter. To enable BQ with the v4 client, set a `quantizer` in the collection definition. | -| `bq` : `rescoreLimit` | integer | -1 | The minimum number of candidates to fetch before rescoring. | +| `bq` : `rescoreLimit` | integer | `-1` | The minimum number of candidates to fetch before rescoring. A default of `-1` lets Weaviate pick the limit.
(only when using the `flat` vector index type)

Under the `hnsw` vector index type, BQ has no `rescoreLimit` setting. A value set there is accepted by the API but silently discarded, and it does not appear when you read the collection definition back. | | `bq` : `cache` | boolean | `false` | Whether to cache the vectors in memory.
(only when using the `flat` vector index type) | | `vectorCacheMaxObjects` | integer | `1e12` | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/weaviate/concepts/vector-index#vector-cache-considerations). | diff --git a/_includes/configuration/rq-compression-parameters.mdx b/_includes/configuration/rq-compression-parameters.mdx index 2e51380fa..52333c815 100644 --- a/_includes/configuration/rq-compression-parameters.mdx +++ b/_includes/configuration/rq-compression-parameters.mdx @@ -1,6 +1,6 @@ | Parameter | Type | Default | Details | | :---------------------- | :------ | :------ | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `rq`: `bits` | integer | `8` | The number of bits used to quantize each data point. Value can be `8` or `1`.

Learn more about [8-bit](/weaviate/concepts/vector-quantization#8-bit-rq) and [1-bit](/weaviate/concepts/vector-quantization#1-bit-rq) RQ. | -| `rq`: `rescoreLimit` | integer | `-1` | The minimum number of candidates to fetch before rescoring. | +| `rq`: `rescoreLimit` | integer | `20` (`hnsw`, 8-bit)
`512` (`hnsw`, 1-bit)
`-1` (`flat`) | The minimum number of candidates to fetch before rescoring.

The default depends on the vector index type, and under `hnsw` also on `bits`: `20` for 8-bit RQ and `512` for 1-bit RQ. Under the `flat` index type the default is `-1`, which lets Weaviate pick the limit.

These defaults apply to the `hnsw` and `flat` index types. For the HFresh index, see [HFresh index parameters](/weaviate/config-refs/indexing/vector-index#hfresh-index-parameters). | | `rq` : `cache` | boolean | `false` | Whether to cache the vectors in memory.
(only when using the `flat` vector index type) | | `vectorCacheMaxObjects` | integer | `1e12` | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/weaviate/concepts/vector-index#vector-cache-considerations). | diff --git a/_includes/configuration/sq-compression-parameters.mdx b/_includes/configuration/sq-compression-parameters.mdx index a49ed00b8..572eb4dc8 100644 --- a/_includes/configuration/sq-compression-parameters.mdx +++ b/_includes/configuration/sq-compression-parameters.mdx @@ -1,6 +1,6 @@ | Parameter | Type | Default | Details | | :---------------------- | :------ | :------ | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `sq`: `enabled` | boolean | `false` | Uses SQ when `true`.

The Python client does not use the `enabled` parameter. To enable SQ with the v4 client, set a `quantizer` in the collection definition. | -| `sq`: `rescoreLimit` | integer | -1 | The minimum number of candidates to fetch before rescoring. | +| `sq`: `rescoreLimit` | integer | `20` (`hnsw`)
`-1` (`flat`) | The minimum number of candidates to fetch before rescoring.

The default depends on the vector index type: `20` under `hnsw`, and `-1` under `flat`, which lets Weaviate pick the limit. | | `sq`: `trainingLimit` | integer | 100000 | The size of the training set to determine scalar bucket boundaries. | | `vectorCacheMaxObjects` | integer | `1e12` | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/weaviate/concepts/vector-index#vector-cache-considerations). | diff --git a/_includes/feature-notes/v137-preview.mdx b/_includes/feature-notes/v137-preview.mdx index 88182bb3f..eeafa5aa2 100644 --- a/_includes/feature-notes/v137-preview.mdx +++ b/_includes/feature-notes/v137-preview.mdx @@ -1,8 +1,6 @@ -:::caution Preview — added in `v1.37` +:::info Added in `v1.37.3` and `v1.38.6` -This is a preview feature. The API may change in future releases. - -- **Python client**: Support is not yet in a released `weaviate-client`. Coming in the next release (tracked in [PR #1997](https://github.com/weaviate/weaviate-python-client/pull/1997)). -- **Multi-node clusters**: MMR reranking may produce suboptimal results for collections whose shards are distributed across multiple nodes, since each shard returns its own candidate set before the coordinator reranks them. We are actively working on improving this. +**Diversity selection (MMR)** for vector search was added in **`v1.37.3`**.
+**Diversity selection (MMR)** for hybrid search was added in **`v1.38.6`**. ::: diff --git a/_includes/release-history.md b/_includes/release-history.md index c830af81a..a06518986 100644 --- a/_includes/release-history.md +++ b/_includes/release-history.md @@ -2,17 +2,18 @@ This table lists recent Weaviate Database versions and corresponding client libr | Weaviate Database
([GitHub][cWeaviate]) | First
release date | Python
([GitHub][cPython]) | TypeScript/
JavaScript
([GitHub][cTypeScript]) | Go
([GitHub][cGo]) | Java
([GitHub][cJava]) | C#
([GitHub][cCSharp]) | | :------------------------------------------------------------------ | :---------------------- | :-------------------------------------------------------------------------------: | :--------------------------------------------------------------------------: | :-------------------------------------------------------------------------: | :-----------------------------------------------------------------: | :-----------------------------------------------------------------------------: | +| [1.39.x](https://github.com/weaviate/weaviate/releases/tag/v1.39.0) | 2026-08-04 | [4.23.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.23.0) | - | - | - | - | | [1.38.x](https://github.com/weaviate/weaviate/releases/tag/v1.38.0) | 2026-06-05 | [4.22.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.22.0) | [3.14.x](https://github.com/weaviate/typescript-client/releases/tag/v3.14.0) | - | [6.3.0](https://github.com/weaviate/java-client/releases/tag/6.3.0) | - | | [1.37.x](https://github.com/weaviate/weaviate/releases/tag/v1.37.0) | 2026-04-16 | [4.21.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.21.0) | [3.13.x](https://github.com/weaviate/typescript-client/releases/tag/v3.13.0) | [5.7.3](https://github.com/weaviate/weaviate-go-client/releases/tag/v5.7.3) | [6.2.0](https://github.com/weaviate/java-client/releases/tag/6.2.0) | N/A | | [1.36.x](https://github.com/weaviate/weaviate/releases/tag/v1.36.0) | 2026-02-24 | [4.20.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.20.0) | [3.12.x](https://github.com/weaviate/typescript-client/releases/tag/v3.12.0) | [5.7.x](https://github.com/weaviate/weaviate-go-client/releases/tag/v5.7.0) | [6.1.0](https://github.com/weaviate/java-client/releases/tag/6.1.0) | [1.0.1](https://github.com/weaviate/weaviate-dotnet-client/releases/tag/v1.0.1) | | [1.35.x](https://github.com/weaviate/weaviate/releases/tag/v1.35.0) | 2025-12-17 | [4.19.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.19.0) | [3.11.x](https://github.com/weaviate/typescript-client/releases/tag/v3.11.0) | [5.6.x](https://github.com/weaviate/weaviate-go-client/releases/tag/v5.6.0) | [6.0.0](https://github.com/weaviate/java-client/releases/tag/6.0.0) | [1.0.0](https://github.com/weaviate/weaviate-dotnet-client/releases/tag/v1.0.0) | -| [1.34.x](https://github.com/weaviate/weaviate/releases/tag/v1.34.0) | 2025-11-05 | [4.18.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.18.0) | [3.10.x](https://github.com/weaviate/typescript-client/releases/tag/v3.10.0) | [5.6.x](https://github.com/weaviate/weaviate-go-client/releases/tag/v5.6.0) | [6.0.0](https://github.com/weaviate/java-client/releases/tag/6.0.0) | - |
Older releases | Weaviate Database
([GitHub][cWeaviate]) | First
release date | Python
([GitHub][cPython]) | TypeScript/
JavaScript
([GitHub][cTypeScript]) | Go
([GitHub][cGo]) | Java
([GitHub][cJava]) | | :------------------------------------------------------------------ | :---------------------- | :-------------------------------------------------------------------------------: | :------------------------------------------------------------------------: | :---------------------------------------------------------------------------: | :-----------------------------------------------------------------------------------------------------------------------------------------: | +| [1.34.x](https://github.com/weaviate/weaviate/releases/tag/v1.34.0) | 2025-11-05 | [4.18.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.18.0) | [3.10.x](https://github.com/weaviate/typescript-client/releases/tag/v3.10.0) | [5.6.x](https://github.com/weaviate/weaviate-go-client/releases/tag/v5.6.0) | [6.0.0](https://github.com/weaviate/java-client/releases/tag/6.0.0) | | [1.33.x](https://github.com/weaviate/weaviate/releases/tag/v1.33.0) | 2025-09-25 | [4.17.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.17.0) | [3.9.x](https://github.com/weaviate/typescript-client/releases/tag/v3.9.0) | [5.5.x](https://github.com/weaviate/weaviate-go-client/releases/tag/v5.5.0) | [5.5.x](https://github.com/weaviate/java-client/releases/tag/5.5.0) | | [1.32.x](https://github.com/weaviate/weaviate/releases/tag/v1.32.0) | 2025-07-14 | [4.16.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.16.0) | [3.8.x](https://github.com/weaviate/typescript-client/releases/tag/v3.8.0) | [5.3.x](https://github.com/weaviate/weaviate-go-client/releases/tag/v5.3.0) | [5.4.x](https://github.com/weaviate/java-client/releases/tag/5.4.0) | | [1.31.x](https://github.com/weaviate/weaviate/releases/tag/v1.31.0) | 2025-05-30 | [4.15.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.15.0) | [3.6.x](https://github.com/weaviate/typescript-client/releases/tag/v3.6.0) | [5.2.x](https://github.com/weaviate/weaviate-go-client/releases/tag/v5.2.0) | [5.3.x](https://github.com/weaviate/java-client/releases/tag/5.3.0) | diff --git a/_includes/wcs/restart-warning.mdx b/_includes/wcs/restart-warning.mdx deleted file mode 100644 index 9dcdb3522..000000000 --- a/_includes/wcs/restart-warning.mdx +++ /dev/null @@ -1 +0,0 @@ -This action restarts the cluster. If you have a stand-alone cluster, there is a short downtime while the cluster restarts. There is no downtime if you have a high availability cluster. \ No newline at end of file diff --git a/_includes/weaviate-embeddings-multimodal-models.mdx b/_includes/weaviate-embeddings-multimodal-models.mdx index e16485aa9..f7420929a 100644 --- a/_includes/weaviate-embeddings-multimodal-models.mdx +++ b/_includes/weaviate-embeddings-multimodal-models.mdx @@ -4,7 +4,7 @@ - Generates multi-vector embeddings (ColBERT-style late-interaction) from document images and text queries. - Ideal for getting documents directly into Weaviate without heavy preprocessing - no OCR or text extraction required. - State-of-the-art performance in its size class, matching models up to 10x larger. -- Query token limit: 8,092 tokens +- Query token limit: 8,192 tokens - Read more at the [Hugging Face model card](https://huggingface.co/ModernVBERT/colmodernvbert) - For integration details, see [Weaviate Embeddings: Multimodal](/weaviate/model-providers/weaviate/embeddings-multimodal) diff --git a/docs/cloud/embeddings/quickstart.mdx b/docs/cloud/embeddings/quickstart.mdx index 804f0457c..18380d339 100644 --- a/docs/cloud/embeddings/quickstart.mdx +++ b/docs/cloud/embeddings/quickstart.mdx @@ -80,14 +80,14 @@ import PromptStarter from "/src/components/PromptStarter"; To use Weaviate Embeddings, you will need: - +- A Weaviate Cloud free cluster +- A Weaviate client library that supports Weaviate Embeddings -- A Weaviate Cloud free cluster running at least Weaviate `1.28.5` -- A Weaviate client library that supports Weaviate Embeddings: - - **Python** client version `4.9.5` or higher - - **JavaScript/TypeScript** client version `3.2.5` or higher - - **Java** or **C#** clients - - **Go** client is not yet officially supported; you must pass the `X-Weaviate-Api-Key` and `X-Weaviate-Cluster-Url` headers manually upon instantiation as shown below. +import CodeClientInstall from "/\_includes/code/quickstart/clients.install.new.mdx"; + + + +The Go client does not support Weaviate Embeddings directly. Pass the `X-Weaviate-Api-Key` and `X-Weaviate-Cluster-Url` headers manually when you instantiate the client. ## Step 1: Set up Weaviate @@ -103,8 +103,6 @@ import LatestWeaviateVersion from "/_includes/latest-weaviate-version.mdx"; We recommend using a [client library](/weaviate/client-libraries) to work with Weaviate. Follow the instructions below to install one of the official client libraries, available in [Python](/weaviate/client-libraries/python), [JavaScript/TypeScript](/weaviate/client-libraries/typescript), [Go](/weaviate/client-libraries/go), and [Java](/weaviate/client-libraries/java). -import CodeClientInstall from "/_includes/code/quickstart/clients.install.mdx"; - ### 1.3 Connect to Weaviate Cloud diff --git a/docs/cloud/img/mfa-enable-icon.jpg b/docs/cloud/img/mfa-enable-icon.jpg deleted file mode 100644 index 7fa724dab..000000000 Binary files a/docs/cloud/img/mfa-enable-icon.jpg and /dev/null differ diff --git a/docs/cloud/img/mfa-one-time-code.jpg b/docs/cloud/img/mfa-one-time-code.jpg deleted file mode 100644 index 7fead9494..000000000 Binary files a/docs/cloud/img/mfa-one-time-code.jpg and /dev/null differ diff --git a/docs/cloud/img/wcs-add-key-details.jpg b/docs/cloud/img/wcs-add-key-details.jpg deleted file mode 100644 index c2ef5eb47..000000000 Binary files a/docs/cloud/img/wcs-add-key-details.jpg and /dev/null differ diff --git a/docs/cloud/img/wcs-api-keys.jpg b/docs/cloud/img/wcs-api-keys.jpg deleted file mode 100644 index 21d310573..000000000 Binary files a/docs/cloud/img/wcs-api-keys.jpg and /dev/null differ diff --git a/docs/cloud/img/wcs-console-url-check.jpg b/docs/cloud/img/wcs-console-url-check.jpg deleted file mode 100644 index 876845a71..000000000 Binary files a/docs/cloud/img/wcs-console-url-check.jpg and /dev/null differ diff --git a/docs/cloud/img/wcs-delete-api-key.jpg b/docs/cloud/img/wcs-delete-api-key.jpg deleted file mode 100644 index cd38bdb91..000000000 Binary files a/docs/cloud/img/wcs-delete-api-key.jpg and /dev/null differ diff --git a/docs/cloud/img/wcs-landing-page-register.jpg b/docs/cloud/img/wcs-landing-page-register.jpg deleted file mode 100644 index 750583e5b..000000000 Binary files a/docs/cloud/img/wcs-landing-page-register.jpg and /dev/null differ diff --git a/docs/cloud/img/weaviate-cloud-roles-create-form.png b/docs/cloud/img/weaviate-cloud-roles-create-form.png deleted file mode 100644 index 1bbc726f4..000000000 Binary files a/docs/cloud/img/weaviate-cloud-roles-create-form.png and /dev/null differ diff --git a/docs/cloud/img/weaviate-cloud-roles-create.png b/docs/cloud/img/weaviate-cloud-roles-create.png deleted file mode 100644 index 9fd09aa80..000000000 Binary files a/docs/cloud/img/weaviate-cloud-roles-create.png and /dev/null differ diff --git a/docs/cloud/img/weaviate-cloud-roles-delete-form.png b/docs/cloud/img/weaviate-cloud-roles-delete-form.png deleted file mode 100644 index d99376032..000000000 Binary files a/docs/cloud/img/weaviate-cloud-roles-delete-form.png and /dev/null differ diff --git a/docs/cloud/img/weaviate-cloud-roles-delete.png b/docs/cloud/img/weaviate-cloud-roles-delete.png deleted file mode 100644 index ac677b4d6..000000000 Binary files a/docs/cloud/img/weaviate-cloud-roles-delete.png and /dev/null differ diff --git a/docs/cloud/img/weaviate-cloud-roles-edit-form.png b/docs/cloud/img/weaviate-cloud-roles-edit-form.png deleted file mode 100644 index 2fa868d1e..000000000 Binary files a/docs/cloud/img/weaviate-cloud-roles-edit-form.png and /dev/null differ diff --git a/docs/cloud/img/weaviate-cloud-roles-edit.png b/docs/cloud/img/weaviate-cloud-roles-edit.png deleted file mode 100644 index 1d9d83547..000000000 Binary files a/docs/cloud/img/weaviate-cloud-roles-edit.png and /dev/null differ diff --git a/docs/cloud/manage-clusters/authentication.mdx b/docs/cloud/manage-clusters/authentication.mdx index 5a88fc1ad..134938234 100644 --- a/docs/cloud/manage-clusters/authentication.mdx +++ b/docs/cloud/manage-clusters/authentication.mdx @@ -5,11 +5,6 @@ description: "Configure authentication options for Weaviate Cloud clusters by ad image: og/wcd/user_guides.jpg --- -import WCDAPIKeys from "/docs/cloud/img/wcs-api-keys.jpg"; -import WCDAddAPIKeys from "/docs/cloud/img/wcs-add-key-details.jpg"; -import WCDDelAPIKeys from "/docs/cloud/img/wcs-delete-api-key.jpg"; -import RestartTheCluster from "/_includes/wcs/restart-warning.mdx"; - [Weaviate Cloud (WCD)](/go/console?utm_content=cloud) uses [RBAC (Role-Based Access Control)](/weaviate/configuration/rbac/index.mdx) to manage authentication. Below, you can find guides on how to create, edit, rotate and delete API keys for accessing Weaviate Cloud. ### Create an API key diff --git a/docs/cloud/manage-clusters/authorization.mdx b/docs/cloud/manage-clusters/authorization.mdx index 2c77ed8fb..da3aa1dea 100644 --- a/docs/cloud/manage-clusters/authorization.mdx +++ b/docs/cloud/manage-clusters/authorization.mdx @@ -5,14 +5,6 @@ description: "Role-Based Access Control (RBAC) configuration guide for Weaviate image: og/wcd/user_guides.jpg --- -import Link from "@docusaurus/Link"; -import WCDCreateRole from "/docs/cloud/img/weaviate-cloud-roles-create.png"; -import WCDCreateRoleForm from "/docs/cloud/img/weaviate-cloud-roles-create-form.png"; -import WCDEditRole from "/docs/cloud/img/weaviate-cloud-roles-edit.png"; -import WCDEditRoleForm from "/docs/cloud/img/weaviate-cloud-roles-edit-form.png"; -import WCDDeleteRole from "/docs/cloud/img/weaviate-cloud-roles-delete.png"; -import WCDDeleteRoleForm from "/docs/cloud/img/weaviate-cloud-roles-delete-form.png"; - [Weaviate Cloud (WCD)](/go/console?utm_content=cloud) uses [RBAC (Role-Based Access Control)](/weaviate/configuration/rbac/index.mdx) to manage authorization. Below, you can find guides on how to create, edit and delete user roles and manage their permissions. ## Create a role diff --git a/docs/cloud/manage-clusters/connect.mdx b/docs/cloud/manage-clusters/connect.mdx index 4e0a7d732..64126573e 100644 --- a/docs/cloud/manage-clusters/connect.mdx +++ b/docs/cloud/manage-clusters/connect.mdx @@ -6,8 +6,6 @@ description: "Multiple connection options and methods for accessing your Weaviat image: og/wcd/user_guides.jpg --- -import CompareURLs from "/docs/cloud/img/wcs-console-url-check.jpg"; - [Weaviate Cloud (WCD)](/go/console?utm_content=cloud) offers multiple options on how to connect to your cluster: - **[Connect with APIs](#connect-with-an-api-programmatically)**: @@ -149,7 +147,7 @@ This section has solutions for some common problems. For additional help, [conta To reset your Weaviate Cloud password, follow these steps: 1. Go to the Weaviate Cloud [login page](/go/console?utm_content=cloud). -1. Click on click the login button. +1. Click the login button. 1. Click `Forgot Password`. 1. Check your email account for a password reset email from Weaviate Cloud. 1. Click the link and follow the instructions to reset your password. The link is only valid for five minutes. @@ -197,7 +195,7 @@ weaviate.exceptions.WeaviateGRPCUnavailableError: gRPC health check could not be **Solution**: Verify the cluster URL is correct and update the URL if needed. -When a Shared Cloud cluster is updated, the cluster URL may change slightly. Weaviate Cloud still routes the old URL, so some connections work, however the new gRPC and the old HTTP URLS are different so connections that require gRCP fail. +When a Shared Cloud cluster is updated, the cluster URL may change slightly. Weaviate Cloud still routes the old URL, so some connections continue to work. However, the new gRPC URL and the old HTTP URL are different, so connections that require gRPC fail. To check the URLs, open the Weaviate Cloud Console and check the details panel for your cluster. If you prefix Cluster URL with `grpc-`, the Cluster URL and the Cluster gRPC URL should match. Compare the Cluster URL with the connection URL in your application. The old URL and the new URL are similar, but the new one may have an extra subdomain such as `.c0.region`. If the URLs are different, update your application's connection code to use the new Cluster URL. diff --git a/docs/cloud/manage-clusters/default-settings.mdx b/docs/cloud/manage-clusters/default-settings.mdx index 7d269b151..7e8dd9a40 100644 --- a/docs/cloud/manage-clusters/default-settings.mdx +++ b/docs/cloud/manage-clusters/default-settings.mdx @@ -22,6 +22,7 @@ import AdvancedOptions from "/docs/cloud/img/weaviate-cloud-cluster-advanced-set | [`ASYNC_REPLICATION_DISABLED`](/deploy/configuration/env-vars/index.md#ASYNC_REPLICATION_DISABLED) | No | false | No | | [`ASYNC_INDEXING`](/deploy/configuration/env-vars/index.md#ASYNC_INDEXING) | Yes | true | No | | [`DEFAULT_QUANTIZATION`](/deploy/configuration/env-vars/index.md#DEFAULT_QUANTIZATION) | No | [RQ-8](/weaviate/configuration/compression/rq-compression.md) | Yes (1) | +| [`DEFAULT_VECTOR_INDEX`](/deploy/configuration/env-vars/index.md#DEFAULT_VECTOR_INDEX) | No | Set by the cluster's [optimization profile](/cloud/manage-clusters/create#optimization-profile) | Yes (via the optimization profile) | | `CORS_ALLOW_ORIGIN` | Yes | https://console.weaviate.cloud | Yes (to allow any) (3) | | [`REPLICATION_MINIMUM_FACTOR`](/deploy/configuration/env-vars/index.md#REPLICATION_MINIMUM_FACTOR) | Yes | 3 (for HA clusters) | No | @@ -37,6 +38,14 @@ import AdvancedOptions from "/docs/cloud/img/weaviate-cloud-cluster-advanced-set
+:::note Default vector index and the optimization profile + +`DEFAULT_VECTOR_INDEX` is not set directly. It is determined by the [optimization profile](/cloud/manage-clusters/create#optimization-profile) you pick when you create the cluster: the `Cost Optimized` profile makes HFresh the default vector index, and the `Performance Optimized` profile makes HNSW the default. Free clusters support the `Cost Optimized` profile only. + +The profile only sets the default for new collections. An explicit vector index in a [collection definition](/weaviate/manage-collections/vector-config) always takes precedence. + +::: + ## User management & permissions These settings control authentication and authorization for your cluster. These settings are not user configurable. diff --git a/docs/cloud/manage-collections/enable-compression.mdx b/docs/cloud/manage-collections/enable-compression.mdx index b261411c8..95b671cd0 100644 --- a/docs/cloud/manage-collections/enable-compression.mdx +++ b/docs/cloud/manage-collections/enable-compression.mdx @@ -85,6 +85,10 @@ Replace `YOUR-WEAVIATE-CLOUD-URL` with your cluster URL (e.g., `https://your-clu The update syntax depends on your collection's **vector index type** (HNSW, flat, or dynamic) and whether it uses **named vectors**. +:::note HFresh collections are already compressed +This procedure does not apply to collections that use the [HFresh index](/weaviate/concepts/vector-index#hfresh-index), which is the index behind the **Cost Optimized** optimization profile and therefore the only index available on [free clusters](../manage-clusters/create.mdx#optimization-profile). HFresh has [rotational quantization (RQ)](../../weaviate/configuration/compression/rq-compression.md) built in and always on: the in-memory centroid index uses 8-bit RQ and the on-disk posting lists use 1-bit RQ. Weaviate rejects a request that tries to add PQ, SQ or BQ to an HFresh index, or that tries to disable its RQ, so there is nothing to enable. HFresh collections are also not listed by the categorization example below. +::: + #### HNSW index (default) Most collections use the HNSW index. To enable compression: diff --git a/docs/cloud/platform/create-account.mdx b/docs/cloud/platform/create-account.mdx index 73489f8e3..9ffc572b1 100644 --- a/docs/cloud/platform/create-account.mdx +++ b/docs/cloud/platform/create-account.mdx @@ -5,8 +5,6 @@ description: "How to create and delete user accounts in the Weaviate Cloud conso image: og/wcd/user_guides.jpg --- -import LandingRegister from "/docs/cloud/img/wcs-landing-page-register.jpg"; - Weaviate Cloud (WCD) offers an interactive console. Create a user account, then login to manage WCD clusters, run queries, and configure your organization details. ## Create a new user account and sign in {#create-a-new-user-account} @@ -57,7 +55,7 @@ To delete your account from Weaviate Cloud (WCD): 1. Remove organization members: If you own an organization, you must remove all other members from it first. 2. Navigate to the [Account Page](https://console.weaviate.cloud/account). -4. In the **Danger Zone** section, click **Delete Account** to permanently remove your account. +3. In the **Danger Zone** section, click **Delete Account** to permanently remove your account. If you encounter any issues, please contact our [Weaviate Support](https://support.weaviate.io). diff --git a/docs/cloud/platform/multi-factor-auth.mdx b/docs/cloud/platform/multi-factor-auth.mdx index 9a61a531d..e5103ed6d 100644 --- a/docs/cloud/platform/multi-factor-auth.mdx +++ b/docs/cloud/platform/multi-factor-auth.mdx @@ -5,9 +5,6 @@ description: "Enhanced security setup with multi-factor authentication for Weavi image: og/wcd/user_guides.jpg --- -import MFAOneTime from "/docs/cloud/img/mfa-one-time-code.jpg"; -import MFAEnableIcon from "/docs/cloud/img/mfa-enable-icon.jpg"; - Multi-factor authentication (MFA) increases the security of browser logins. MFA is not enabled by default. ## Enable multi-factor authentication @@ -85,7 +82,7 @@ To disable MFA, [contact support](https://support.weaviate.io). If you use a JavaScript/TypeScript client to connect a browser hosted application to Weaviate, do not enable MFA for that client's account. -There is no way to pass the the one-time authentication code to the application, so the application cannot connect to Weaviate Cloud. +There is no way to pass the one-time authentication code to the application, so the application cannot connect to Weaviate Cloud. Use API keys to connect browser based client applications to Weaviate Cloud. diff --git a/docs/cloud/platform/users-and-organizations.mdx b/docs/cloud/platform/users-and-organizations.mdx index 11cda7894..70891e6ea 100644 --- a/docs/cloud/platform/users-and-organizations.mdx +++ b/docs/cloud/platform/users-and-organizations.mdx @@ -22,7 +22,7 @@ Organizations group user accounts together. An organization owns its clusters an - Organization management - Billing configuration -Be cautious when granting elevated roles (`Owner`, `Admin`) in production — those users can modify the organization, its clusters, and its billing. +Be cautious when granting elevated roles (`Owner`, `Admin`) in production. Those users can modify the organization, its clusters, and its billing. ## Manage organizations {#manage-organizations} diff --git a/docs/cloud/quickstart.mdx b/docs/cloud/quickstart.mdx index 7039863d7..0897533c7 100644 --- a/docs/cloud/quickstart.mdx +++ b/docs/cloud/quickstart.mdx @@ -7,9 +7,6 @@ image: og/docs/quickstart-tutorial.jpg # tags: ['getting started'] --- -import Tabs from "@theme/Tabs"; -import TabItem from "@theme/TabItem"; - Expected time: 30 minutes

@@ -74,7 +71,7 @@ Notes: ## Requirements - A [Weaviate Cloud account](./platform/create-account.mdx). -- In order to perform Retrieval Augmented Generation (RAG) in the last step, you will need a [Cohere](https://dashboard.cohere.com/) account. You can use a free Cohere trial API key. If you have another preferred [model provider](/weaviate/model-providers), you can use that instead of Cohere. +- In order to perform Retrieval Augmented Generation (RAG) in the last step, you will need an [OpenAI](https://platform.openai.com/) account and an OpenAI API key. If you have another preferred [model provider](/weaviate/model-providers), you can use that instead of OpenAI.
@@ -221,11 +218,11 @@ We can now add data to our collection. The following example: - Loads objects, and -- Adds objects to the target collection (`Question`) using a batch process. +- Adds objects to the target collection (`Question`) with a batch import. :::tip Batch imports -([Batch imports](/weaviate/manage-objects/import.mdx)) are the most efficient way to add large amounts of data, as it sends multiple objects in a single request. See the [How-to: Batch import](/weaviate/manage-objects/import.mdx) guide for more information. +Batch imports are the most efficient way to add large amounts of data, because they send objects in groups instead of one request per object. See the [How-to: Batch import](/weaviate/manage-objects/import.mdx) guide for the available methods, including [server-side batching](/weaviate/manage-objects/import.mdx#server-side-batching), where the server tells the client how much data to send next. ::: @@ -254,20 +251,18 @@ import QueryNearText from "/_includes/code/quickstart/quickstart.query.neartext. Run this code to perform the query. Our query found entries for `DNA` and `species`.
- Example full response in JSON format + Example response ```json { - { - "answer": "DNA", - "question": "In 1953 Watson & Crick built a model of the molecular structure of this, the gene-carrying substance", - "category": "SCIENCE" - }, - { - "answer": "species", - "question": "2000 news: the Gunnison sage grouse isn't just another northern sage grouse, but a new one of this classification", - "category": "SCIENCE" - } + "answer": "DNA", + "question": "In 1953 Watson & Crick built a model of the molecular structure of this, the gene-carrying substance", + "category": "SCIENCE" +} +{ + "answer": "species", + "question": "2000 news: the Gunnison sage grouse isn't just another northern sage grouse, but a new one of this classification", + "category": "SCIENCE" } ``` @@ -386,9 +381,9 @@ import QueryRAG from "/_includes/code/quickstart/quickstart.query.rag.mdx"; -:::info Cohere API key in the header +:::info OpenAI API key in the header -Note that this code includes an additional header for the Cohere API key. Weaviate uses this key to access the Cohere generative AI model and perform retrieval augmented generation (RAG). +Note that this code includes an additional header for the OpenAI API key. Weaviate uses this key to access the OpenAI generative AI model and perform retrieval augmented generation (RAG). ::: diff --git a/docs/cloud/tools/collections-tool.mdx b/docs/cloud/tools/collections-tool.mdx index 9d46212b9..60be49aac 100644 --- a/docs/cloud/tools/collections-tool.mdx +++ b/docs/cloud/tools/collections-tool.mdx @@ -262,7 +262,7 @@ Some collection settings that are not available in the console can be updated pr ### Enable TTL for a collection -[Time-to-live (TTL)](/docs/weaviate/manage-collections/time-to-live.mdx) allows you to set an expiration time for objects in a collection. TTLs are currently defined at the collection level. They can be set relative to an object's creation time, the last update time, or a specific DATE property within the object. +[Time-to-live (TTL)](/weaviate/manage-collections/time-to-live.mdx) allows you to set an expiration time for objects in a collection. TTLs are currently defined at the collection level. They can be set relative to an object's creation time, the last update time, or a specific DATE property within the object.
Enable TTL step by step 1. Open the [Weaviate Cloud console](/go/console?utm_content=cloud). -2. Select you organization and cluster in the left sidebar. +2. Select your organization and cluster in the left sidebar. 3. Open the `Collections` tool from the left sidebar and select the collection you want to modify. 4. Click the pencil icon next to the Time to live (TTL) field. -5. Select you desired [TTL settings](/docs/weaviate/manage-collections/time-to-live.mdx) and save them. +5. Select your desired [TTL settings](/weaviate/manage-collections/time-to-live.mdx) and save them.
diff --git a/docs/deploy/configuration/backups.md b/docs/deploy/configuration/backups.md index de8ac6b11..419bacbe2 100644 --- a/docs/deploy/configuration/backups.md +++ b/docs/deploy/configuration/backups.md @@ -29,7 +29,7 @@ Weaviate's Backup feature is designed to work natively with cloud technology. Mo :::caution Important backup considerations - **Version Requirements**: If you are running Weaviate `v1.23.12` or older, you must [update](/deploy/migration/index.md) to `v1.23.13` or higher before restoring a backup to prevent data corruption. -- **[Multi-tenancy](/weaviate/concepts/data.md#multi-tenancy) limitations**: Starting in `v1.37`, backups include both `active` (HOT) and `inactive` (COLD) tenants — inactive tenants are backed up directly from disk without activation. `Offloaded` (FROZEN) tenants are still skipped since they have no local data. In versions prior to `v1.37`, only active tenants are included, so be sure to [activate](/weaviate/manage-collections/multi-tenancy.mdx#manage-tenant-states) any required tenants before creating a backup. +- **[Multi-tenancy](/weaviate/concepts/data.md#multi-tenancy) limitations**: Starting in `v1.37`, backups include both `active` (HOT) and `inactive` (COLD) tenants. Inactive tenants are backed up directly from disk without activation. `Offloaded` (FROZEN) tenants are still skipped since they have no local data. In versions prior to `v1.37`, only active tenants are included, so be sure to [activate](/weaviate/manage-collections/multi-tenancy.mdx#manage-tenant-states) any required tenants before creating a backup. ::: ## Backup Quickstart @@ -316,7 +316,7 @@ The `*` character matches any sequence of characters. For example, `Article*` ma | name | type | required | default | description | | ---- | ---- | ---- | ---- |---- | | `CPUPercentage` | number | no | `50%` | An optional integer to set the desired CPU core utilization ranging from 1%-80%. | -| `ChunkSize` | number | no | `128MB` | An optional integer represents the desired size for chunks. Weaviate will attempt to come close the specified size, with a minimum of 2MB, default of 128MB, and a maximum of 512MB.| +| `ChunkSize` | number | no | - | **Deprecated. This option has no effect.** Weaviate ignores any value sent here, so it neither sets nor caps the chunk size. Chunk sizing is now controlled by the [`BACKUP_CHUNK_TARGET_SIZE`](#chunking-and-file-splitting) environment variable, which replaced it. | | `CompressionLevel`| string | no | `DefaultCompression` | An optional [compression level](#compression-levels) to be used. | | `Path` | string | no | `""` | An optional string to manually set the backup location. If not provided, the backup will be stored in the default location. Introduced in Weaviate `v1.27.2`. | | `incremental_base_backup_id` | string | no | `None` | The ID of a previous backup to use as the base for an [incremental backup](#incremental-backups). Files unchanged since the base backup are stored as references rather than copied. Introduced in Weaviate `v1.37`. | @@ -466,7 +466,11 @@ This can result in dramatically smaller backups and much faster backup times. #### How it works -When creating a backup, Weaviate splits large files into individual chunks. During an incremental backup, Weaviate compares each file against the base backup. Files that haven't changed are stored as pointers to the base backup rather than being copied again. On restore, Weaviate automatically fetches the referenced files from the base backup. +When creating a backup, Weaviate packs a shard's files into chunks. During an incremental backup, Weaviate compares each file against the base backup. Files that haven't changed are stored as pointers to the base backup rather than being copied again. On restore, Weaviate automatically fetches the referenced files from the base backup. + +The base backup can itself be an incremental backup, so you can build a [chain of incremental backups](#chained-incremental-backups) that ends at a full backup. Weaviate walks the whole chain to find unchanged files, so every backup in the chain must remain available. + +Only a file large enough to get a chunk of its own can be referenced individually, so the way Weaviate groups files into chunks determines how much an incremental backup can reuse. For how Weaviate decides which files get their own chunk, and the environment variables that control chunking, see [Chunking and file splitting](#chunking-and-file-splitting). #### Create a full (base) backup @@ -518,6 +522,77 @@ Base backups (and any intermediate incremental backups in a chain) must remain a ::: +### List Backups + +You can list the backups that are stored in a backup backend. The listing reports the status of each backup, the collections it holds, its size, and, for an incremental backup, the backup it was built on. This is how you inspect an existing [chain of incremental backups](#chained-incremental-backups) and confirm that every backup the chain depends on is still present. + +```js +GET /v1/backups/{backend} +``` + +#### Parameters + +##### URL Parameters + +| Name | Type | Required | Description | +| ---- | ---- | ---- | ---- | +| `backend` | string | yes | The name of the backup provider module without the `backup-` prefix, for example `s3`, `gcs`, or `filesystem`. | + +##### Query Parameters + +| Name | Type | Required | Default | Description | +| ---- | ---- | ---- | ---- | ---- | +| `order` | string | no | `desc` | Sort the returned backups by start time, either `asc` (oldest first) or `desc` (newest first). | + +##### Response fields + +| name | type | description | +| ---- | ---- | ---- | +| `id` | string | The identifier of the backup. | +| `classes` | array | The collections the backup contains. | +| `status` | string | The status of the backup, such as `SUCCESS` or `FAILED`. | +| `startedAt` | date | When the backup started. | +| `completedAt` | date | When the backup finished, successfully or not. | +| `size` | number | The size of the backup in GiB, measured before compression. | +| `incremental_base_backup_id` | string | The identifier of the backup that this [incremental backup](#incremental-backups) was built on. Empty when the backup is a full backup. Only returned to root users, see below. Introduced in Weaviate `v1.37.6`. | + +The listing only includes backups whose collections you are authorized to read. Backups you have no read access to are left out rather than causing an error. + +:::caution One name, two different things + +`incremental_base_backup_id` appears on both sides of the backup API, and the two are not interchangeable: + +- On the **create** side it is an input that you supply. It names the backup that the new backup should build on, as described under [Create an incremental backup](#create-an-incremental-backup). +- On the **list** side it is a read-only output. It reports the backup that an already-created backup was built on, which is what lets you walk a chain back to its full base backup. + +You cannot choose a base backup through the list API. The value it returns reflects a decision that was made when that backup was created. + +::: + +:::info The base backup identifier is only returned to root users + +The list-side `incremental_base_backup_id` is treated as sensitive. Weaviate only fills it in when it has confirmed that the caller is a [root user](/docs/deploy/configuration/configuring-rbac.md). Any other caller receives an empty value for this field, even one with full backup permissions and even when the backup really is incremental. If you are auditing a backup chain and every entry comes back empty, check the identity you are connecting with before concluding that no incremental backups exist. + +::: + + + +
+ Code output + +``` +base-backup BackupStatus.SUCCESS None +incremental-backup-1 BackupStatus.SUCCESS base-backup +incremental-backup-2 BackupStatus.SUCCESS incremental-backup-1 +``` + +
+ ### Cancel Backup An ongoing backup can be cancelled at any time. The backup process will be stopped, and the backup will be marked as `CANCELLED`. @@ -778,6 +853,87 @@ The backup API is built in a way that no long-running network requests are requi If you would like your application to wait for the background backup process to complete, you can use the "wait for completion" feature that is present in all language clients. The clients will poll the status endpoint in the background and block until the status is either `SUCCESS` or `FAILED`. This makes it easy to write simple synchronous backup scripts, even with the async nature of the API. +### Chunking and file splitting + +:::info Added in `v1.36.0` +Chunking and `BACKUP_CHUNK_TARGET_SIZE` were backported to `v1.33.14`, `v1.34.11`, and `v1.35.4`. The other variables were added later. See the table below. +::: + +On every backup, full or incremental, Weaviate packs each [shard's](/weaviate/concepts/storage.md#logical-storage-units-indexes-shards-stores) files into chunks: + +- Each of the shard's biggest files gets a **chunk of its own**. +- A file larger than `BACKUP_SPLIT_FILE_SIZE` is **split into parts**, and each part gets a chunk of its own that holds nothing else. +- The remaining smaller files are packed together into **shared chunks** of roughly `BACKUP_CHUNK_TARGET_SIZE`. + +Chunking matters because it determines how much a later [incremental backup](#incremental-backups) can reuse: + +- Only a file with a chunk of its own can be referenced from the base backup instead of copied again, and only if Weaviate also treats the file as immutable. +- A big file that Weaviate keeps rewriting has its own chunk but is still re-uploaded on every backup. +- Shared chunks are re-uploaded on every incremental backup, even if nothing in them changed. + +A file gets its own chunk when it reaches the **qualifying size**: the larger of `BACKUP_MIN_CHUNK_SIZE` and the size of the shard's Nth largest file, where N is `BACKUP_MAX_INDIVIDUAL_FILES`, reduced on an incremental backup by the number of files already reused from the base backup. This budget is shared across a whole backup chain rather than renewed for each backup in it. Because the larger value wins, `BACKUP_MIN_CHUNK_SIZE` is only a floor. Lowering it never reduces how many files qualify. Which knob qualifies more files depends on the shard. With N or more files above the floor, raise `BACKUP_MAX_INDIVIDUAL_FILES`. With fewer, lower `BACKUP_MIN_CHUNK_SIZE`. + +
+ + Diagram: how a file becomes a chunk + + +```mermaid +flowchart TD + File["Shard file"] + Qualifies{"Reaches the
qualifying size?"} + OverSplit{"Larger than
the split size?"} + Own["Own chunk"] + Parts["Split into parts,
one chunk per part"] + Shared["Packed into a
shared chunk"] + Immutable{"Immutable
file?"} + Reusable["Reusable by later
incremental backups"] + Reuploaded["Re-uploaded on
every backup"] + + File --> Qualifies + Qualifies -->|"No"| Shared + Qualifies -->|"Yes"| OverSplit + OverSplit -->|"No"| Own + OverSplit -->|"Yes"| Parts + Own --> Immutable + Parts --> Immutable + Immutable -->|"Yes"| Reusable + Immutable -->|"No"| Reuploaded + Shared --> Reuploaded + + style File fill:#ffffff,stroke:#B9C8DF,color:#130C49 + style Qualifies fill:#ffffff,stroke:#B9C8DF,color:#130C49 + style OverSplit fill:#ffffff,stroke:#B9C8DF,color:#130C49 + style Own fill:#ffffff,stroke:#B9C8DF,color:#130C49 + style Parts fill:#ffffff,stroke:#B9C8DF,color:#130C49 + style Shared fill:#ffffff,stroke:#B9C8DF,color:#130C49 + style Immutable fill:#ffffff,stroke:#B9C8DF,color:#130C49 + style Reusable fill:#ffffff,stroke:#B9C8DF,color:#130C49 + style Reuploaded fill:#ffffff,stroke:#B9C8DF,color:#130C49 +``` + +
+ +The parts of a split file are sized by `BACKUP_SPLIT_FILE_SIZE`, not by the chunk target: Weaviate divides the file into roughly equal parts that are each at least half and at most the full split size. At the defaults, a chunk carrying a split part is therefore up to `50GiB`. That is far larger than the `10MiB` chunk target, not smaller. + +The three size variables accept a plain number of bytes or a number with a case-sensitive unit suffix (`B`, `KB`, `MB`, `GB`, `TB`, `KiB`, `MiB`, `GiB`, `TiB`), for example `4MiB`. Decimal and binary suffixes are distinct: `MB` is 1,000,000 bytes while `MiB` is 1,048,576 bytes. They also accept `unlimited` or `nolimit`, which is how you disable file splitting through `BACKUP_SPLIT_FILE_SIZE`. All three are read at startup, so changing them requires a restart. + +| Environment variable | Required | Description | +| --- | --- | --- | +| `BACKUP_MIN_CHUNK_SIZE` | no | A floor on the qualifying size a file must reach to get a chunk of its own. Defaults to `1MiB`.

Added in `v1.36.3` (backported to `v1.34.18`, `v1.35.13`). | +| `BACKUP_CHUNK_TARGET_SIZE` | no | The size Weaviate aims for when packing several smaller files into a shared chunk. Defaults to `10MiB`.

Added in `v1.36.0` (backported to `v1.33.14`, `v1.34.11`, `v1.35.4`). | +| `BACKUP_SPLIT_FILE_SIZE` | no | The size above which a file is split into parts. Set it to `unlimited` to disable splitting. Defaults to `50GiB`.

Added in `v1.36.5` (backported to `v1.35.15`). | +| `BACKUP_MAX_INDIVIDUAL_FILES` | no | How many of a shard's biggest files Weaviate aims to give a chunk of their own. This is a target rather than a hard cap. The value is a count, not a size, and it must be greater than `0`. Defaults to `100`. Settable without a restart through the `backup_max_individual_files` [runtime configuration](./env-vars/runtime-config.md) key.

Added in `v1.37.14` and `v1.38.7`. | + +:::note How Weaviate adjusts these values + +These settings are lower bounds rather than exact values: + +- `BACKUP_CHUNK_TARGET_SIZE` and `BACKUP_SPLIT_FILE_SIZE` are raised to the qualifying size if you set them lower. +- If a shard holds fewer files than the `BACKUP_MAX_INDIVIDUAL_FILES` budget (after subtracting files already reused on an incremental backup), the qualifying size falls back to the size of the shard's smallest file, still raised to `BACKUP_MIN_CHUNK_SIZE` if that is larger. + +::: + ### Skip the storage access check When a cloud backup backend (`backup-s3`, `backup-gcs`, or `backup-azure`) initializes, Weaviate verifies that the configured credentials can write to and delete from the target bucket. It does this by writing a temporary `access-check` object and then removing it. This probe fails on immutable (write-once / WORM) buckets, or with least-privilege credentials that are not permitted to delete objects. diff --git a/docs/deploy/configuration/env-vars/index.md b/docs/deploy/configuration/env-vars/index.md index 0efd4366c..60026acc0 100644 --- a/docs/deploy/configuration/env-vars/index.md +++ b/docs/deploy/configuration/env-vars/index.md @@ -32,11 +32,14 @@ import APITable from '@site/src/components/APITable'; | --- | --- | --- | --- | | `ASYNC_INDEXING` | If set, Weaviate creates vector indexes asynchronously to the object creation process. This can be useful for importing large amounts of data. (default: `false`) | `boolean` | `false` | | `AUTOSCHEMA_ENABLED` | Whether to infer the schema where necessary with the autoschema (default: `true`) | `boolean` | `true` | +| `CORS_ALLOW_HEADERS` | Value of the `Access-Control-Allow-Headers` response header on the REST API, which controls the request headers a browser may send cross-origin. The default is the long list of headers Weaviate itself reads, including `Content-Type`, `Authorization` and the per-provider API-key headers. Default: the built-in header list | `string - comma separated names` | `Content-Type, Authorization` | +| `CORS_ALLOW_METHODS` | Value of the `Access-Control-Allow-Methods` response header on the REST API, which controls the HTTP methods a browser may use cross-origin. Default: `*` | `string - comma separated names` | `GET, POST, OPTIONS` | +| `CORS_ALLOW_ORIGIN` | Value of the `Access-Control-Allow-Origin` response header on the REST API, which controls the origins a browser may call Weaviate from. Set this to reach Weaviate directly from browser code on a specific site. Default: `*` | `string` | `https://example.com` | | `DEFAULT_QUANTIZATION` | Default quantization technique - can be overridden by the quantization method specified in the collection definition. Available values: `rq-8`, `rq-1`, `pq`, `bq`, `sq` and `none`. Default: `none`.

Note: If the selected quantization method isn't supported for the index type of a collection (for example PQ & SQ aren't supported for the flat index), the quantization won't be applied to that collection.

Added in `v1.33` | `string` | `rq-8` | | `DEFAULT_SHARDING_COUNT` | Default `desiredCount` for new single-tenant collections, used when the collection definition does not specify one. An explicit `desiredCount` in the class creation request still takes precedence. A value of `0` (default) uses the cluster node count. Multi-tenant collections are unaffected. Must be `<= 512`. Runtime-configurable. Default: `0`
Added in `v1.37` | `string - number` | `12` | | `DEFAULT_VECTOR_INDEX` | Default vector index type for new collections (and named vectors), used when the collection definition does not specify one. An explicit `vectorIndexType` in the collection definition still takes precedence. Available values: `hnsw`, `flat`, `dynamic`, and `hfresh`. Runtime-configurable. Default: `hnsw`
Added in `v1.37.3` | `string` | `flat` | | `DEFAULT_VECTORIZER_MODULE` | Default vectorizer module - can be overridden by the vectorizer in the collection definition. | `string` | `text2vec-contextionary` | -| `API_BASED_MODULES_DISABLED` | Weaviate automatically enables the usage of all [API based modules](../../../weaviate/model-providers/index.md#api-based). Set this variable to `true` in order to limit access and only allow specific modules through the [`ENABLE_MODULES`](#ENABLE_MODULES) variable. Default: `false`
Added in `v1.33` | `boolean` | `true` | +| `API_BASED_MODULES_DISABLED` | Weaviate automatically enables the usage of all [API-based modules](../../../weaviate/model-providers/index.md#api-based). Set this variable to `true` in order to limit access and only allow specific modules through the [`ENABLE_MODULES`](#ENABLE_MODULES) variable. Default: `false`
Added in `v1.33` | `boolean` | `true` | | `DISABLE_LAZY_LOAD_SHARDS` | When `false`, enable lazy shard loading to improve mean time to recovery in multi-tenant deployments. **Deprecated in `v1.36.6`.** Use `LAZY_LOAD_SHARD_COUNT_THRESHOLD` and `LAZY_LOAD_SHARD_SIZE_THRESHOLD_GB` instead. Weaviate now auto-detects when lazy loading is needed per collection. | `string` | `false` | | `DISABLE_TELEMETRY` | Disable [telemetry](/deploy/configuration/telemetry.md) data collection | boolean | `false` | | `DISK_USE_READONLY_PERCENTAGE` | If disk usage is higher than the given percentage all shards on the affected node will be marked as `READONLY`, meaning all future write requests will fail. See [Disk Pressure Warnings and Limits for details](/deploy/configuration/persistence.md#disk-pressure-warnings-and-limits). | `string - number` | `90` | @@ -45,7 +48,7 @@ import APITable from '@site/src/components/APITable'; | `ENABLE_MODULES` | Specify which modules are enabled and can be used. | `string - comma separated names` | `text2vec-openai,generative-openai` | | `ENABLE_TOKENIZER_GSE` | Enable the [`GSE` tokenizer](/weaviate/config-refs/collections.mdx) for use | `boolean` | `true` | | `ENABLE_TOKENIZER_KAGOME_JA` | Enable the [`Kagome` tokenizer for Japanese](/weaviate/config-refs/collections.mdx) for use | `boolean` | `true` | -| `ENABLE_TOKENIZER_KAGOME_KR` | Enable the [`Kagome` tokenizer for Korean](/weaviate/config-refs/collections.mdx#) for use | `boolean` | `true` | +| `ENABLE_TOKENIZER_KAGOME_KR` | Enable the [`Kagome` tokenizer for Korean](/weaviate/config-refs/collections.mdx) for use | `boolean` | `true` | | `EXPORT_DEFAULT_BUCKET` | Storage bucket name for [collection exports](/docs/deploy/configuration/export.md). Required for S3, GCS, and Azure backends.
Added in `v1.37` | `string` | `my-export-bucket` | | `EXPORT_DEFAULT_PATH` | Optional base path prefix for exported files within the bucket for [collection exports](/docs/deploy/configuration/export.md). Defaults to `""` (no prefix). _Changed in `v1.37.1`: previously required to be explicitly set._
Added in `v1.37` | `string` | `exports/my-cluster` | | `EXPORT_ENABLED` | Enable the [collection export](/docs/deploy/configuration/export.md) API. Default: `false`
Added in `v1.37` | `boolean` | `true` | @@ -57,8 +60,8 @@ import APITable from '@site/src/components/APITable'; | `INVERTED_SORTER_DISABLED` | Forces the "objects bucket" strategy and doesn't consider inverted sorting. Most users should never set this flag; it exists for benchmarking and as a safety net. Default: `false` | `boolean` | `false` | | `GO_PROFILING_DISABLE` | If `true`, disables Go profiling. Default: `false`. | `boolean` | `false` | | `GO_PROFILING_PORT` | Sets the port for the Go profiler. Default: `6060` | `integer` | `6060` | -| `DEBUG_ENDPOINTS_ENABLED` | Gate for the debug HTTP listener (the profiling port set by `GO_PROFILING_PORT`, default `6060`), which serves Weaviate's **unauthenticated** internal debug and profiling endpoints — `/debug/config`, Go profiling (`/debug/pprof/*`, `/debug/fgprof`), and various maintenance and diagnostic routes. [Runtime-configurable](/deploy/configuration/env-vars/runtime-config.md) via the `debug_endpoints_enabled` override. Default: `false`. `GO_PROFILING_DISABLE` still controls whether the listener binds at all.
Added in `v1.37.9` | `boolean` | `true` | -| `GRPC_MAX_MESSAGE_SIZE` | Maximum gRPC message size in bytes. Default: 10MB | `string - number` | `2000000000` | +| `DEBUG_ENDPOINTS_ENABLED` | Gate for the debug HTTP listener (the profiling port set by `GO_PROFILING_PORT`, default `6060`), which serves Weaviate's **unauthenticated** internal debug and profiling endpoints: `/debug/config`, Go profiling (`/debug/pprof/*`, `/debug/fgprof`), and various maintenance and diagnostic routes. [Runtime-configurable](/deploy/configuration/env-vars/runtime-config.md) via the `debug_endpoints_enabled` override. Default: `false`. `GO_PROFILING_DISABLE` still controls whether the listener binds at all.
Added in `v1.37.9` | `boolean` | `true` | +| `GRPC_MAX_MESSAGE_SIZE` | Maximum gRPC message size in bytes. Requests larger than this limit (e.g. a large `insert_many` call) are rejected. Default: `104858000` (approximately 100 MB) | `string - number` | `2000000000` | | `GRPC_PORT` | The port on which Weaviate's gRPC server listens for incoming requests. Default: `50051` | `string - number` | `50052` | | `HNSW_GEO_INDEX_EF` | Balance geo index search speed and recall. This value controls the search depth for geo-based queries. Default: `800`
Added in `v1.31.22` | `string - number` | `1000` | | `LAZY_LOAD_SHARD_COUNT_THRESHOLD` | Number of shards (tenants) in a collection before lazy shard loading activates. Set to `0` to force lazy loading for all collections. Default: `1000`. See [dynamic lazy shard loading](/weaviate/concepts/storage#dynamic-lazy-shard-loading).
Added in `v1.36.6` | `string - number` | `1000` | @@ -80,7 +83,7 @@ import APITable from '@site/src/components/APITable'; | `OBJECTS_TTL_DELETE_SCHEDULE` | Schedule for deleting expired objects. Accepts standard 5-field cron format, 6-field (with seconds), 7-field (with seconds and year), descriptors (`@yearly`, `@monthly`, `@weekly`, `@daily`, `@hourly`), or hash expressions. Default: `""` (disabled)
Added in `v1.36` | `string - cron format` | `0 */6 * * *` (every 6 hours) | | `OBJECTS_TTL_PAUSE_DURATION` | How long to pause the TTL deletion process between batches. Longer pauses reduce resource pressure but slow down cleanup. If `0` there is no pause. Can be modified at runtime. Default: `1m`
Added in `v1.36` | `string - duration` | `20s`, `2m` | | `OBJECTS_TTL_PAUSE_EVERY_NO_BATCHES` | Number of batch deletions to process before pausing. With the default batch size of 10,000, a pause occurs every 100,000 deleted objects. If `0` there is no pause. Can be modified at runtime. Default: `10`
Added in `v1.36` | `string - number` | `3` | -| `OPERATIONAL_MODE` | Sets the [mode of operation](../status.md#operational-modes) for the instance. Options: `READ_WRITE` (default), `READ_ONLY`, `WRITE_ONLY`, `SCALE_OUT`. Limits available operations based on the mode selected. | `string` | `READ_WRITE` | +| `OPERATIONAL_MODE` | Sets the [mode of operation](../status.md#operational-modes) for the instance, limiting the available operations based on the mode selected. Options: `ReadWrite` (default), `ReadOnly`, `WriteOnly`, `ScaleOut`. These values are case-sensitive and matched exactly. An unrecognized value (such as `READ_ONLY`) silently falls back to the default `ReadWrite`, without a warning or an error, so a node you intended to restrict stays fully writable. | `string` | `ReadWrite` | | `ORIGIN` | Set the http(s) origin for Weaviate | `string - HTTP origin` | `https://my-weaviate-deployment.com` | | `PERSISTENCE_DATA_PATH` | Path to the Weaviate data store.
[Note about file systems and performance](/weaviate/concepts/resources.md#file-system). | `string - file path` | `/var/lib/weaviate`
Defaults to `./data`| | `PERSISTENCE_HNSW_DISABLE_SNAPSHOTS` | **Deprecated in `v1.39`.** Ignored, and logs a startup warning. In `v1.31` through `v1.38`: if set, HNSW snapshotting is disabled. Default: `false` (enabled) as of `v1.36`; `true` (disabled) in `v1.31` through `v1.35`. See [snapshot configuration before `v1.39`](/weaviate/concepts/storage.md#pre-v1-39-configuration).
Added in `v1.31` | `boolean` | `false` | @@ -96,11 +99,12 @@ import APITable from '@site/src/components/APITable'; | `QUERY_BOOST_DEFAULT_DEPTH` | Default candidate-pool size used when a [Boost](/weaviate/search/boost.md) query does not set its own `depth`. The primary search retrieves this many candidates before the boost rescorer runs. Must be a positive integer and is hard-capped by `QUERY_MAXIMUM_RESULTS`. Default: `100`
Added in `v1.38` | `string - number` | `200` | | `QUERY_CROSS_REFERENCE_DEPTH_LIMIT` | Sets the maximum depth of cross-references to be resolved in a query. Defaults to 5. | `string - number` | `3` | | `QUERY_DEFAULTS_LIMIT` | Sets the default number of objects to be returned in a query. | `string - number` | `25`
Defaults to `10`| +| `QUERY_HYBRID_MAXIMUM_RESULTS` | Minimum size of the candidate pool that each leg of a [hybrid search](/weaviate/search/hybrid.md) retrieves before fusion. Each of the keyword and vector sub-searches fetches at least `offset` plus this many candidates, so that paging deeper into a hybrid result set stays consistent. Raise it if hybrid results shift between pages; higher values cost more work per query. Default: `100`
Added in `v1.32`, and backported to `v1.30.12` and `v1.31.6` | `string - number` | `200` | | `QUERY_MAXIMUM_RESULTS` | Sets the maximum total number of objects that can be retrieved. | `string - number` | `10000` | | `QUERY_SLOW_LOG_ENABLED` | Log slow queries for debugging. Requires a restart to update. | `boolean` | `False` | | `QUERY_SLOW_LOG_THRESHOLD` | Set a threshold time for slow query logging. Requires a restart to update. | `string` | `2s`
Values are times: `3h`, `2s`, `100ms` | | `REINDEX_SET_TO_ROARINGSET_AT_STARTUP` | Allow Weaviate to perform a one-off re-indexing to use Roaring Bitmaps. | `boolean` | `true` | -| `REVECTORIZE_CHECK_DISABLED` | Disables the optimization that reuses an object's existing vector when it is updated and none of its vectorized properties changed. By default (`false`) this check runs and skips unnecessary re-vectorization; set to `true` to re-vectorize on every update. Disabling removes a read-before-write — which can raise write throughput, or force always-fresh vectors — at the cost of an extra embedding call (and API cost, for remote vectorizers) per update. Default: `false` | `boolean` | `false` | +| `REVECTORIZE_CHECK_DISABLED` | Disables the optimization that reuses an object's existing vector when it is updated and none of its vectorized properties changed. By default (`false`) this check runs and skips unnecessary re-vectorization; set to `true` to re-vectorize on every update. Disabling removes a read-before-write. That can raise write throughput, and it guarantees a fresh vector on every update, at the cost of one extra embedding call per update (and the associated API cost, for remote vectorizers). Default: `false` | `boolean` | `false` | | `TENANT_ACTIVITY_READ_LOG_LEVEL` | Sets the log level for tenant read activity. Useful for analysis or debugging purposes. Default: `debug` | `string` | `info` | | `TENANT_ACTIVITY_WRITE_LOG_LEVEL` | Sets the log level for tenant write activity. Useful for analysis or debugging purposes. Default: `debug` | `string` | `info` | | `TOKENIZER_CONCURRENCY_COUNT` | Limit the combined number of GSE and Kagome tokenizers running at the same time. Default: `GOMAXPROCS` | `string - number` | `NUMBER_OF_CPU_CORES` | @@ -224,7 +228,7 @@ For more information on authentication and authorization, see the [Authenticatio | `CLUSTER_GOSSIP_BIND_PORT` | Port for exchanging network state information. | `string - number` | `7102` | | `CLUSTER_HOSTNAME` | Hostname of a node. Always set this value if the default OS hostname might change over time. | `string` | `node1` | | `CLUSTER_JOIN` | The service name of the "founding" member node in a cluster setup | `string` | `weaviate-node-1:7100` | -| `HNSW_STARTUP_WAIT_FOR_VECTOR_CACHE` | If `true`, vector cache prefill is synchronous when a node starts. The node reports ready to serve when the cache is hot. Default changed to `true` in `v1.36.6`. For collections where [dynamic lazy shard loading](/weaviate/concepts/storage#dynamic-lazy-shard-loading) is active, this is always overridden to `false` regardless of the configured value. The configured value only applies to eagerly-loaded collections. | `boolean` | `true` | +| `HNSW_STARTUP_WAIT_FOR_VECTOR_CACHE` | If `true`, vector cache prefill is synchronous when a node starts. The node reports ready to serve when the cache is hot. Default changed to `true` in `v1.36.6`. **Deprecated in `v1.36.6`.** Setting it still overrides auto-detection, but Weaviate logs a deprecation warning at startup and the variable will be removed in a future version. When it is unset, prefill behavior is governed by [dynamic lazy shard loading](/weaviate/concepts/storage#dynamic-lazy-shard-loading). For collections where dynamic lazy shard loading is active, this is always overridden to `false` regardless of the configured value. The configured value only applies to eagerly-loaded collections. | `boolean` | `true` | | `COLLECTION_RETRIEVAL_STRATEGY`| Set collection definition retrieval behavior for a data request.

  • `LeaderOnly` (default): Always requests the definition from the leader node.
  • `LocalOnly`: Always use the local definition
  • `LeaderOnMismatch`: Requests the definition if outdated.
([Read more](/weaviate/concepts/replication-architecture/consistency.md#collection-definition-requests-in-queries)) | `string` | `LeaderOnly` | | `RAFT_BOOTSTRAP_EXPECT` | The number of voter notes at bootstrapping time | `string - number` | `1` | | `RAFT_BOOTSTRAP_TIMEOUT` | The time in seconds to wait for the cluster to bootstrap | `string - number` | `90` | diff --git a/docs/deploy/configuration/env-vars/runtime-config.md b/docs/deploy/configuration/env-vars/runtime-config.md index f59609fdf..722cafaea 100644 --- a/docs/deploy/configuration/env-vars/runtime-config.md +++ b/docs/deploy/configuration/env-vars/runtime-config.md @@ -61,6 +61,7 @@ The following overrides are currently supported: | `async_replication_hashtree_init_concurrency` | `ASYNC_REPLICATION_HASHTREE_INIT_CONCURRENCY`| | `async_replication_cluster_max_workers` _(removed in `v1.38`)_ | `ASYNC_REPLICATION_CLUSTER_MAX_WORKERS` _(removed in `v1.38`)_ | | `autoschema_enabled` | `AUTOSCHEMA_ENABLED` | +| `backup_max_individual_files` | `BACKUP_MAX_INDIVIDUAL_FILES` | | `debug_endpoints_enabled` | `DEBUG_ENDPOINTS_ENABLED` | | `default_quantization` | `DEFAULT_QUANTIZATION` | | `default_sharding_count` | `DEFAULT_SHARDING_COUNT` | @@ -121,14 +122,14 @@ The following overrides are currently supported: ### MCP -Added in `v1.38`. Toggling these at runtime does not require a cluster restart — the HTTP handlers stay registered and per-request checks pick up the new value. See [MCP server — Toggle without restart](/weaviate/configuration/mcp-server.mdx#toggle-without-restart) for behavior details. +Added in `v1.38`. Toggling these at runtime does not require a cluster restart: the HTTP handlers stay registered and per-request checks pick up the new value. See [MCP server: Toggle without restart](/weaviate/configuration/mcp-server.mdx#toggle-without-restart) for behavior details. | Runtime override name | Environment variable name | | :-------------------------------- | :---------------------------------- | | `mcp_server_enabled` | `MCP_SERVER_ENABLED` | | `mcp_server_write_access_enabled` | `MCP_SERVER_WRITE_ACCESS_ENABLED` | -`MCP_SERVER_CONFIG_PATH` is intentionally **not** runtime-configurable — tool descriptions are baked into the tool schemas at registration. +`MCP_SERVER_CONFIG_PATH` is intentionally **not** runtime-configurable, because tool descriptions are baked into the tool schemas at registration. Refer to the [Environment variables](./index.md) page for descriptions on each configuration option diff --git a/docs/deploy/configuration/export.md b/docs/deploy/configuration/export.md index b172b9815..7ede285ab 100644 --- a/docs/deploy/configuration/export.md +++ b/docs/deploy/configuration/export.md @@ -10,7 +10,7 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl import PyCode from '!!raw-loader!/\_includes/code/howto/configure.export.py'; import CsCode from '!!raw-loader!/\_includes/code/csharp/ManageDataExportTest.cs'; -:::caution Preview — added in `v1.37` +:::caution Preview (added in `v1.37`) This is a preview feature. The API may change in future releases. ::: @@ -206,7 +206,7 @@ Files are named `{collection}_{shard}_{rangeIndex}.parquet`. Collection and tena | COLD | Exported directly from disk without loading into memory (remains COLD). | | OFFLOADED | Skipped. The skip reason is recorded in the shard status. | -The tenant list is snapshotted when the export is created — tenants created during the export are not included. +The tenant list is snapshotted when the export is created. Tenants created during the export are not included. ## Permissions diff --git a/docs/deploy/configuration/monitoring.md b/docs/deploy/configuration/monitoring.md index 5268dc341..4cecd3f0d 100644 --- a/docs/deploy/configuration/monitoring.md +++ b/docs/deploy/configuration/monitoring.md @@ -492,10 +492,10 @@ Added in `v1.38`. These metrics track tool traffic, latency, auth failures, and Label values: -- **`tool`** — the MCP tool name (e.g. `weaviate-query-hybrid`, `weaviate-objects-upsert`). -- **`status`** — `success` · `error` · `denied` · `write_disabled`. `denied` covers authorization failures classified via the `Forbidden` / `Unauthenticated` error families. `write_disabled` is emitted when a write call hits the runtime guard. -- **`reason`** — `missing_token` · `invalid_token` · `forbidden` · `unauthenticated`. `missing_token` and `invalid_token` are detected at the principal-extraction step; `forbidden` and `unauthenticated` are detected at authorization time. -- **`write_access`** — `enabled` / `disabled`, matching the live state of `MCP_SERVER_WRITE_ACCESS_ENABLED` at the time of the `tools/list` call. +- **`tool`**: the MCP tool name (e.g. `weaviate-query-hybrid`, `weaviate-objects-upsert`). +- **`status`**: `success` · `error` · `denied` · `write_disabled`. `denied` covers authorization failures classified via the `Forbidden` / `Unauthenticated` error families. `write_disabled` is emitted when a write call hits the runtime guard. +- **`reason`**: `missing_token` · `invalid_token` · `forbidden` · `unauthenticated`. `missing_token` and `invalid_token` are detected at the principal-extraction step; `forbidden` and `unauthenticated` are detected at authorization time. +- **`write_access`**: `enabled` / `disabled`, matching the live state of `MCP_SERVER_WRITE_ACCESS_ENABLED` at the time of the `tools/list` call. --- @@ -522,7 +522,7 @@ your uses perfectly: ## Query profiling -For per-query performance analysis, Weaviate provides [query profiling](/weaviate/search/query-profile.md). Unlike Prometheus metrics which show aggregate performance, query profiling provides per-shard timing breakdowns for individual queries — useful for diagnosing specific slow queries. +For per-query performance analysis, Weaviate provides [query profiling](/weaviate/search/query-profile.md). Unlike Prometheus metrics which show aggregate performance, query profiling provides per-shard timing breakdowns for individual queries, which is useful for diagnosing specific slow queries. ## `nodes` API Endpoint diff --git a/docs/deploy/configuration/replica-movement.mdx b/docs/deploy/configuration/replica-movement.mdx index 87f874e69..86899d478 100644 --- a/docs/deploy/configuration/replica-movement.mdx +++ b/docs/deploy/configuration/replica-movement.mdx @@ -146,10 +146,13 @@ The movement operation can have one of the following states: - `REGISTERED` - `HYDRATING` - `FINALIZING` +- `INTEGRATING` - `DEHYDRATING` - `READY` - `CANCELLED` +`INTEGRATING` was added in `v1.38.0`. On earlier versions the operation goes straight from `FINALIZING` to `DEHYDRATING` for a move, or to `READY` for a copy. + To learn more about the replication states, check out [Concepts: Replication architecture](/docs/weaviate/concepts/replication-architecture/consistency.md#replica-movement). ::: diff --git a/docs/deploy/configuration/replication.md b/docs/deploy/configuration/replication.md index 989a1ed99..2d8c81e88 100644 --- a/docs/deploy/configuration/replication.md +++ b/docs/deploy/configuration/replication.md @@ -48,7 +48,7 @@ When Weaviate detects inconsistent data across nodes, it attempts to repair the Weaviate offers [async replication](/weaviate/concepts/replication-architecture/consistency.md#async-replication) to proactively detect inconsistencies. In earlier versions, Weaviate uses a [repair-on-read](/weaviate/concepts/replication-architecture/consistency.md#repair-on-read) strategy to repair inconsistencies at read time. -Repair-on-read is automatic. As of Weaviate `v1.38`, async replication is also **enabled by default** for any collection with a replication factor greater than `1` — there is no longer a per-collection flag to switch it on. To turn it off cluster-wide, set the [`ASYNC_REPLICATION_DISABLED`](/deploy/configuration/env-vars/index.md#async-replication) environment variable to `true`. The `replicationConfig` section is used to set the replication factor and to fine-tune async replication via `asyncConfig`: +Repair-on-read is automatic. As of Weaviate `v1.38`, async replication is also **enabled by default** for any collection with a replication factor greater than `1`. There is no longer a per-collection flag to switch it on. To turn it off cluster-wide, set the [`ASYNC_REPLICATION_DISABLED`](/deploy/configuration/env-vars/index.md#async-replication) environment variable to `true`. The `replicationConfig` section is used to set the replication factor and to fine-tune async replication via `asyncConfig`: import ReplicationConfigWithAsyncRepair from '/\_includes/code/configuration/replication-consistency.mdx'; diff --git a/docs/deploy/faqs/troubleshooting.md b/docs/deploy/faqs/troubleshooting.md index a5c013f29..c877c2d7e 100644 --- a/docs/deploy/faqs/troubleshooting.md +++ b/docs/deploy/faqs/troubleshooting.md @@ -39,7 +39,7 @@ To confirm and identify the issue, you'll want to first run the same query multi #### Resolving the issue -Check your settings to check if you have asynchronous replication enabled. If `async_replication_disabled` is set to "true" then you'll need to set that variable to "false." Once it is enabled, the logs will show messages that indicate successful peers checks and synchronization for the nodes. +Check whether asynchronous replication is enabled. If `ASYNC_REPLICATION_DISABLED` is set to `true`, set it to `false`. Once async replication is enabled, the logs will show successful peer checks and node synchronization.
@@ -51,11 +51,21 @@ Check your settings to check if you have asynchronous replication enabled. If `a #### Identifying the issue -To confirm and identify the issue, you'll want to first run the same query multiple times to confirm that the results are inconsistent. If the inconsistent results are persisting, then you probably have asynchronous replication disabled for your deployment. +Start with the logs of a node that is failing to join. A membership problem reads differently from a data problem: you'll see repeated attempts to contact the founding member, gossip timeouts, or Raft messages about an election that never settles on a leader. A node in this state can still pass its own health checks, so if the live endpoint answers while the node stays outside the cluster, the process is healthy and the problem is membership. + +To confirm it, query the `/v1/cluster/statistics` endpoint. If it reports fewer nodes than you expect, or the top-level `synchronized` field is `false`, then your cluster has not reached consensus. #### Resolving the issue -Check your settings to check if you have asynchronous replication enabled. If `async_replication_disabled` is set to "true" then you'll need to set that variable to "false." Once it is enabled, the logs will show messages that indicate successful peers checks and synchronization for the nodes. Additionally, test the live and ready REST endpoints. and check the network configuration of the nodes. +Work outward from each node's own identity to the network between the nodes. + +- Point every joining node at the founding member with [`CLUSTER_JOIN`](/deploy/configuration/env-vars/index.md#CLUSTER_JOIN). The value is the service name and gossip port of that founding member, such as `weaviate-node-1:7100`, and every joining node must name the same one. +- Set [`CLUSTER_HOSTNAME`](/deploy/configuration/env-vars/index.md#CLUSTER_HOSTNAME) explicitly on every node. If you leave the hostname to the operating system and it changes across a restart, the node rejoins under a new name while the cluster is still holding a place for the old one. If the hostname cannot be resolved through DNS, set [`CLUSTER_ADVERTISE_ADDR`](/deploy/configuration/env-vars/index.md#CLUSTER_ADVERTISE_ADDR) to advertise the node's address directly. +- Give each node a [`CLUSTER_GOSSIP_BIND_PORT`](/deploy/configuration/env-vars/index.md#CLUSTER_GOSSIP_BIND_PORT), used to exchange network state information, and a [`CLUSTER_DATA_BIND_PORT`](/deploy/configuration/env-vars/index.md#CLUSTER_DATA_BIND_PORT), used to exchange data. By convention the data port is one higher than the gossip port. Then confirm that every node can actually reach every other node on both of those ports. A firewall rule, an unpublished container port, or a network policy that only exposes the HTTP port will let a node start up perfectly well and still leave it unable to find anyone. +- For consensus specifically, check the Raft settings. [`RAFT_JOIN`](/deploy/configuration/env-vars/index.md#RAFT_JOIN) names the voter nodes, and [`RAFT_BOOTSTRAP_EXPECT`](/deploy/configuration/env-vars/index.md#RAFT_BOOTSTRAP_EXPECT) sets how many voters the cluster waits for at bootstrap. If you set `RAFT_JOIN`, you must adjust `RAFT_BOOTSTRAP_EXPECT` by hand to match the number of voters you listed. When the two disagree, the cluster waits for a member that will never arrive. + +Once the nodes are talking, check `/v1/cluster/statistics` again. Every node should appear in the response, and `synchronized` should be `true`. + ### You've downgraded and now your clusters won't reach the `Ready` state. diff --git a/docs/deploy/installation-guides/docker-installation.md b/docs/deploy/installation-guides/docker-installation.md index 7da7faaa0..0d1f59c02 100644 --- a/docs/deploy/installation-guides/docker-installation.md +++ b/docs/deploy/installation-guides/docker-installation.md @@ -418,7 +418,7 @@ Alternatively you can run docker compose entirely detached with `docker compose ### Set `CLUSTER_HOSTNAME` if it may change over time -In some systems, the cluster hostname may change over time. This is known to create issues with a single-node Weaviate deployment. To avoid this, set the `CLUSTER_HOSTNAME` environment variable in the `values.yaml` file to the cluster hostname. +In some systems, the cluster hostname may change over time. This is known to create issues with a single-node Weaviate deployment. To avoid this, set the `CLUSTER_HOSTNAME` environment variable in your `docker-compose.yml` file to the cluster hostname. ```yaml --- diff --git a/docs/deploy/installation-guides/eks.md b/docs/deploy/installation-guides/eks.md index 36b5d4784..5a97e91f2 100644 --- a/docs/deploy/installation-guides/eks.md +++ b/docs/deploy/installation-guides/eks.md @@ -203,7 +203,7 @@ replicas: 3 ```bash helm upgrade --install weaviate weaviate/weaviate \ --namespace weaviate \ - --values values.yaml \ + --values values.yaml ``` #### Verify your deployment diff --git a/docs/deploy/installation-guides/embedded.md b/docs/deploy/installation-guides/embedded.md index 686787c63..e00be92cc 100644 --- a/docs/deploy/installation-guides/embedded.md +++ b/docs/deploy/installation-guides/embedded.md @@ -43,11 +43,20 @@ To configure Embedded Weaviate, set these variables in your instantiation code o | Parameter | Type | Default | Description | | :-- | :-- | :-- | :-- | | `additional_env_vars` | string | None. | Pass additional environment variables, such as API keys, to the server. | -| `binary_path` | string | varies | Binary download directory. If the binary is not present, the client downloads the binary.

If `XDG_CACHE_HOME` is set, the default is: `XDG_CACHE_HOME/weaviate-embedded/`

If `XDG_CACHE_HOME` is not set, the default is: `~/.cache/weaviate-embedded` | +| `binary_path` | string | varies | Binary download directory. If the binary is not present, the client downloads the binary.

If `XDG_CACHE_HOME` is set, its value is used verbatim as the default. No subdirectory is appended, so `XDG_CACHE_HOME=/foo` makes the default exactly `/foo`.

If `XDG_CACHE_HOME` is not set, the default is: `~/.cache/weaviate-embedded/` | +| `grpc_port` | integer | 50060 | The Weaviate server gRPC port. The client passes this value to the server as `GRPC_PORT`. | | `hostname` | string | 127.0.0.1 | Hostname or IP address | -| `persistence_data_path` | string | varies | Data storage directory.

If `XDG_DATA_HOME` is set, the default is: `XDG_DATA_HOME/weaviate/`

If `XDG_DATA_HOME` is not set, the default is: `~/.local/share/weaviate` | +| `persistence_data_path` | string | varies | Data storage directory.

If `XDG_DATA_HOME` is set, its value is used verbatim as the default. No subdirectory is appended, so `XDG_DATA_HOME=/foo` makes the default exactly `/foo`.

If `XDG_DATA_HOME` is not set, the default is: `~/.local/share/weaviate` | | `port` | integer | 8079 | The Weaviate server request port. | -| `version` | string | Latest stable | Specify the version with one of the following:
-`"latest"`
- The version number as a string: `"1.19.6"`
- The URL of a Weaviate binary ([See below](/deploy/installation-guides/embedded.md#file-url)) | +| `version` | string | A version pinned in the client (see note) | Specify the version with one of the following:
-`"latest"`
- The version number as a string: `"1.19.6"`
- The URL of a Weaviate binary ([See below](/deploy/installation-guides/embedded.md#file-url)) | + +:::note Set `version` explicitly + +If you do not set `version`, Embedded Weaviate does not run the latest Weaviate release. The client falls back to a single Weaviate version that is fixed in the client source code when that client release is published. That pin only moves when the client is released again, so it can be several Weaviate minor releases behind the current one, and upgrading your client can also change which Weaviate version your embedded instance runs. + +Set `version` explicitly so you control which server version you get. Use `"latest"` to resolve the newest Weaviate release at startup, or pin a version number such as `"1.19.6"` for a reproducible environment. The version in use is printed in the embedded server startup logs. + +::: :::warning Do not modify `XDG_CACHE_HOME` or `XDG_DATA_HOME` The `XDG_DATA_HOME` and `XDG_CACHE_HOME` environment variables are widely used system variables. If you modify them, you may break other applications. @@ -63,7 +72,7 @@ The following modules are enabled by default: - `text2vec-huggingface` - `text2vec-openai` -To enabled additional modules, add them to your instantiation code. +To enable additional modules, add them to your instantiation code. For example, to add the `backup-s3` module, instantiate your client like this: @@ -122,7 +131,7 @@ Embedded Weaviate is supported for Python and TypeScript clients. ### Python clients -[Python](docs/weaviate/client-libraries/python/index.mdx) v3 client support is new in `v3.15.4` for Linux and `v3.21.0` for macOS. The Python client v4 requires server version v1.23.7 or higher. +Embedded Weaviate is built into the [Python client](docs/weaviate/client-libraries/python/index.mdx), so there is no separate package to install. The Python client requires Weaviate `v1.23.7` or later. ### TypeScript clients diff --git a/docs/deploy/installation-guides/k8s-installation.md b/docs/deploy/installation-guides/k8s-installation.md index fd7386d87..f7dec4114 100644 --- a/docs/deploy/installation-guides/k8s-installation.md +++ b/docs/deploy/installation-guides/k8s-installation.md @@ -261,7 +261,7 @@ In some systems, the cluster hostname may change over time. This is known to cre ```yaml env: - - CLUSTER_HOSTNAME: "node-1" + CLUSTER_HOSTNAME: "node-1" ``` ## Questions and feedback diff --git a/docs/deploy/installation-guides/spcs-integration.mdx b/docs/deploy/installation-guides/spcs-integration.mdx index 8c724bbd7..862614201 100644 --- a/docs/deploy/installation-guides/spcs-integration.mdx +++ b/docs/deploy/installation-guides/spcs-integration.mdx @@ -316,7 +316,7 @@ response = collection.query.near_text(query="animal",limit=2, include_vector=Tru for o in response.objects: print(o.vector) -# Hybrid search client.close() +# Hybrid search response = collection.query.hybrid( query="animals", limit=5 diff --git a/docs/deploy/production/aws/hardening-eks.md b/docs/deploy/production/aws/hardening-eks.md index c338082aa..ce516d9aa 100644 --- a/docs/deploy/production/aws/hardening-eks.md +++ b/docs/deploy/production/aws/hardening-eks.md @@ -4,7 +4,7 @@ sidebar_label: Hardening EKS deployments description: Harden your self-hosted Weaviate deployment on Amazon EKS. --- -You've got a Weaviate deployment running on EKS—awesome! Now it's time to make it production-ready and secure. +You've got a Weaviate deployment running on EKS. Awesome! Now it's time to make it production-ready and secure. While Weaviate is a powerful vector database, like any self-hosted service, it needs proper security hardening. Your first deployment focused on getting things running; for production, we need to tighten things up. @@ -254,7 +254,7 @@ Set up alerts for issues that matter: - 🚨 Backup failures :::tip Alert Fatigue -Don't go overboard—alert fatigue is real. Focus on actionable, critical alerts. +Don't go overboard. Alert fatigue is real. Focus on actionable, critical alerts. ::: ### Use proper dashboards diff --git a/docs/deploy/production/aws/network-security.md b/docs/deploy/production/aws/network-security.md index 6be3645f5..29b4f9e88 100644 --- a/docs/deploy/production/aws/network-security.md +++ b/docs/deploy/production/aws/network-security.md @@ -28,7 +28,7 @@ Access control is the cornerstone of this network security strategy. It implemen #### Private subnet strategy -Network isolation is the foundation of our this strategy. Critical infrastructure should reside in private subnets with no direct internet connectivity. This eliminates internet-based attacks and forces all access through controlled entry points. +Network isolation is the foundation of this strategy. Critical infrastructure should reside in private subnets with no direct internet connectivity. This eliminates internet-based attacks and forces all access through controlled entry points. #### Core components @@ -124,8 +124,6 @@ Data transfer for backups and application data can be secured, this is what is n ### Scaling and performance strategies -#### Application load balancer (ALB) configuration - #### SSL/TLS management - Automated certificate provisioning through AWS Certificate Manager. @@ -166,8 +164,6 @@ Secure scaling policies maintains security posture during capacity changes. ### High availability and disaster recovery -#### Multi-AZ architecture - #### Weaviate configuration - **Minimum** 3 replicas distributed across AZs. @@ -185,7 +181,7 @@ Secure scaling policies maintains security posture during capacity changes. #### Network visibility - VPC flow logs to capture traffic metadata for security analysis. -- Real0time streaming to SIEMs. +- Real-time streaming to SIEMs. - Baseline establishment and anomaly detection. #### Security monitoring diff --git a/docs/deploy/production/kubernetes/get-to-production.md b/docs/deploy/production/kubernetes/get-to-production.md index 5bd68a0d7..21c772487 100644 --- a/docs/deploy/production/kubernetes/get-to-production.md +++ b/docs/deploy/production/kubernetes/get-to-production.md @@ -53,12 +53,12 @@ Check out the Academy course [“Run Weaviate on Kubernetes”](https://docs.wea An example of RBAC enabled on your Helm chart ```yaml - authorization: +authorization: rbac: enabled: true - root_users: - - admin_user1 - - admin_user2 + root_users: + - admin_user1 + - admin_user2 ``` diff --git a/docs/query-agent/_includes/code/search_mode.mts b/docs/query-agent/_includes/code/search_mode.mts index fc9f355b8..5b386734b 100644 --- a/docs/query-agent/_includes/code/search_mode.mts +++ b/docs/query-agent/_includes/code/search_mode.mts @@ -102,4 +102,15 @@ for (const obj of filteringResponse.searchResults.objects) { } // END FilteringExample +// START EffortExample +const effortResponse = await qa.search("What are Setwise Rerankers?", { + limit: 10, + effort: "ultrahigh", +}); + +for (const obj of effortResponse.searchResults.objects) { + console.log(obj.properties); +} +// END EffortExample + await client.close(); diff --git a/docs/query-agent/_includes/code/search_mode.py b/docs/query-agent/_includes/code/search_mode.py index d1bab68d4..69d95c89f 100644 --- a/docs/query-agent/_includes/code/search_mode.py +++ b/docs/query-agent/_includes/code/search_mode.py @@ -107,6 +107,17 @@ print(f"Product: {obj.properties['name']} - ${obj.properties['price']}") # END FilteringExample +# START EffortExample +search_response = qa.search( + "What are Setwise Rerankers?", + limit=10, + effort="ultrahigh", +) + +for obj in search_response.search_results.objects: + print(obj.properties) +# END EffortExample + # --- Async code examples in string as top-level await doesn't work, full code will be executed in # asyncio.run below diff --git a/docs/query-agent/_includes/code/structured_outputs.mts b/docs/query-agent/_includes/code/structured_outputs.mts new file mode 100644 index 000000000..d3537e6cb --- /dev/null +++ b/docs/query-agent/_includes/code/structured_outputs.mts @@ -0,0 +1,118 @@ +import 'dotenv/config' +const { loadClientInternally, populateWeaviate } = await import('./util.mjs').catch(() => import('../docs/query-agent/_includes/code/util.mjs')); + +const client = await loadClientInternally(); +await populateWeaviate(client, false); + + +// START SOInstantiate +import { QueryAgent } from 'weaviate-agents'; +import { z } from 'zod'; + +const qa = new QueryAgent(client, { collections: ['FinancialContracts'] }); +// END SOInstantiate + + +{ +// START SOBasicExampleBaseModel +const ContractSummary = z.object({ + contract_id: z.string(), + contract_title: z.string(), + auto_renew: z.boolean(), + parties_involved: z.array(z.string()), + requires_action: z.boolean(), +}); + +const res = await qa.ask( + "Find the oldest contract and include if it automatically renews, who is involved, and if user action is needed", + { outputFormat: ContractSummary } +); + +console.log(res.finalAnswerParsed); +// END SOBasicExampleBaseModel +} + + +{ +// START SOBasicDictExample +const res = await qa.ask( + "Find the oldest contract and include if it automatically renews, who is involved, and if user action is needed", + { + outputFormat: { + type: "object", + properties: { + contract_id: { title: "Contract Id", type: "string" }, + contract_title: { title: "Contract Title", type: "string" }, + auto_renew: { title: "Auto Renew", type: "boolean" }, + parties_involved: { items: { type: "string" }, title: "Parties Involved", type: "array" }, + requires_action: { title: "Requires Action", type: "boolean" }, + }, + required: ["contract_id", "contract_title", "auto_renew", "parties_involved", "requires_action"], + title: "ContractSummary", + additionalProperties: false, + }, + } +); + +console.log(res.finalAnswerParsed); +// END SOBasicDictExample +} + + +{ +// START SOReasoningExample +const FinalAnswer = z.object({ + reasoning: z.string(), + final_answer: z.string(), +}); + +const res = await qa.ask("What is the most recent contract about AI?", { outputFormat: FinalAnswer }); + +console.log(res.finalAnswerParsed); +// END SOReasoningExample +} + + +{ +// START SONestedExampleBaseModel +const ContractInfo = z.object({ + names_mentioned: z.array(z.string()).describe("All names within the contract text"), + contract_type: z.enum(["sales", "purchase", "other"]).describe("Determine the type of contract"), + summary: z.string().describe("Provide a brief summary of the contract."), + contract_uuid: z.uuid(), +}); + +const ContractInfoResponse = z.object({ + contract_infos: z.array(ContractInfo), + overall_summary: z.string(), +}); + +const res = await qa.ask("Find and return all contracts about AI in 2023", { outputFormat: ContractInfoResponse }); + +console.dir(res.finalAnswerParsed, { depth: null }); +// END SONestedExampleBaseModel +} + + +{ +// START SOCitationExample +const CitedText = z.object({ + sentence: z.string().describe("A single sentence from your answer, to be combined with other sentences"), + sources: z.array(z.uuid()).describe("The UUIDs of the sources that support the sentence"), +}); + +const CitedAnswer = z.object({ + reasoning: z.string(), + final_answer: z.array(CitedText).describe( + "A list of cited sentences, that will combine together in a paragraph to be a full answer" + ), +}); + +const res = await qa.ask("What is the most recent contract about AI?", { outputFormat: CitedAnswer }); + +console.dir(res.finalAnswerParsed, { depth: null }); +// END SOCitationExample +} + + +await client.close(); diff --git a/docs/query-agent/_includes/code/structured_outputs.py b/docs/query-agent/_includes/code/structured_outputs.py new file mode 100644 index 000000000..2288f99ff --- /dev/null +++ b/docs/query-agent/_includes/code/structured_outputs.py @@ -0,0 +1,105 @@ +import sys +sys.path.insert(0, "docs/query-agent/_includes/code") +from util import load_client_internally, populate_weaviate + +client = load_client_internally() +populate_weaviate(client, False) + + +# START SOInstantiate +from weaviate.agents.query import QueryAgent + +qa = QueryAgent(client=client, collections=["FinancialContracts"]) +# END SOInstantiate + + +# START SOBasicExampleBaseModel +from pydantic import BaseModel + +class ContractSummary(BaseModel): + contract_id: str + contract_title: str + auto_renew: bool + parties_involved: list[str] + requires_action: bool + +res = qa.ask( + "Find the oldest contract and include if it automatically renews, who is involved, and if user action is needed", + output_format=ContractSummary, +) + +print(res.final_answer_parsed) +# END SOBasicExampleBaseModel + +# START SOBasicDictExample +res = qa.ask( + "Find the oldest contract and include if it automatically renews, who is involved, and if user action is needed", + output_format={ + 'properties': { + 'contract_id': {'title': 'Contract Id', 'type': 'string'}, + 'contract_title': {'title': 'Contract Title', 'type': 'string'}, + 'auto_renew': {'title': 'Auto Renew', 'type': 'boolean'}, + 'parties_involved': {'items': {'type': 'string'}, 'title': 'Parties Involved', 'type': 'array'}, + 'requires_action': {'title': 'Requires Action', 'type': 'boolean'} + }, + 'required': ['contract_id', 'contract_title', 'auto_renew', 'parties_involved', 'requires_action'], + 'title': 'ContractSummary', + 'type': 'object' + } +) + +print(res.final_answer_parsed) +# END SOBasicDictExample + +# START SOReasoningExample +from pydantic import BaseModel + +class FinalAnswer(BaseModel): + reasoning: str + final_answer: str + +res = qa.ask("What is the most recent contract about AI?", output_format=FinalAnswer) + +print(res.final_answer_parsed) +# END SOReasoningExample + +# START SONestedExampleBaseModel +from pydantic import BaseModel, Field +from uuid import UUID +from typing import Literal + +class ContractInfo(BaseModel): + names_mentioned: list[str] = Field(description="All names within the contract text") + contract_type: Literal["sales", "purchase", "other"] = Field(description="Determine the type of contract") + summary: str = Field(description="Provide a brief summary of the contract.") + contract_uuid: UUID + +class ContractInfoResponse(BaseModel): + contract_infos: list[ContractInfo] + overall_summary: str + +res = qa.ask("Find and return all contracts about AI in 2023", output_format=ContractInfoResponse) + +print(res.final_answer_parsed) +# END SONestedExampleBaseModel + +# START SOCitationExample +from pydantic import BaseModel, Field +from uuid import UUID + +class CitedText(BaseModel): + sentence: str = Field(description="A single sentence from your answer, to be combined with other sentences") + sources: list[UUID] = Field(description="The UUIDs of the sources that support the sentence") + +class CitedAnswer(BaseModel): + reasoning: str + final_answer: list[CitedText] = Field( + description="A list of cited sentences, that will combine together in a paragraph to be a full answer" + ) + +res = qa.ask("What is the most recent contract about AI?", output_format=CitedAnswer) + +print(res.final_answer_parsed) +# END SOCitationExample + +client.close() diff --git a/docs/query-agent/_includes/code/suggest_queries.mts b/docs/query-agent/_includes/code/suggest_queries.mts index eaec99af4..29dfadd7a 100644 --- a/docs/query-agent/_includes/code/suggest_queries.mts +++ b/docs/query-agent/_includes/code/suggest_queries.mts @@ -41,4 +41,30 @@ for (const suggestedQuery of response.queries) { } // END AccessResponse +// START SuggestQueriesWithConversation +import { ChatMessage } from 'weaviate-agents'; + +// Build a conversation history +const suggestConversation: ChatMessage[] = [ + { + role: 'user', + content: 'What are some popular machine learning frameworks?', + }, + { + role: 'assistant', + content: 'Some popular ML frameworks include TensorFlow, PyTorch, and JAX.', + }, +]; + +// Suggest follow-up queries based on the conversation context +const suggestWithConvoResponse = await qa.suggestQueries({ + conversation: suggestConversation, + numQueries: 3, +}); + +for (const suggestedQuery of suggestWithConvoResponse.queries) { + console.log(suggestedQuery.query); +} +// END SuggestQueriesWithConversation + await client.close(); diff --git a/docs/query-agent/_includes/code/suggest_queries.py b/docs/query-agent/_includes/code/suggest_queries.py index 518a33035..f9dfa4f21 100644 --- a/docs/query-agent/_includes/code/suggest_queries.py +++ b/docs/query-agent/_includes/code/suggest_queries.py @@ -29,6 +29,28 @@ print(suggested_query.query) # END AccessResponse +# START SuggestQueriesWithConversation +from weaviate.agents.classes import ChatMessage + +# Build a conversation history +conversation = [ + ChatMessage(role="user", content="What are some popular machine learning frameworks?"), + ChatMessage( + role="assistant", + content="Some popular ML frameworks include TensorFlow, PyTorch, and JAX.", + ), +] + +# Suggest follow-up queries based on the conversation context +response = qa.suggest_queries( + conversation=conversation, + num_queries=3, +) + +for suggested_query in response.queries: + print(suggested_query.query) +# END SuggestQueriesWithConversation + """ # START AsyncInstantiation import os diff --git a/docs/query-agent/guides/ask_mode.md b/docs/query-agent/guides/ask_mode.md index 9913257af..85cfee2d3 100644 --- a/docs/query-agent/guides/ask_mode.md +++ b/docs/query-agent/guides/ask_mode.md @@ -66,7 +66,8 @@ The `.ask()` method accepts several arguments: | --- | --- | --- | | `query` | `str \| list[ChatMessage]` | The user query you want the agent to answer. This can be a simple string (`"What is the highest-grossing product?"`) or a list of chat messages (for conversational context). [See the page on multi-turn conversations for more detail](../reference/multi_turn_conversations.md). | | `collections` | `list[str \| QueryAgentCollectionConfig] \| None` | The name(s) of the collections to search. You can pass one or many collection names as a list of strings (e.g., `["ECommerce", "BookSales"]`), or provide collection configuration objects for more control. If specified in the `ask` method, it will overwrite those defined in the instantiation of `QueryAgent`. [See the page on collection configuration for more detail](../reference/advanced_collections.md). | -| `result_evaluation` | `Literal["llm", "none"]` | Controls whether the agent will ask an LLM to "evaluate" (i.e., rewrite or rephrase) the result based on all retrieved context. Accepts either:
• `"none"` (default): faster and cheaper; where the final answer is the last LLM call and no further analysis is completed.
• `"llm"`: higher cost/latency - enables a final step where an LLM subsets the sources retrieved to only those used in the answer, as well as enabling the optional fields `is_partial_answer` and `missing_information`. See [the response class](#response) for more details. | +| `result_evaluation` | `Literal["llm", "none"]` | Controls whether the agent will ask an LLM to "evaluate" the result based on all retrieved context. Accepts either:
• `"none"` (default): faster and cheaper; where the final answer is the last LLM call and no further analysis is completed.
• `"llm"`: higher cost/latency - enables a final step where an LLM subsets the sources retrieved to only those used in the answer, as well as enabling the optional fields `is_partial_answer` and `missing_information`. See [the response class](#response) for more details. | +| `output_format` | `dict \| type[BaseModel] \| None` | Optional schema for structured output in the final response. When set, `.ask()` returns a `ParsedAskModeResponse` instead of an `AskModeResponse`: the parsed result is added on a new `final_answer_parsed` field, and `final_answer` still holds the raw model output. See [the response class](#response) and [the page on structured outputs for more details](../reference/structured_outputs.md). |
@@ -74,8 +75,8 @@ The `.ask()` method accepts several arguments: | --- | --- | --- | | `query` | `string \| ChatMessage[]` | The user query you want the agent to answer. This can be a simple string (`"What is the highest-grossing product?"`) or a list of chat messages (for conversational context). [See the page on multi-turn conversations for more detail](../reference/multi_turn_conversations.md). | | `collections` | `(string \| QueryAgentCollectionConfig)[]` | The name(s) of the collections to search. You can pass one or many collection names as a list of strings (e.g., `["ECommerce", "BookSales"]`), or provide collection configuration objects for more control. [See the page on collection configuration for more detail](../reference/advanced_collections.md). If specified in the `ask` method, it will overwrite those defined in the instantiation of `QueryAgent`. | -| `resultEvaluation` | `"llm" \| "none"` | Controls whether the agent will ask an LLM to "evaluate" (i.e., rewrite or rephrase) the result based on all retrieved context. Accepts either:
• `"none"`: faster and cheaper; default setting where the final answer is the last LLM call.
• `"llm"`: higher cost/latency - enables a final step where an LLM subsets the sources retrieved to only those used in the answer, as well as enabling the optional fields `is_partial_answer` and `missing_information`. See [the response class](#response) for more details. | - +| `resultEvaluation` | `"llm" \| "none"` | Controls whether the agent will ask an LLM to "evaluate" the result based on all retrieved context. Accepts either:
• `"none"`: faster and cheaper; default setting where the final answer is the last LLM call.
• `"llm"`: higher cost/latency - enables a final step where an LLM subsets the sources retrieved to only those used in the answer, as well as enabling the optional fields `is_partial_answer` and `missing_information`. See [the response class](#response) for more details. | +| `outputFormat` | `ZodType \| object` | Optional schema for structured output in the final response. Pass a [Zod](https://zod.dev/) schema (parsed and validated) or a raw [Draft 2020-12 JSON Schema](https://json-schema.org/draft/2020-12) object (parsed only). When set, `.ask()` returns a `ParsedAskModeResponse` instead of an `AskModeResponse`: the typed result is added on a new `finalAnswerParsed` field, and `finalAnswer` still holds the raw model output. See [the response class](#response) and [the page on structured outputs for more details](../reference/structured_outputs.md). |
@@ -100,6 +101,14 @@ The `AskModeResponse` class has the following properties: | `sources` | `list[Source] \| None` | A list of `Source` objects, which have an `object_id` property correlating to the UUID of the Weaviate object that was retrieved during the run. If `result_evaluation` is `"llm"`, these are subset to only those that are relevant to the `final_answer`. | [See the client documentation for more detail.](https://weaviate-python-client.readthedocs.io/en/latest/weaviate-agents-python-client/docs/weaviate_agents.classes.html#weaviate_agents.classes.AskModeResponse) + +If you provide the `output_format` parameter (`qa.ask(..., output_format=...)`), Ask Mode returns a `ParsedAskModeResponse` instead. It is a subclass of `AskModeResponse`, so it keeps every field above and adds one more. `final_answer` still holds the raw string from the model. + +| Field | Type | Description | +| --- | --- | --- | +| `final_answer_parsed` | `` | The final response, parsed into the schema given in `output_format`. | + +The type of `final_answer_parsed` is a `dict` if a dictionary was supplied to `output_format`, otherwise it will be the exact type of the `BaseModel` given. @@ -116,6 +125,16 @@ The `AskModeResponse` class has the following properties: | `sources` | `Source[]` | A list of `Source` objects, which have an `objectId` property correlating to the UUID of the Weaviate object that was retrieved during the run. If `resultEvaluation` is `"llm"`, these are subset to only those that are relevant to the `finalAnswer`. | [See the client documentation for more detail.](https://weaviate.github.io/agents-typescript-client/types/AskModeResponse.html) + +If you provide the `outputFormat` parameter (`qa.ask(..., { outputFormat: ... })`), Ask Mode returns a `ParsedAskModeResponse` instead. It is an `AskModeResponse` with one more field, so it keeps every field above. `finalAnswer` still holds the raw string from the model. + +| Field | Type | Description | +| --- | --- | --- | +| `finalAnswerParsed` | `` | The final response, parsed into the schema given in `outputFormat`. | + +The type of `finalAnswerParsed` is `Record` if a raw JSON Schema object was supplied to `outputFormat`, otherwise it will be the inferred type of the Zod schema given (`z.infer`). + +[See the client documentation for more detail.](https://weaviate.github.io/agents-typescript-client/types/ParsedAskModeResponse.html) diff --git a/docs/query-agent/guides/search_mode.md b/docs/query-agent/guides/search_mode.md index ff4d6a24c..94bb5edd7 100644 --- a/docs/query-agent/guides/search_mode.md +++ b/docs/query-agent/guides/search_mode.md @@ -13,7 +13,7 @@ import TSCode from '!!raw-loader!/docs/query-agent/_includes/code/search_mode.mt -Search Mode transforms your query into actionable searches and returns the matching Weaviate objects directly — without generating an LLM-authored answer. +Search Mode combines AI-powered semantic search with structured filtering and returns the matching Weaviate objects directly. For example, you could ask: @@ -21,6 +21,14 @@ For example, you could ask: And the agent will perform semantic search for `vintage shoes`, apply a filter for `price < 70`, and return the matching objects from your collections, ready for you to render or post-process. +You could also ask: + +> "Something comfortable to wear on a long flight" + +And the agent will use AI-powered search to find relevant objects, even when terms like `comfortable` or `long flight` never appear in your data. + +Under the hood, Search Mode does more than embed your query as-is. The agent writes one or more optimized semantic and structured queries, executes them against your collections, and reranks the retrieved objects by how well each one matches your original request. + For more details, see the page for [the Python client](https://weaviate-python-client.readthedocs.io/en/stable/weaviate-agents-python-client/docs/weaviate_agents.query.html#weaviate_agents.query.QueryAgent.search) or [the Typescript Client](https://weaviate.github.io/agents-typescript-client/classes/QueryAgent.html#search). ## Usage @@ -69,6 +77,7 @@ The `.search()` method accepts several arguments: | `limit` | `int` | The maximum number of results returned in this page of results. Defaults to `20`. Use [`.next()`](#pagination) to fetch additional pages. | | `filtering` | `Literal["recall", "precision"]` | Either `"recall"` or `"precision"` to control filter generation. `"recall"` favors more results across filter interpretations; `"precision"` favors strict intent match. See [Customized filtering](#customized-filtering) below. | | `diversity_weight` | `float \| None` | A value between `0.0` and `1.0` that biases the result ranking towards diversity using Maximal Marginal Relevance (MMR). See [Diversity ranking](#diversity-ranking) below. | +| `effort` | `Literal["medium", "high", "ultrahigh"] \| None` | The amount of effort the agent puts into the search. Higher effort may improve result quality at the expense of increased latency and cost. See [Effort](#effort) below. | @@ -79,12 +88,36 @@ The `.search()` method accepts several arguments: | `limit` | `number` | The maximum number of results returned in this page of results. Defaults to `20`. Use [`.next()`](#pagination) to fetch additional pages. | | `filtering` | `"recall" \| "precision"` | Either `"recall"` or `"precision"` to control filter generation. `"recall"` favors more results across filter interpretations; `"precision"` favors strict intent match. See [Customized filtering](#customized-filtering) below. | | `diversityWeight` | `number` | A value between `0.0` and `1.0` that biases the result ranking towards diversity using Maximal Marginal Relevance (MMR). See [Diversity ranking](#diversity-ranking) below. | +| `effort` | `"medium" \| "high" \| "ultrahigh"` | The amount of effort the agent puts into the search. Higher effort may improve result quality at the expense of increased latency and cost. See [Effort](#effort) below. | For more advanced searches, you can also specify _additional filters_ within the collection configuration. [See the page on additional filters for more detail](../reference/additional_filters.md). +### Effort + +The optional `effort` parameter controls the amount of effort the agent puts into the search. It accepts one of `"medium"`, `"high"`, or `"ultrahigh"`. Higher effort may improve result quality at the expense of increased latency and cost. + + + + + + + + + + ### Customized filtering Search Mode uses query rewriting to transform your original query into one or multiple Weaviate queries, each with either a search query, metadata filters, or both. The `filtering` parameter controls how many Weaviate queries are generated. diff --git a/docs/query-agent/guides/suggest_queries.md b/docs/query-agent/guides/suggest_queries.md index 72d26e664..ecdba6a3a 100644 --- a/docs/query-agent/guides/suggest_queries.md +++ b/docs/query-agent/guides/suggest_queries.md @@ -70,6 +70,7 @@ Suggest Queries can be called with the following arguments: | `collections` | `list[str \| QueryAgentCollectionConfig] \| None` | Override the collections configured at instantiation. [See the page on collection configuration for more detail](../reference/advanced_collections.md). | | `num_queries` | `int` | The number of queries to suggest (default: `3`). | | `instructions` | `str \| None` | Guide the style or focus of the suggested queries. This is provided in addition to any system instructions. Useful for e.g. specifying language. | +| `conversation` | `list[ChatMessage] \| None` | A conversation history used to generate follow-up query suggestions. | @@ -78,6 +79,32 @@ Suggest Queries can be called with the following arguments: | `collections` | `(string \| QueryAgentCollectionConfig)[]` | Override the collections configured at instantiation. [See the page on collection configuration for more detail](../reference/advanced_collections.md). | | `numQueries` | `number` | The number of queries to suggest (default: `3`). | | `instructions` | `string` | Guide the style or focus of the suggested queries. This is provided in addition to any system instructions. Useful for e.g. specifying language. | +| `conversation` | `ChatMessage[]` | A conversation history used to generate follow-up query suggestions. | + + + +### Follow-up queries + +You can pass a `conversation` to Suggest Queries to generate follow-up query suggestions based on the conversation history. This is useful for guiding users toward relevant next questions after an initial exchange. + +The `conversation` parameter accepts a list of `ChatMessage` objects, using the same format as [multi-turn conversations](../reference/multi_turn_conversations.md). + + + + + + + diff --git a/docs/query-agent/reference/index.md b/docs/query-agent/reference/index.md index f121d088d..0bbff9b29 100644 --- a/docs/query-agent/reference/index.md +++ b/docs/query-agent/reference/index.md @@ -12,7 +12,7 @@ See the different configuration options for the Query Agent and how you can cust * **[Multi-turn Conversations](./multi_turn_conversations.md)**: Learn how to include multiple turns of conversations in a message history instead of a single user query. * **[Additional Filters](./additional_filters.md)**: Define persistent filters that get added to every search the Query Agent performs. * **[Collection Configuration](./advanced_collections.md)**: Setup your collections with more advanced configurations, such as named vectors, multi-tenancy and additional filters. - +* **[Structured Outputs](./structured_outputs.md)**: Configure the format of the Ask Mode response to conform to a schema. ## Questions and feedback diff --git a/docs/query-agent/reference/structured_outputs.md b/docs/query-agent/reference/structured_outputs.md new file mode 100644 index 000000000..a849cdaa0 --- /dev/null +++ b/docs/query-agent/reference/structured_outputs.md @@ -0,0 +1,435 @@ +--- +title: Structured outputs +description: "Conform the final response to a particular schema." +image: og/docs/query-agent.png +# tags: ['agents', 'query-agent', 'configuration'] +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; +import PyCode from '!!raw-loader!/docs/query-agent/_includes/code/structured_outputs.py'; +import TSCode from '!!raw-loader!/docs/query-agent/_includes/code/structured_outputs.mts'; + + +Structured outputs ensure that the Query Agent's final response adheres to a schema that you provide. Instead of parsing a free-text answer, you get back an object whose fields are customized to your use case. + +Structured outputs are supported in **[Ask Mode](../guides/ask_mode.md) only.** + +:::info Client versions +Structured outputs require `weaviate-agents` **1.7.0 or later** in Python, and **1.6.0 or later** in JavaScript/TypeScript. The JavaScript/TypeScript examples that use a Zod schema also require **Zod 4 or later**. [See the installation page](../installation.md). +::: + +The example outputs on this page will not match yours exactly. The Query Agent is non-deterministic, so the wording and the records it picks vary between runs. Your schema controls the shape of the response, not the content. + +## Basic usage + +Set the schema per request, with the `output_format` argument of `.ask()` (`outputFormat` in JavaScript/TypeScript). The examples on this page use a Query Agent instantiated over a `FinancialContracts` collection ([see the class instantiation page for more detail](./instantiation.md)): + + + + + + + + + + +The examples below use structured outputs to generate a set of metadata associated with a single retrieved item. + + + + In Python, you can either provide a Pydantic `BaseModel` or a raw dictionary conforming to the [Draft 2020-12 JSON Schema](https://json-schema.org/draft/2020-12) specification. + + The structured output is available on a new field, `final_answer_parsed`, which appears when you provide `output_format`. The raw string from the model can still be accessed at `final_answer`. + + **Pydantic BaseModel** + + +
+ Example output + ```python + ContractSummary( + contract_id='46.0', + contract_title='Employment Contract', + auto_renew=False, + parties_involved=['Weaviate (Employer)', 'Mark Robson (Employee)', 'Hans Zimmer (Chief Executive Officer/signatory)'], + requires_action=True + ) + ``` +
+ + + **Dictionary** + + +
+ Example output + ```python + { + 'contract_id': '46.0', + 'contract_title': 'Employment Contract', + 'auto_renew': False, + 'parties_involved': ['Weaviate (Employer)', 'Mark Robson (Employee)', 'Hans Zimmer, Chief Executive Officer'], + 'requires_action': True + } + ``` +
+ +
+ + In JavaScript/TypeScript, you can either provide a [Zod](https://zod.dev/) schema or a raw object conforming to the [Draft 2020-12 JSON Schema](https://json-schema.org/draft/2020-12) specification. + + The structured output is available on a new field, `finalAnswerParsed`, which appears when you provide `outputFormat`. The raw string from the model can still be accessed at `finalAnswer`. + + **Zod schema** + +
+ Example output + ```typescript + { + contract_id: '46.0', + contract_title: 'Employment Contract', + auto_renew: false, + parties_involved: [ 'Weaviate (Employer)', 'Mark Robson (Employee)' ], + requires_action: true + } + ``` +
+ + **Object** + +
+ Example output + ```typescript + { + contract_id: '46', + contract_title: 'Employment Contract', + auto_renew: false, + parties_involved: [ + 'Weaviate (Employer)', + 'Mark Robson (Employee)', + 'Hans Zimmer (Chief Executive Officer and signatory)' + ], + requires_action: true + } + ``` +
+ + Zod adds `additionalProperties: false` when it converts a schema to JSON Schema, so both examples above describe the same thing. The field is optional. The Query Agent accepts a schema with or without it. + +
+
+ +## Example: Reasoning + +As a basic example, consider adding an additional field `reasoning` to the response. Order is preserved in the specification, so if this is provided _before_ the answer field, the model will produce a reasoning string before writing its answer, which can provide explainability to a response. + + + + +
+ Example output + ```python + FinalAnswer( + reasoning='Among the provided contracts, the latest one explicitly concerning AI is dated November 15, 2023. It is a partnership agreement between Weaviate and OpenAI for collaboration on artificial intelligence research and development. The March 15, 2024 contracts are sales and lease agreements and do not concern AI.', + final_answer='The most recent AI-related contract is a **Partnership Agreement dated November 15, 2023**, between **Weaviate and OpenAI**. It establishes a three-year partnership to collaborate on **artificial intelligence research and development**. Weaviate is responsible for marketing and promotion, while OpenAI provides technical expertise and development support; profits are to be split equally.\n\n- **Contract type:** Partnership agreement\n- **Date:** November 15, 2023\n- **Author/signatory:** Johnathan Smith, CEO of Weaviate\n- **Document ID:** 60' + ) + ``` +
+
+ + +
+ Example output + ```typescript + { + reasoning: 'The latest dated contract explicitly concerning AI is dated May 15, 2023. Later contracts in the provided data are dated November 15, 2023 and March 15, 2024, but do not concern AI. There are multiple duplicate records for the May 15 contract; the matching record identifies doc_id 56.0 and author Alice Johnson.', + final_answer: 'The most recent AI-related contract is a **Partnership Agreement** dated **May 15, 2023**.\n\n- **Author:** Alice Johnson\n- **Doc ID:** 56.0\n- **Summary:** Weaviate and OpenAI agree to collaborate on developing AI-driven solutions to improve data management and retrieval. Weaviate contributes $391.74, OpenAI contributes $302.40, profits are split 60/40, and the agreement lasts three years.\n\nNo later contract in the provided records explicitly concerns AI.' + } + ``` +
+
+
+ +## Example: Nested schemas + +Nested schemas are supported, for example, you can define two schemas and have one reference the other, allowing more complex structured outputs to be crafted. + +In the below example, the final response will generate a list of information for each object that was retrieved, either extracted or generated from the content of the data, as well as providing an overall answer. + + + + + A `Field` can be used to provide additional metadata, such as a `description`, or even constraints on numeric objects. A `Literal` can be used to constrain a field to produce only one of a few different objects. +
+ Example output + ```python + ContractInfoResponse( + contract_infos=[ + ContractInfo( + names_mentioned=['Weaviate', 'OpenAI', 'Mark Robson', 'Kaladin Stormblessed'], + contract_type='other', + summary='Partnership Agreement dated March 15, 2023, establishing collaboration between Weaviate and OpenAI on artificial intelligence research and development. Weaviate contributes technology resources valued at $112.85 and staff time valued at $550.09; OpenAI contributes research expertise and project-management support valued at $98.14. Net profits are split 60% to Weaviate and 40% to OpenAI.', + contract_uuid=UUID('8ec8f74a-2d38-4aca-80ca-e66f12ae0cb6') + ), + ContractInfo( + names_mentioned=['Weaviate', 'OpenAI', 'Alice Johnson', 'Mark Robson'], + contract_type='other', + summary='Partnership Agreement dated March 15, 2023, for collaboration on artificial intelligence projects. Weaviate contributes $210.97 toward initial project costs, while OpenAI contributes $194.05 toward research and development. OpenAI is responsible for technical development and AI research expertise.', + contract_uuid=UUID('056b6b5c-d6d3-4235-9003-4822dbbef9ef') + ), + ContractInfo( + names_mentioned=['Weaviate', 'OpenAI', 'Arthur Penndragon', 'Mark Robson', 'Danny Williams'], + contract_type='other', + summary='Partnership Agreement dated March 15, 2023, for artificial intelligence research and development, with shared resources and expertise. Weaviate contributes technology resources and staff time; OpenAI contributes research expertise and project-management support. Profits are divided 60% to Weaviate and 40% to OpenAI.', + contract_uuid=UUID('c50c3b9b-339a-4830-b2f7-4b0b9b84bc56') + ), + ContractInfo( + names_mentioned=['Weaviate', 'OpenAI', 'Edward Elric', 'Mark Robson'], + contract_type='other', + summary='Partnership Agreement dated March 15, 2023, to develop advanced data-processing technologies. Weaviate provides technological support and resources valued at $416.56; OpenAI contributes AI and machine-learning expertise valued at $567.91. Revenue is split 60% to Weaviate and 40% to OpenAI.', + contract_uuid=UUID('3cec1521-3d26-46b3-b63f-1af2ddaa3db4') + ), + ContractInfo( + names_mentioned=['Weaviate', 'OpenAI'], + contract_type='other', + summary='Partnership Agreement dated March 15, 2023, focused on collaborative projects in AI technology development. Weaviate contributes $234.12 toward project funding, and OpenAI contributes $173.25 for marketing and promotion. Weaviate handles technical development, while OpenAI conducts research and data analysis.', + contract_uuid=UUID('3d2afbc4-24c1-4f64-b676-193e4dccb451') + ), + ContractInfo( + names_mentioned=['Weaviate', 'OpenAI', 'Johnathan Smith', 'Mark Robson'], + contract_type='other', + summary='Partnership Agreement dated March 15, 2023, to advance artificial intelligence technologies and develop innovative AI solutions. Weaviate contributes $177.98 and OpenAI contributes $67.09; each party is responsible for roles specified in an attached exhibit.', + contract_uuid=UUID('8ac31eff-9936-4a1b-a5a4-8e148a4596bf') + ), + ContractInfo( + names_mentioned=['Weaviate', 'OpenAI', 'Alice Johnson', 'Mark Robson'], + contract_type='other', + summary='Partnership Agreement dated October 15, 2023, for development of innovative AI solutions. Weaviate contributes $726.88 and project resources, while OpenAI contributes $251.09 and technical expertise. Profits are shared 60% to Weaviate and 40% to OpenAI.', + contract_uuid=UUID('aa3c40bc-8bea-42d3-9692-75a417a99a0d') + ), + ContractInfo( + names_mentioned=['Weaviate', 'OpenAI', 'Johnathan Smith', 'Mark Robson'], + contract_type='other', + summary='Partnership Agreement dated November 15, 2023, covering projects including artificial intelligence research and development. Weaviate contributes $244.46 and handles marketing; OpenAI contributes $151.01 and provides technical expertise. Profits are shared equally.', + contract_uuid=UUID('1e85f2b6-f2f7-4e52-86ba-c4f508f6c06d') + ), + ContractInfo( + names_mentioned=['Weaviate', 'OpenAI', 'Alice Johnson', 'Danny Williams'], + contract_type='other', + summary='Service Agreement dated March 15, 2023, for AI development and consulting services. The total fee is $249.44, payable in two installments of $124.72.', + contract_uuid=UUID('1363c2ae-83e6-4049-8f95-cd6c873a997e') + ) + ], + overall_summary='Nine distinct contracts concerning AI, artificial intelligence research and development, AI technology development, AI solutions, or AI development and consulting were identified in 2023. Several duplicate records in the provided data were consolidated by document identity.' + ) + ``` +
+
+ + + A `.describe()` call can be used to provide additional metadata, such as a description, or even constraints on numeric objects. A `z.enum()` can be used to constrain a field to produce only one of a few different objects. +
+ Example output + ```typescript + { + contract_infos: [ + { + names_mentioned: [ 'Weaviate', 'OpenAI', 'Alice Johnson', 'Mark Robson' ], + contract_type: 'other', + summary: 'Partnership agreement dated March 15, 2023, between Weaviate and OpenAI to collaborate on artificial intelligence projects. Weaviate contributes $210.97 and OpenAI contributes $194.05; OpenAI is responsible for technical development and AI research expertise.', + contract_uuid: '056b6b5c-d6d3-4235-9003-4822dbbef9ef' + }, + { + names_mentioned: [ 'Weaviate', 'Danny Williams', 'Alice Johnson' ], + contract_type: 'other', + summary: 'Service agreement dated March 15, 2023, for AI development and consulting services. The total compensation is $249.44, payable in two installments, with a two-year term ending March 15, 2025.', + contract_uuid: '1363c2ae-83e6-4049-8f95-cd6c873a997e' + }, + { + names_mentioned: [ 'Weaviate', 'Danny Williams', 'Kaladin Stormblessed' ], + contract_type: 'other', + summary: 'Service agreement dated March 15, 2023, for consulting on artificial intelligence and data management systems, including implementation, training, and technical support. The total fee is $428.14, with a two-year term ending March 15, 2025.', + contract_uuid: 'dbc30467-5f6a-4630-b7ea-7a17ade8b70a' + }, + { + names_mentioned: [ 'Weaviate', 'Danny Williams', 'Kaladin Stormblessed' ], + contract_type: 'other', + summary: 'Duplicate record of the service agreement for artificial intelligence and data management systems consulting, implementation, training, and technical support. Total fee: $428.14; term ends March 15, 2025.', + contract_uuid: '010aeced-3328-415c-b93d-c19dc4baeefa' + }, + { + names_mentioned: [ 'Weaviate', 'OpenAI', 'Edward Elric', 'Mark Robson' ], + contract_type: 'other', + summary: 'Partnership agreement dated March 15, 2023, for developing advanced data processing technologies. OpenAI contributes AI and machine-learning expertise valued at $567.91, while Weaviate provides technological support and resources valued at $416.56. Revenue is split 60% to Weaviate and 40% to OpenAI.', + contract_uuid: '3cec1521-3d26-46b3-b63f-1af2ddaa3db4' + }, + { + names_mentioned: [ 'Weaviate', 'OpenAI', 'Johnathan Smith', 'Mark Robson' ], + contract_type: 'other', + summary: 'Partnership agreement dated March 15, 2023, to collaborate on advancements in artificial intelligence technologies and develop innovative AI solutions. Weaviate contributes $177.98 and OpenAI contributes $67.09.', + contract_uuid: '8ac31eff-9936-4a1b-a5a4-8e148a4596bf' + }, + { + names_mentioned: [ 'Weaviate', 'OpenAI', 'Johnathan Smith', 'Mark Robson' ], + contract_type: 'other', + summary: 'Duplicate record of the partnership agreement to collaborate on artificial intelligence technologies and develop AI solutions. Weaviate contributes $177.98 and OpenAI contributes $67.09.', + contract_uuid: '6fcb6899-6344-41e0-8963-10b49bd17625' + }, + { + names_mentioned: [ 'Weaviate', 'OpenAI', 'Alice Johnson', 'Mark Robson' ], + contract_type: 'other', + summary: 'Partnership agreement dated May 15, 2023, for developing AI-driven solutions to improve data management and retrieval. Weaviate contributes $391.74 and OpenAI contributes $302.40; profits are split 60% and 40%, respectively.', + contract_uuid: 'e32c13b7-b552-45f6-b58b-3f0c325715ab' + } + ], + overall_summary: 'Eight unique 2023 contracts concern AI or artificial intelligence-related services and projects. Several duplicate records were omitted from the main list. The matching documents include five partnership agreements and three AI-related service-agreement records; no invoice was explicitly identified as AI-related based on its text.' + } + ``` +
+
+
+ +## Example: Citations + +For a custom implementation of citing text (for example, if you want citations in-line), you could create a schema that iteratively builds a response from objects consisting of pairs of text and source IDs. + +:::note Supported Citations +The Query Agent natively supports subsetting and evaluating the quality of the response via the `result_evaluation` parameter in Ask Mode. [See the Ask Mode parameters for more details](../guides/ask_mode.md#parameters). +::: + + + + +
+ Example output + ```python + CitedAnswer( + reasoning='The latest dated contract in the provided records that explicitly concerns AI is dated March 15, 2024, but it is a sales agreement for unspecified products and does not mention AI. The latest contract that explicitly concerns artificial intelligence is the partnership agreement dated November 15, 2023 (doc_id 60.0), which covers collaboration on AI research and development.', + final_answer=[ + CitedText( + sentence='The most recent contract explicitly about AI is a Partnership Agreement dated November 15, 2023, between Weaviate and OpenAI (doc_id 60.0), authored by Johnathan Smith.', + sources=[UUID('a06ab40a-bcc3-4fc2-bb8a-b2b2598f706c')] + ), + CitedText( + sentence='It establishes a three-year collaboration on projects including artificial-intelligence research and development, with Weaviate contributing $244.46, OpenAI contributing $151.01, shared marketing and technical responsibilities, and profits split equally.', + sources=[UUID('a06ab40a-bcc3-4fc2-bb8a-b2b2598f706c')] + ) + ] + ) + ``` +
+
+ + +
+ Example output + ```typescript + { + reasoning: 'The latest dated contract in the provided records that explicitly concerns AI is dated May 15, 2023. It is a partnership agreement for developing AI-driven solutions; the later March 15, 2024 contracts do not mention AI.', + final_answer: [ + { + sentence: 'The most recent AI-related contract is a Partnership Agreement dated May 15, 2023, authored by Alice Johnson, with document ID 56.0.', + sources: [ 'e32c13b7-b552-45f6-b58b-3f0c325715ab' ] + }, + { + sentence: 'It establishes a partnership between Weaviate and OpenAI to develop AI-driven solutions for improving data management and retrieval, with Weaviate contributing $391.74, OpenAI contributing $302.40, and profits split 60% to Weaviate and 40% to OpenAI over a three-year term.', + sources: [ 'e32c13b7-b552-45f6-b58b-3f0c325715ab' ] + } + ] + } + ``` +
+
+
+ +## What is supported? + + +| Feature | Supported? | Notes | +|---------|:----------:|-------| +| Min / max number of items in an array | ✅ | | +| Min / max value of a number property | ✅ | E.g. constrain a value to be within a certain range. | +| String formats: `uuid`, `date-time`, `time`, `date`, `duration`, `email`, `hostname`, `ipv4`, `ipv6` | ✅ | Guides the model to produce a string in that format. The result is not checked afterwards, so a field your data cannot fill may still come back invalid. | +| Regular expression (pattern) on a string | ✅ | | +| Recursive schemas (a schema referencing itself) | ✅ | | +| Default values (e.g. `x: int = 1`) | ❌ | The schema is accepted, but the field is always populated by the model, so the default is never used. Consider using nullable entries and transforming them afterwards. | +| Schemas with 5000+ properties | ❌ | Rejected with a `SCHEMA_VALIDATION_ERROR` before the agent runs. | +| 1000 or more enum values across all properties | ❌ | Rejected with a `SCHEMA_VALIDATION_ERROR` before the agent runs. | +| More than 10 levels of nesting in a single property | ❌ | Rejected with a `SCHEMA_VALIDATION_ERROR` before the agent runs. The error names the offending field path and its depth. | + + +## Streaming + +Structured outputs are supported with [streaming in Ask Mode](../guides/ask_mode.md#streaming). + +When streaming, the structured output is delivered incrementally as raw string fragments through `StreamedTokens` instances. No special parsing is applied during the stream — each token is a fragment of the final output. To use the partial result, accumulate the streamed tokens into a single string, then partially validate the string against your schema. + +To use the final result after completion, you do not need to use the streamed tokens. Read the `final_answer_parsed` attribute (`finalAnswerParsed` in TypeScript) of the [final state output](../guides/ask_mode.md#responses). + +## Questions and feedback + +import DocsFeedback from '/\_includes/docs-feedback.mdx'; + + diff --git a/docs/weaviate/api/graphql/filters.md b/docs/weaviate/api/graphql/filters.md index e4a150a43..cc274b75b 100644 --- a/docs/weaviate/api/graphql/filters.md +++ b/docs/weaviate/api/graphql/filters.md @@ -243,7 +243,7 @@ The `ContainsAny`, `ContainsAll` and `ContainsNone` operators filter objects usi These operators expect an array of values and return objects that match based on the input values. -They are not limited to text. They work on `text`/`string`, `int`, `number`, `boolean`, `date`, and `uuid` properties — and on the array variants of each (`int[]`, `number[]`, etc.). A scalar property is treated as a single-element set: the object matches if its value satisfies the operator against the candidate list. +They are not limited to text. They work on `text`/`string`, `int`, `number`, `boolean`, `date`, and `uuid` properties, and on the array variants of each (`int[]`, `number[]`, etc.). A scalar property is treated as a single-element set: the object matches if its value satisfies the operator against the candidate list. Pass the candidate values using the argument that matches the property's data type: `valueText` for `text`/`string`, `valueInt` for `int`, `valueNumber` for `number`, `valueBoolean` for `boolean`, and `valueDate` for `date`. For `uuid`/`uuid[]` properties, pass the UUIDs as strings using `valueText` (there is no `valueUuid` argument). For example, a `ContainsAny` query on an `int` property with a value of `[10, 20, 30]` returns objects whose property holds at least one of those integers. @@ -588,8 +588,11 @@ Using the `IsNull` operator allows you to do filter for objects where given prop Get { (where: { operator: IsNull, - valueBoolean: + valueBoolean: , path: [] + }) { + + } } } ``` diff --git a/docs/weaviate/api/graphql/get.md b/docs/weaviate/api/graphql/get.md index d00de5559..0616c021a 100644 --- a/docs/weaviate/api/graphql/get.md +++ b/docs/weaviate/api/graphql/get.md @@ -229,8 +229,8 @@ The following search operators are available. | --- | --- | --- | --- | | `nearObject` | Vector search using a Weaviate object | *none* | [Learn more](./search-operators.md#nearobject) | | `nearVector` | Vector search using a raw vector | *none* | [Learn more](./search-operators.md#nearvector) | -| `nearText` | Vector search using a text query | Text embedding model | | -| `nearImage` | Vector search using an image | Multi-modal embedding model | +| `nearText` | Vector search using a text query | Text embedding model | [Learn more](./search-operators.md#neartext) | +| `nearImage` | Vector search using an image | Multi-modal embedding model | [Learn more](./search-operators.md#multimodal-search) | | `hybrid` | Combine vector and BM25 search results | *none* | [Learn more](../graphql/search-operators.md#hybrid) | | `bm25` | Keyword search with BM25F ranking | *none* | [Learn more](../graphql/search-operators.md#bm25) | diff --git a/docs/weaviate/api/graphql/search-operators.md b/docs/weaviate/api/graphql/search-operators.md index 17374ee78..b2474aad4 100644 --- a/docs/weaviate/api/graphql/search-operators.md +++ b/docs/weaviate/api/graphql/search-operators.md @@ -268,13 +268,13 @@ This operator allows you to combine [BM25](#bm25) and vector search to get a "be | `bm25SearchOperator` | no | `object` | set how many of the (bm25) query tokens must be present within a single searched property for an object to be considered a match. (available from `v1.31.0`) | * Notes: - * `alpha` can be any number from 0 to 1, defaulting to 0.75. + * `alpha` can be any number from 0 to 1, defaulting to 0.75. See [Alpha parameter](/weaviate/concepts/search/hybrid-search.md#alpha-parameter). * `alpha` = 0 forces using a pure **keyword** search method (BM25) * `alpha` = 1 forces using a pure **vector** search method * `alpha` = 0.5 weighs the BM25 and vector methods evenly * `fusionType` can be `rankedFusion` or `relativeScoreFusion` - * `rankedFusion` (default) adds inverted ranks of the BM25 and vector search methods - * `relativeScoreFusion` adds normalized scores of the BM25 and vector search methods + * `relativeScoreFusion` (default from `v1.24`) adds normalized scores of the BM25 and vector search methods + * `rankedFusion` (default for `v1.23` and lower) adds inverted ranks of the BM25 and vector search methods ### Fusion algorithms @@ -435,9 +435,15 @@ To mitigate this effect, Weaviate automatically performs a search with a higher -Use `bm25SearchOperator` to set how many of the query tokens must be present within a single searched property for an object to be considered a match in the keyword (bm25) search portion of the hybrid search. This is useful when you want to ensure that only objects with a certain number of relevant keywords are returned. +Use `bm25SearchOperator` to set how the query tokens must match for an object to be considered a match in the keyword (bm25) search portion of the hybrid search. This is useful when you want to ensure that only objects with a certain number of relevant keywords are returned. -The available options are `And`, or `Or`. With `And`, all of the query tokens must appear together within a single searched property; tokens spread across different properties do not match. If `Or` is set, an additional parameter `minimumOrTokensMatch` must be specified, which defines how many of the query tokens must be present within a single searched property for the object to be considered a match. +The available options are `And`, `Or`, and `AndCross`: + +| Option | Description | +| ------ | ----------- | +| `And` | All of the query tokens must appear together within a single searched property. Tokens spread across different properties do not match. | +| `Or` | An additional parameter `minimumOrTokensMatch` must be specified, which defines how many of the query tokens must be present within a single searched property for the object to be considered a match. | +| `AndCross` | Every query token must be matched by at least one of the searched properties, so the tokens can be spread across different properties. All searched properties must share the same tokenization and analyzer settings, otherwise the query fails with an error. (available from `v1.38.8`) | If not set, the keyword search behaves as if `Or` was set with a `minimumOrTokensMatch` of `1`. @@ -458,7 +464,7 @@ The `bm25` operator supports the following variables: | --------- | -------- | ----------- | | `query` | yes | The keyword search query. | | `properties` | no | Array of properties (fields) to search in, defaulting to all properties in the collection. | -| `searchOperator` | no | set how many of the query tokens must be present within a single searched property for an object to be considered a match. (available from `v1.31.0`) | +| `searchOperator` | no | Set how the query tokens must match for an object to be considered a match. See [Search operator](#search-operator) for the available options. (available from `v1.31.0`) | :::info Boosting properties Specific properties can be boosted by a factor specified as a number after the caret sign, for example `properties: ["title^3", "summary"]`. @@ -537,12 +543,39 @@ import GraphQLFiltersBM25FilterExample from '/_includes/code/graphql.filters.bm2 -Use `searchOperator` to set how many of the query tokens must be present within a single searched property for an object to be considered a match. This is useful when you want to ensure that only objects with a certain number of relevant keywords are returned. +Use `searchOperator` to set how the query tokens must match for an object to be considered a match. This is useful when you want to ensure that only objects with a certain number of relevant keywords are returned. + +The available options are `And`, `Or`, and `AndCross`: -The available options are `And`, or `Or`. With `And`, all of the query tokens must appear together within a single searched property; tokens spread across different properties do not match. If `Or` is set, an additional parameter `minimumOrTokensMatch` must be specified, which defines how many of the query tokens must be present within a single searched property for the object to be considered a match. +| Option | Description | +| ------ | ----------- | +| `And` | All of the query tokens must appear together within a single searched property. Tokens spread across different properties do not match. | +| `Or` | An additional parameter `minimumOrTokensMatch` must be specified, which defines how many of the query tokens must be present within a single searched property for the object to be considered a match. | +| `AndCross` | Every query token must be matched by at least one of the searched properties, so the tokens can be spread across different properties. All searched properties must share the same tokenization and analyzer settings, otherwise the query fails with an error. (available from `v1.38.8`) | If not set, the keyword search behaves as if `Or` was set with a `minimumOrTokensMatch` of `1`. +An `AndCross` query example: + +```graphql +{ + Get { + JeopardyQuestion( + limit: 3 + bm25: { + query: "African desert wind" + searchOperator: { + operator: AndCross + } + } + ) { + question + answer + } + } +} +``` + ## ask Enabled by the module: [Question Answering](/weaviate/modules/qna-transformers.md). diff --git a/docs/weaviate/api/grpc.md b/docs/weaviate/api/grpc.md index ee400aec6..ccba6f0ea 100644 --- a/docs/weaviate/api/grpc.md +++ b/docs/weaviate/api/grpc.md @@ -7,7 +7,7 @@ image: og/docs/api.jpg Starting with Weaviate `v1.19.0`, a gRPC interface has been progressively added to Weaviate. gRPC is a high-performance, open-source universal RPC framework that is contract-based and can be used in any environment. It is based on HTTP/2 and Protocol Buffers, and is therefore very fast and efficient. -As of Weaviate `v1.23.7`, the gRPC interface is considered stable. The [Python (`v4` version)](../client-libraries/python/index.mdx) and [TypeScript (`v3` version)](../client-libraries/typescript/index.mdx) client libraries support gRPC, and the other client libraries will follow. +As of Weaviate `v1.23.7`, the gRPC interface is considered stable. The [Python](../client-libraries/python/index.mdx), [TypeScript](../client-libraries/typescript/index.mdx), [Java](../client-libraries/java/index.mdx), and [C#](../client-libraries/csharp.mdx) client libraries use gRPC. The [Go](../client-libraries/go.md) client uses gRPC for batch imports, and offers gRPC search through its experimental API. ## Protocol Buffer (Protobuf) definitions @@ -33,8 +33,6 @@ We suggest using the default port `50051` for gRPC calls. It can be modified thr Note that [Weaviate Cloud](/go/console?utm_content=api) uses port `443` for gRPC. ::: -````yaml: - ```yaml --- services: @@ -44,11 +42,11 @@ services: - "8080:8080" # REST calls - "50051:50051" # gRPC calls # ... Other settings -```` +``` ### Client-side -You can use the gRPC interface through the [Python (`v4` version)](../client-libraries/python/index.mdx) and [TypeScript (`v3` version)](../client-libraries/typescript/index.mdx) client libraries. Other client libraries will also introduce gRPC support in the near future. +You can use the gRPC interface through the [Python](../client-libraries/python/index.mdx), [TypeScript](../client-libraries/typescript/index.mdx), [Java](../client-libraries/java/index.mdx), and [C#](../client-libraries/csharp.mdx) client libraries. The [Go](../client-libraries/go.md) client sends batch imports over gRPC. Its gRPC search API is still experimental, and is reached through `Experimental().Search()` rather than the regular query builder. Alternatively, you can use other tools, such as the `grpcurl` command-line tool, to interact with the gRPC API. Some options include: diff --git a/docs/weaviate/api/index.mdx b/docs/weaviate/api/index.mdx index fbfd98a89..e77712b7f 100644 --- a/docs/weaviate/api/index.mdx +++ b/docs/weaviate/api/index.mdx @@ -32,7 +32,7 @@ Weaviate offers official client libraries for these programming languages: The client libraries abstract away the complexities of making direct REST, GraphQL, or gRPC calls. You interact with Weaviate using idiomatic methods and objects in your preferred language. -Modern versions of the client libraries (e.g., Python v4+, TypeScript v3+) are designed to **automatically utilize the more performant gRPC interface** for search and query operations whenever possible and supported by the Weaviate instance you are connected to. They handle the negotiation and use the optimal protocol under the hood, typically requiring only the standard connection details (like the REST endpoint URL and authentication keys) for setup. +Modern versions of the client libraries are designed to **automatically utilize the more performant gRPC interface** for search and query operations whenever possible and supported by the Weaviate instance you are connected to. They handle the negotiation and use the optimal protocol under the hood, typically requiring only the standard connection details (like the REST endpoint URL and authentication keys) for setup. ::: diff --git a/docs/weaviate/best-practices/code-generation.md b/docs/weaviate/best-practices/code-generation.md index e15a344d0..96a07712b 100644 --- a/docs/weaviate/best-practices/code-generation.md +++ b/docs/weaviate/best-practices/code-generation.md @@ -18,12 +18,12 @@ Here are some tips for writing Weaviate client library code with generative AI m Weaviate provides two [MCP](https://modelcontextprotocol.io/) servers that integrate with AI development tools like Claude Code, Claude Desktop, Cursor, and VS Code: -- **[Weaviate MCP Server](../configuration/mcp-server.mdx)** — Built into Weaviate itself. Lets AI assistants inspect schemas, search data, and modify objects in your Weaviate instance directly. Enable with `MCP_SERVER_ENABLED=true`. -- **[Weaviate Docs MCP Server](../mcp/docs-mcp-server.mdx)** — A standalone server that gives AI assistants access to Weaviate's documentation, reducing hallucinations when generating Weaviate code. +- **[Weaviate MCP Server](../configuration/mcp-server.mdx)**: Built into Weaviate itself. Lets AI assistants inspect schemas, search data, and modify objects in your Weaviate instance directly. Enable with `MCP_SERVER_ENABLED=true`. +- **[Weaviate Docs MCP Server](../mcp/docs-mcp-server.mdx)**: A standalone server that gives AI assistants access to Weaviate's documentation, reducing hallucinations when generating Weaviate code. ### Weaviate Agent Skills -**[Weaviate Agent Skills](https://github.com/weaviate/agent-skills)** gives AI coding agents (Claude Code, Cursor, GitHub Copilot, and others) built-in knowledge of Weaviate — covering search, collection management, data import, and complete application blueprints such as RAG, agentic RAG, and chatbots. When the skill is installed, agents can discover and use it automatically, reducing hallucinations and speeding up Weaviate development. +**[Weaviate Agent Skills](https://github.com/weaviate/agent-skills)** gives AI coding agents (Claude Code, Cursor, GitHub Copilot, and others) built-in knowledge of Weaviate, covering search, collection management, data import, and complete application blueprints such as RAG, agentic RAG, and chatbots. When the skill is installed, agents can discover and use it automatically, reducing hallucinations and speeding up Weaviate development. Install with: @@ -100,7 +100,7 @@ Review the documentation of your specific IDE to see if it has this feature, and ### Consider using the Query Agent -The [Query Agent](/query-agent) is a pre-built agentic search service that decides the search terms, filters, sorts, and other search parameters for you — the [modes overview](/query-agent/guides/index.md) covers what it can do. +The [Query Agent](/query-agent) is a pre-built agentic search service that decides the search terms, filters, sorts, and other search parameters for you. The [modes overview](/query-agent/guides/index.md) covers what it can do. The Query Agent is available to Weaviate Cloud users for interacting with their Weaviate Cloud instance in natural language. For some use cases, this may be a better approach than using AI-powered code generation tools. diff --git a/docs/weaviate/best-practices/index.md b/docs/weaviate/best-practices/index.md index 8403e0162..cbad4f313 100644 --- a/docs/weaviate/best-practices/index.md +++ b/docs/weaviate/best-practices/index.md @@ -99,7 +99,7 @@ As the size of your dataset grows, the accompanying vector indexes can lead to h If you have a large number of vectors, consider using vector quantization to reduce the memory footprint of the vector index. This will reduce the required memory, and allow you to scale more effectively at lower costs. -If memory is your priority, you can also consider the disk-based [HFresh index](/weaviate/concepts/vector-index#hfresh-index) — an index-level alternative to quantization for reducing the memory footprint. +If memory is your priority, you can also consider the disk-based [HFresh index](/weaviate/concepts/vector-index#hfresh-index), an index-level alternative to quantization for reducing the memory footprint. ![Overview of quantization schemes](../../../_includes/images/concepts/quantization_overview_light.png#gh-light-mode-only "Overview of quantization schemes") ![Overview of quantization schemes](../../../_includes/images/concepts/quantization_overview_dark.png#gh-dark-mode-only "Overview of quantization schemes") @@ -270,8 +270,8 @@ from weaviate.classes.config import Property, DataType client.collections.create( name="WikiArticle", properties=[ - Property(name="title", data_type=DataType.TEXT) - Property(name="category", data_type=DataType.TEXT) + Property(name="title", data_type=DataType.TEXT), + Property(name="category", data_type=DataType.TEXT), ], ) ``` @@ -295,14 +295,24 @@ When importing any significant amount of data (i.e. more than 10 objects), use b for obj in objects: collection.data.insert(properties=obj) -# ✅ Do this -with collection.batch.fixed_size(batch_size=200) as batch: +# ✅ Do this: server-side batching (recommended) - the server +# tells the client how much data to send next +with collection.batch.stream() as batch: for obj in objects: batch.add_object(properties=obj) + +# ✅ Or, if your objects are already in an in-memory list, +# ingest the whole list with a single call +result = collection.data.ingest(objects) ``` +[Server-side batching](../concepts/data-import.mdx#server-side-batching) requires Weaviate `v1.36` or later and a client that supports it. If it is not available, use a client-side batching method such as `collection.batch.fixed_size(batch_size=200)` or `collection.batch.dynamic()` instead. + +Avoid passing large lists to `collection.data.insert_many()`. It sends all objects in a single request, which fails if the request exceeds the server's [`GRPC_MAX_MESSAGE_SIZE`](/deploy/configuration/env-vars/index.md#GRPC_MAX_MESSAGE_SIZE) limit. `collection.data.ingest()` is a drop-in replacement that does not have this limitation. + :::tip Further resources - [How-to: Batch import data](../manage-objects/import.mdx) +- [Concepts: Data import](../concepts/data-import.mdx) ::: ### Minimize costs by offloading inactive tenants @@ -352,11 +362,11 @@ When using Weaviate in an asynchronous environment, consider using the asynchron #### Python -The Weaviate Python client `4.7.0` and higher includes an [asynchronous client API (`WeaviateAsyncClient`)](../client-libraries/python/async.md). +The Weaviate Python client includes an [asynchronous client API (`WeaviateAsyncClient`)](../client-libraries/python/async.md). #### Java -The Weaviate Java client `5.0.0` and higher includes an [asynchronous client API (`WeaviateAsyncClient`)](https://javadoc.io/doc/io.weaviate/client/latest/io/weaviate/client/v1/async/WeaviateAsyncClient.html). +The Weaviate Java client includes an [asynchronous client API (`WeaviateClientAsync`)](https://javadoc.io/doc/io.weaviate/client6/latest/io/weaviate/client6/v1/api/WeaviateClientAsync.html). ## Questions and feedback diff --git a/docs/weaviate/client-libraries/_includes/feedback.mdx b/docs/weaviate/client-libraries/_includes/feedback.mdx deleted file mode 100644 index 864943be8..000000000 --- a/docs/weaviate/client-libraries/_includes/feedback.mdx +++ /dev/null @@ -1,7 +0,0 @@ -:::note Questions -Please answer in relation to this section of the API. - -- Any specific features that you liked or disliked? -- Did you encounter any errors or difficulties? If yes, what were they? -- Any other notes or suggestions for improvement? -::: \ No newline at end of file diff --git a/docs/weaviate/client-libraries/csharp.mdx b/docs/weaviate/client-libraries/csharp.mdx index c1f1590c3..a5acd5ac4 100644 --- a/docs/weaviate/client-libraries/csharp.mdx +++ b/docs/weaviate/client-libraries/csharp.mdx @@ -46,7 +46,7 @@ dotnet add package Weaviate.Client --version ||site.csharp_client_version|| #### Weaviate version compatibility -The C# client requires Weaviate `1.33.0` or higher. Generally, we encourage you to use the latest version of the C# client and the Weaviate Database. +The C# client requires Weaviate `v1.32.0` and later. Generally, we encourage you to use the latest version of the C# client and the Weaviate Database. #### gRPC @@ -74,10 +74,10 @@ import BasicPrereqs from "/_includes/prerequisites-quickstart.md"; Get started with Weaviate using this C# example. The code walks you through these key steps: -1. **[Connect to Weaviate](docs/weaviate/connections/index.mdx)**: Establish a connection to a local (or Cloud) Weaviate instance. -1. **[Create a collection](../manage-collections/index.mdx)**: Define the data schema for a `Question` collection, using an Ollama model to vectorize the data. -1. **[Import data](../manage-objects/import.mdx)**: Fetch sample Jeopardy questions and use Weaviate's batch import for efficient ingestion and automatic vector embedding generation. -1. **[Search/query the database](../search/index.mdx)**: Execute a vector search to find questions semantically similar to the query `biology`. +1. **[Connect to Weaviate](/weaviate/connections/index.mdx)**: Establish a connection to a Weaviate Cloud instance, using credentials read from environment variables. +1. **[Create a collection](../manage-collections/index.mdx)**: Define a `Movie` collection that uses a Weaviate Embeddings model to vectorize the data. +1. **[Import data](../manage-objects/import.mdx)**: Insert a small set of movie objects in one batch, so Weaviate generates their vector embeddings automatically. +1. **[Search/query the database](../search/index.mdx)**: Execute a vector search to find movies semantically similar to the query `sci-fi`. diff --git a/docs/weaviate/client-libraries/go.md b/docs/weaviate/client-libraries/go.md index 095e543e9..f7d553846 100644 --- a/docs/weaviate/client-libraries/go.md +++ b/docs/weaviate/client-libraries/go.md @@ -24,7 +24,7 @@ The latest Go client is version `v||site.go_client_version||`. ::: -The Weaviate Go client is compatible with Go 1.16+. +For the minimum supported Go version, see the [`go` directive in `go.mod`](https://github.com/weaviate/weaviate-go-client/blob/v||site.go_client_version||/go.mod) for client `v||site.go_client_version||`. ## Installation The client doesn't support the old Go modules system. Create a repository for your code before you import the Weaviate client. diff --git a/docs/weaviate/client-libraries/index.mdx b/docs/weaviate/client-libraries/index.mdx index f4eebd3fb..e56a6a528 100644 --- a/docs/weaviate/client-libraries/index.mdx +++ b/docs/weaviate/client-libraries/index.mdx @@ -15,13 +15,13 @@ export const clientLibrariesData = [ { title: "Python Client", description: - "Install and use the official Python client (v4) to interact with Weaviate.", + "Install and use the official Python client to interact with Weaviate.", link: "/weaviate/client-libraries/python/", icon: "fab fa-python", }, { title: "TypeScript / JavaScript Client", - description: "Use the official client (v3) with Node.js.", + description: "Use the official client with Node.js.", link: "/weaviate/client-libraries/typescript/", icon: "fab fa-js", }, diff --git a/docs/weaviate/client-libraries/java/index.mdx b/docs/weaviate/client-libraries/java/index.mdx index 8caaf9690..e62f4de07 100644 --- a/docs/weaviate/client-libraries/java/index.mdx +++ b/docs/weaviate/client-libraries/java/index.mdx @@ -75,7 +75,7 @@ This ensures that all dynamically-loaded dependencies of `io.grpc` are resolved #### Weaviate version compatibility -The `v6` Java client requires Weaviate `1.33.0` or higher. Generally, we encourage you to use the latest version of the Java client and the Weaviate Database. +The `v6` Java client requires Weaviate `v1.32.0` and later. Generally, we encourage you to use the latest version of the Java client and the Weaviate Database. #### gRPC @@ -104,7 +104,7 @@ import BasicPrereqs from "/_includes/prerequisites-quickstart.md"; Get started with Weaviate using this Java example. The code walks you through these key steps: -1. **[Connect to Weaviate](docs/weaviate/connections/index.mdx)**: Establish a connection to a local (or Cloud) Weaviate instance. +1. **[Connect to Weaviate](../../connections/index.mdx)**: Establish a connection to a local (or Cloud) Weaviate instance. 1. **[Create a collection](../../manage-collections/index.mdx)**: Define the data schema for a `Question` collection, using an Ollama model to vectorize the data. 1. **[Import data](../../manage-objects/import.mdx)**: Fetch sample Jeopardy questions and use Weaviate's batch import for efficient ingestion and automatic vector embedding generation. 1. **[Search/query the database](../../search/index.mdx)**: Execute a vector search to find questions semantically similar to the query `biology`. diff --git a/docs/weaviate/client-libraries/python/async.md b/docs/weaviate/client-libraries/python/async.md index 2af0897e3..fdb65dc9b 100644 --- a/docs/weaviate/client-libraries/python/async.md +++ b/docs/weaviate/client-libraries/python/async.md @@ -130,7 +130,8 @@ Methods that involve sending requests to Weaviate will be async functions. For e - `async_client.connect()`: Connect to a Weaviate server - `async_client.collections.create()`: Create a new collection -- `.data.insert_many()`: Insert a list of objects into a collection +- `.data.insert_many()`: Insert a list of objects into a collection in a single request +- `.data.ingest()`: Insert a list of objects into a collection using [server-side batching](../../manage-objects/import.mdx#server-side-batching) ### Example sync methods @@ -193,7 +194,11 @@ Note the use of a context manager in the async function. The context manager is The async client supports server-side batching through the `stream()` method, which uses the same feedback-based flow as the synchronous client. For client-side batching methods (`dynamic`, `fixed_size`, `rate_limit`), use the synchronous client. -The async client also offers `insert` and `insert_many` methods for data insertion, which can be used in an async context. +The async client also offers `insert` and `insert_many` methods for data insertion, which can be used in an async context. The one-shot `data.ingest()` method is also available on the async client and is preferred over `insert_many` for large lists. + +:::caution `insert_many` and large lists +`insert_many` sends all objects in a **single request**. The server rejects requests larger than its [`GRPC_MAX_MESSAGE_SIZE`](/deploy/configuration/env-vars/index.md#GRPC_MAX_MESSAGE_SIZE) limit, so the whole call fails for large lists. Use `data.ingest()` instead: it is a drop-in replacement that splits the list into server-paced batches. +::: ### Application-level example diff --git a/docs/weaviate/client-libraries/python/index.mdx b/docs/weaviate/client-libraries/python/index.mdx index b4b4cb635..aadc6757c 100644 --- a/docs/weaviate/client-libraries/python/index.mdx +++ b/docs/weaviate/client-libraries/python/index.mdx @@ -55,7 +55,7 @@ pip install --pre -U "weaviate-client==4.*"` #### Weaviate version compatibility -The `v4` Python client requires Weaviate `1.23.7` or higher. Generally, we encourage you to use the latest version of the Python client and the Weaviate Database. +The `v4` Python client requires Weaviate `v1.23.7` and later. Generally, we encourage you to use the latest version of the Python client and the Weaviate Database. In Weaviate Cloud, clusters are compatible with the `v4` client as of 31 January, 2024. Clusters created before this date will not be compatible with the `v4` client. @@ -94,7 +94,7 @@ import BasicPrereqs from "/_includes/prerequisites-quickstart.md"; Get started with Weaviate using this Python example. The code walks you through these key steps: -1. **[Connect to Weaviate](docs/weaviate/connections/index.mdx)**: Establish a connection to a local (or Cloud) Weaviate instance. +1. **[Connect to Weaviate](../../connections/index.mdx)**: Establish a connection to a local (or Cloud) Weaviate instance. 1. **[Create a collection](../../manage-collections/index.mdx)**: Define the data schema for a `Question` collection, using an Ollama model to vectorize the data. 1. **[Import data](../../manage-objects/import.mdx)**: Fetch sample Jeopardy questions and use Weaviate's batch import for efficient ingestion and automatic vector embedding generation. 1. **[Search/query the database](../../search/index.mdx)**: Execute a vector search to find questions semantically similar to the query `biology`. diff --git a/docs/weaviate/client-libraries/python/notes-best-practices.mdx b/docs/weaviate/client-libraries/python/notes-best-practices.mdx index e2ec89f33..d54190bd3 100644 --- a/docs/weaviate/client-libraries/python/notes-best-practices.mdx +++ b/docs/weaviate/client-libraries/python/notes-best-practices.mdx @@ -245,6 +245,19 @@ These methods return a new context manager for each batch. Attributes that are r If the background thread that is responsible for sending the batches raises an exception during batch processing, the error is raised to the main thread. +### One-shot ingest + +`collection.data.ingest(objs)` is a one-shot convenience that uses server-side batching under the hood (no batching context required). It accepts any iterable of plain property dicts or `DataObject` instances, and returns the same `BatchObjectReturn` object as `insert_many`. Pass a list of objects that you already hold in memory to use `ingest` as a drop-in replacement for `insert_many` on large lists. Pass a generator, or any other lazy iterable, to import from a source that does not fit in memory: the client sends each object to the server as the generator produces it. + + + +For a generator that reads a source file line by line, see [Batch import](../../manage-objects/import.mdx#server-side-batching). + ### Error handling During a batch import, any failed objects or references will be stored for retrieval. Additionally, a running count of failed objects and references is maintained. @@ -262,6 +275,8 @@ Note that these lists are reset when a batching process is initialized. So make language="py" /> +`collection.data.ingest()` does not use a batching context, so it reports failures through its return value instead. Check `result.has_errors` for a quick summary flag that tells you whether anything failed. For the detail, check `result.errors`, a dictionary that holds one entry per failed object, keyed by the position of the object in the input. The [one-shot ingest](#one-shot-ingest) example above shows this pattern. + ### Batch vectorization import BatchVectorizationOverview from "/_includes/code/client-libraries/batch-import.mdx"; diff --git a/docs/weaviate/client-libraries/typescript/notes-best-practices.mdx b/docs/weaviate/client-libraries/typescript/notes-best-practices.mdx index 44ec04a23..e42d56cff 100644 --- a/docs/weaviate/client-libraries/typescript/notes-best-practices.mdx +++ b/docs/weaviate/client-libraries/typescript/notes-best-practices.mdx @@ -124,7 +124,7 @@ You can set `skipInitChecks` to `true` to skip these checks. ```js import weaviate from 'weaviate-client'; -Add commentMore actions + const client = await weaviate.connectToLocal({ skipInitChecks: true, }) @@ -147,7 +147,7 @@ type Article = { wordcount: number, }; -const collection = client.collections.get < Article > "Article"; +const collection = client.collections.get
("Article"); await collection.data.insert({ // compiler error since 'body' field is missing in '.insert' title: "TS is awesome!", diff --git a/docs/weaviate/concepts/cluster.md b/docs/weaviate/concepts/cluster.md index d5706acd1..efd253971 100644 --- a/docs/weaviate/concepts/cluster.md +++ b/docs/weaviate/concepts/cluster.md @@ -120,9 +120,17 @@ This only applies when creating a new class, rather than when adding more data t ## Consistency and current limitations -* Weaviate adopts the [Raft consensus algorithm](https://raft.github.io/) which is a log-based algorithm coordinated by an elected leader. This brings an additional benefit in that concurrent schema changes are now supported.
If you are a Kubernetes user, see the [`1.25 migration guide`](/deploy/migration/weaviate-1-25.md) before you upgrade. To upgrade, you have to delete your existing StatefulSet. -* As of `v1.8.0`, the process of broadcasting schema changes across the cluster uses a form of two-phase transaction that as of now cannot tolerate node failures during the lifetime of the transaction. -* As of `v1.8.0`, dynamically scaling a cluster is not fully supported yet. New nodes can be added to an existing cluster, however it does not affect the ownership of shards. Existing nodes can not yet be removed if data is present, as shards are not yet being moved to other nodes prior to a removal of a node. +* From `v1.25`, Weaviate uses the [Raft consensus algorithm](https://raft.github.io/) for cluster metadata such as collection definitions and tenant activity statuses. Raft is a log-based algorithm coordinated by an elected leader, so cluster metadata changes remain consistent even if a minority of nodes fail, and concurrent schema changes are supported. For details, see [Replication architecture: Cluster metadata](/weaviate/concepts/replication-architecture/consistency.md#cluster-metadata).
If you are a Kubernetes user, see the [`1.25 migration guide`](/deploy/migration/weaviate-1-25.md) before you upgrade. To upgrade, you have to delete your existing StatefulSet. +* Adding a node to an existing cluster does not by itself change the ownership of existing shards. To rebalance data across nodes, or to drain a node before you remove it, move its shard replicas with [replica movement](/deploy/configuration/replica-movement.mdx) as described in [Shard replica movement](#shard-replica-movement) above. + +
+ Behavior before `v1.25` and `v1.32` + +Prior to `v1.25`, schema changes were broadcast across the cluster with a form of two-phase transaction that could not tolerate node failures during the lifetime of the transaction. Raft replaced this mechanism. See [Replication architecture: Cluster metadata](/weaviate/concepts/replication-architecture/consistency.md#cluster-metadata) for the comparison. + +Prior to `v1.32`, shard replicas could not be moved between nodes, so a node that still held data could not be removed from a cluster. [Replica movement](/deploy/configuration/replica-movement.mdx) removes that limitation. + +
## Questions and feedback diff --git a/docs/weaviate/concepts/data-import.mdx b/docs/weaviate/concepts/data-import.mdx index 92713352f..b6858e502 100644 --- a/docs/weaviate/concepts/data-import.mdx +++ b/docs/weaviate/concepts/data-import.mdx @@ -50,7 +50,7 @@ This architecture centralizes the complex batching logic on the server, resultin - **Simplified client code**: No need to tweak the batch size and the number of concurrent requests manually. The server determines the optimal batch size based on its current workload. - **Improved stability**: The system automatically applies **backpressure**. If the server is busy, it will instruct the client to send less data, preventing overloads and request timeouts, which is especially useful during long-running vectorization tasks. -- **Enhanced resilience**: It's designed to handle cluster events like node scaling more gracefully, reducing the risk of interrupted batches. +- **Enhanced resilience**: It's designed to handle cluster events like node scaling more gracefully, reducing the risk of interrupted batches. Because the server paces the client, the import load tracks the actual server capacity, which behaves well on autoscaling clusters and under memory pressure. ::: diff --git a/docs/weaviate/concepts/indexing/inverted-index.md b/docs/weaviate/concepts/indexing/inverted-index.md index b10375e51..c2eedb878 100644 --- a/docs/weaviate/concepts/indexing/inverted-index.md +++ b/docs/weaviate/concepts/indexing/inverted-index.md @@ -312,7 +312,7 @@ Beyond the built-in `en` and `none` presets, you can declare custom stopword pre #### Per-property stopword overrides -Each text property can override the collection-level stopword behavior via `textAnalyzer.stopwordPreset`. This is useful for multilingual collections where different properties contain text in different languages. The override is only supported on properties with `tokenization: "word"` — schema validation rejects it on other tokenizers. +Each text property can override the collection-level stopword behavior via `textAnalyzer.stopwordPreset`. This is useful for multilingual collections where different properties contain text in different languages. The override is only supported on properties with `tokenization: "word"`. Schema validation rejects it on other tokenizers. ```json "properties": [ @@ -331,7 +331,7 @@ Each text property can override the collection-level stopword behavior via `text ] ``` -Stopwords are still **indexed** — they are only filtered at query time. Changing the stopword configuration does **not** require reindexing your data. +Stopwords are still **indexed**: they are only filtered at query time. Changing the stopword configuration does **not** require reindexing your data. See the [custom stopwords tutorial](../../tutorials/tokenization.md#example-5-custom-and-per-property-stopword-presets) for a worked example and the [stopwordPresets configuration reference](../../config-refs/indexing/inverted-index.mdx#stopwordpresets) for all options. diff --git a/docs/weaviate/concepts/interface.md b/docs/weaviate/concepts/interface.md index da45c9c4e..19e682187 100644 --- a/docs/weaviate/concepts/interface.md +++ b/docs/weaviate/concepts/interface.md @@ -6,7 +6,7 @@ image: og/docs/concepts.jpg # tags: ['architecture', 'interface', 'API design'] --- -You can manage and use Weaviate through its APIs. Weaviate has a RESTful API and a GraphQL API. The client libraries in all languages support all API functions. Some clients, e.g. the Python client, have additional functionality, such as full schema management and batching operations. This way, Weaviate is easy to use in custom projects. Additionally, the APIs are intuitive, so it is easy to integrate into your existing data landscape. +You can manage and use Weaviate through its APIs. Weaviate has RESTful, GraphQL, and gRPC APIs. The client libraries broadly mirror this API surface, although feature coverage can vary by language; see the [client library pages](/weaviate/client-libraries/index.mdx) for what each one supports. Some clients, e.g. the Python client, have additional functionality, such as full schema management and batching operations. This way, Weaviate is easy to use in custom projects. Additionally, the APIs are intuitive, so it is easy to integrate into your existing data landscape. This page contains information on how Weaviate's APIs are designed, and how you can use Weaviate Console to search through your Weaviate instance with GraphQL. @@ -27,8 +27,8 @@ Weaviate has both a RESTful API and a GraphQL API. Currently, there is no featur - **Data search** -> GraphQL API - **Explorative data search** -> GraphQL API - **Data analysis (meta data)** -> GraphQL API -- **Near real time on very large datasets in production** -> Client libraries (Python, Go, Java, JavaScript) using both APIs under the hood -- **Easy to integrate in applications** -> Client libraries (Python, Go, Java, JavaScript) using both APIs under the hood +- **Near real time on very large datasets in production** -> Client libraries (Python, Go, Java, JavaScript, C#) using both APIs under the hood +- **Easy to integrate in applications** -> Client libraries (Python, Go, Java, JavaScript, C#) using both APIs under the hood ## GraphQL @@ -125,9 +125,11 @@ There are currently three main functions in a GraphQL request: "Get{}", "Explore ## gRPC API support -Starting with version `1.19`, Weaviate is introducing support for the gRPC (gRPC Remote Procedure Calls) API, with the aim of making Weaviate even faster over time. +Alongside the RESTful and GraphQL APIs, Weaviate serves a gRPC API. gRPC is built on HTTP/2 and Protocol Buffers, which makes it faster and more efficient than sending the equivalent request as JSON over HTTP. It was introduced in Weaviate `v1.19.0` and has been considered stable since `v1.23.7`. -This will not result in any user-facing API changes. As of May 2023, gRPC has been added at a very small scale, with the goal of rolling it out further over time to the core library as well as the clients. +gRPC carries most of the search and batch import traffic that the client libraries generate, so a Weaviate deployment usually exposes a gRPC port (`50051` by default, configurable with the `GRPC_PORT` [environment variable](/deploy/configuration/env-vars/index.md)) in addition to the REST port. Client coverage is not uniform: the [Python](/weaviate/client-libraries/python/index.mdx), [TypeScript](/weaviate/client-libraries/typescript/index.mdx), [Java](/weaviate/client-libraries/java/index.mdx), and [C#](/weaviate/client-libraries/csharp.mdx) clients use gRPC for queries and batch operations, while the [Go](/weaviate/client-libraries/go.md) client uses it for batch imports and reaches gRPC search through its experimental API. + +For the Protobuf definitions and for ways to call the API without a client library, see the [gRPC API reference](../api/grpc.md). ## Weaviate Console @@ -137,7 +139,7 @@ The [Weaviate Console](/go/console?utm_content=others) is a dashboard to manage ## Weaviate Clients -Weaviate has several client libraries: in [Go](/weaviate/client-libraries/go.md), [Java](/weaviate/client-libraries/java/index.mdx), [Python](/weaviate/client-libraries/python/index.mdx) and [TypeScript/JavaScript](/weaviate/client-libraries/typescript/index.mdx). The client libraries in all languages support all API functions. Some clients, e.g. the Python client, have additional functionality, such as full schema management and batching operations. This way, Weaviate is easy to use in custom projects. The APIs are intuitive to use, so it is easy to integrate Weaviate into your existing data landscape. +Weaviate has several client libraries: in [C#](/weaviate/client-libraries/csharp.mdx), [Go](/weaviate/client-libraries/go.md), [Java](/weaviate/client-libraries/java/index.mdx), [Python](/weaviate/client-libraries/python/index.mdx), and [TypeScript/JavaScript](/weaviate/client-libraries/typescript/index.mdx). The client libraries broadly mirror the server API surface, although feature coverage varies by language. See the [client library pages](/weaviate/client-libraries/index.mdx) for what each one supports. Some clients, e.g. the Python client, have additional functionality, such as full schema management and batching operations. This way, Weaviate is easy to use in custom projects. The APIs are intuitive to use, so it is easy to integrate Weaviate into your existing data landscape. ## Further resources diff --git a/docs/weaviate/concepts/modules.md b/docs/weaviate/concepts/modules.md index 1998a9e4e..6791ee8f9 100644 --- a/docs/weaviate/concepts/modules.md +++ b/docs/weaviate/concepts/modules.md @@ -43,7 +43,7 @@ Reader or Generator modules can be used on top of a Vectorizer module. These mod ### Other modules -These include those such as `gcs-backup` or `text-spellcheck`. +These include those such as `backup-gcs` or `text-spellcheck`. ## Dependencies diff --git a/docs/weaviate/concepts/replication-architecture/consistency.md b/docs/weaviate/concepts/replication-architecture/consistency.md index 7f28f9be0..122e21103 100644 --- a/docs/weaviate/concepts/replication-architecture/consistency.md +++ b/docs/weaviate/concepts/replication-architecture/consistency.md @@ -355,6 +355,8 @@ Each replica movement operation progresses through a workflow designed to mainta - **FINALIZING**: The bulk data transfer is complete, and the new replica is catching up on any writes that occurred during the transfer. This ensures the replica is fully synchronized with the latest data. You can use the [`REPLICA_MOVEMENT_MINIMUM_ASYNC_WAIT` environment variable](/docs/deploy/configuration/env-vars/index.md#REPLICA_MOVEMENT_MINIMUM_ASYNC_WAIT) to adjust the wait time which ensures that any in progress writes have been completed and replicated to the target node. +- **INTEGRATING** (added in `v1.38.0`): The new replica has joined the shard's replica set and is being brought into the write path on every node. The operation waits until all nodes agree that the new replica is a write target, so that no node can acknowledge a write that skips it. Each node reports that it reached this state only after its own in-flight writes to the shard have drained, which prevents a write that was already accepted from being lost. Once the cluster agrees, the last writes recorded on the source during the transition are applied to the new replica, and the source stops recording further changes for this operation. For copy operations, the next state is **READY**. For move operations, the next state is **DEHYDRATING**. + - **DEHYDRATING**: For move operations, after the new replica is ready, the original replica on the source node is being removed. - **READY**: The operation has completed successfully. The new replica is fully synchronized and ready to serve traffic. For move operations, the source replica has been removed. diff --git a/docs/weaviate/concepts/search/hybrid-search.md b/docs/weaviate/concepts/search/hybrid-search.md index 38159e38d..181fe96b5 100644 --- a/docs/weaviate/concepts/search/hybrid-search.md +++ b/docs/weaviate/concepts/search/hybrid-search.md @@ -82,7 +82,7 @@ With `relativeScoreFusion` (default from `v1.24`), each object is scored by *nor With `rankedFusion` (default for `v1.23` and lower), each object is scored according to its position in the results for the given search, starting from the highest score for the top-ranked object and decreasing down the order. The total score is calculated by adding these rank-based scores from the vector and keyword searches. -Generally, `relativeScoreFusion` might be a a good choice, which is why it is the default. +Generally, `relativeScoreFusion` might be a good choice, which is why it is the default. The main reason is that `relativeScoreFusion` retains more information from the original searches than `rankedFusion`, which only retains the rankings. More generally we believe that the nuances captured in the vector and keyword search metrics are more likely to be reflected in rankings produced by `relativeScoreFusion`. @@ -161,9 +161,19 @@ In contrast, for `rankedFusion`, the object **ID 2** is the top result, closely The alpha value determines the weight of the vector search results in the final hybrid search results. The alpha value can range from 0 to 1: -- `alpha = 0.5` (default): Equal weight to both searches -- `alpha > 0.5`: More weight to vector search +- `alpha = 0`: Keyword search only - `alpha < 0.5`: More weight to keyword search +- `alpha = 0.5`: Equal weight to both searches +- `alpha > 0.5`: More weight to vector search (`0.75` is the default) +- `alpha = 1`: Vector search only + +Lower `alpha` towards `0` to give the keyword component more influence. + +:::caution Set `alpha` explicitly +`0.75` is the server default. It applies only when a request reaches Weaviate with no `alpha` value, which is the case for GraphQL, and over gRPC from Weaviate `v1.36.7` and later, which added the ability for a client to leave `alpha` unset. + +Client libraries do not all leave `alpha` unset. Depending on your client and your server version, the effective weighting can differ from `0.75`, and in some cases can be a pure keyword search. Set `alpha` explicitly whenever the weighting matters, and check your client library page for its behavior. +::: ## Search thresholds diff --git a/docs/weaviate/concepts/search/keyword-search.md b/docs/weaviate/concepts/search/keyword-search.md index ef6b5ecd9..86c3a0ef9 100644 --- a/docs/weaviate/concepts/search/keyword-search.md +++ b/docs/weaviate/concepts/search/keyword-search.md @@ -47,7 +47,7 @@ Tokenization for keyword searches refers to how each source text is split up int The default tokenization method is `word`. -Other tokenization methods such as `whitespace`, `lowercase`, and `field` are available, as well as specialized ones such as `GSE` or `kagome_kr` for other languages ([more details](../../config-refs/collections.mdx#tokenization)). +Other tokenization methods such as `whitespace`, `lowercase`, and `field` are available, as well as specialized ones such as `gse` or `kagome_kr` for other languages ([more details](../../config-refs/collections.mdx#tokenization)). Set the tokenization option [in the inverted index configuration](../../search/bm25.md#set-tokenization) for a collection. @@ -67,7 +67,7 @@ Weaviate uses configurable stopwords in calculating the BM25 score. Any tokens t See the [reference page](../../config-refs/indexing/inverted-index.mdx#stopwords) for more details. -Stopword lists are also configurable per collection **and** per property. You can define custom presets on `invertedIndexConfig.stopwordPresets` and assign them to individual text properties via `textAnalyzer.stopwordPreset`. This is useful for multilingual collections — for example, English and French properties using different stopword lists. Stopwords are still indexed and only filtered at query time, so changing your stopword configuration does not require reindexing. See [Inverted index: Custom stopword presets](../indexing/inverted-index.md#custom-stopword-presets) for details. +Stopword lists are also configurable per collection **and** per property. You can define custom presets on `invertedIndexConfig.stopwordPresets` and assign them to individual text properties via `textAnalyzer.stopwordPreset`. This is useful for multilingual collections. For example, English and French properties can use different stopword lists. Stopwords are still indexed and only filtered at query time, so changing your stopword configuration does not require reindexing. See [Inverted index: Custom stopword presets](../indexing/inverted-index.md#custom-stopword-presets) for details. ### BM25 parameters @@ -142,7 +142,7 @@ Conceptually, it works as though a filter is applied to the results of the BM25 - `and`: All tokens must be present within a single searched property - `or`: At least one token must be present within a single searched property, with the minimum number of tokens being configurable (`minimumOrTokensMatch`) -As an example, a BM25 query of `computer networking guide` with the `and` operator would only return objects where all of the tokens `computer`, `networking`, and `guide` appear together within a single searched property. If the tokens are spread across different properties — for example, `computer` in `title` and `networking` in `description` — the object does not match under `and`. In contrast, the same query with the `or` operator would return objects where at least one of those tokens appears in a searched property. If the `or` operator is used with a `minimumOrTokensMatch` of `2`, then at least two of the tokens must be present within a single searched property. +As an example, a BM25 query of `computer networking guide` with the `and` operator would only return objects where all of the tokens `computer`, `networking`, and `guide` appear together within a single searched property. If the tokens are spread across different properties (for example, `computer` in `title` and `networking` in `description`), the object does not match under `and`. In contrast, the same query with the `or` operator would return objects where at least one of those tokens appears in a searched property. If the `or` operator is used with a `minimumOrTokensMatch` of `2`, then at least two of the tokens must be present within a single searched property. If not specified, the default operator is `or`, with a `minimumOrTokensMatch` of `1`. This means that at least one token must be present in a searched property for the object to be returned. @@ -194,7 +194,7 @@ Here are some key considerations when using keyword search: 1. **Tokenization Choice** - Choose based on your data and search requirements. For example, use `word` tokenization for natural language text, but consider `field` for URLs or email addresses that need exact matching as a whole. - - For multilingual content, consider specialized tokenizers like `GSE` for Chinese/Japanese or `kagome_kr` for Korean + - For multilingual content, consider specialized tokenizers like `gse` for Chinese/Japanese or `kagome_kr` for Korean - Consider special characters and case sensitivity needs - Test your tokenization choice with subsets of your data and queries to ensure it handles special characters and case sensitivity as expected. You could perform these experiments with vectorization disabled to save resources/costs, as the two processes are independent. diff --git a/docs/weaviate/concepts/search/vector-search.md b/docs/weaviate/concepts/search/vector-search.md index 1a4888c93..7300b4f35 100644 --- a/docs/weaviate/concepts/search/vector-search.md +++ b/docs/weaviate/concepts/search/vector-search.md @@ -262,9 +262,9 @@ Standard vector search returns the closest matches to a query, which often means - **Diversity**: how different is the item from the results already selected? The algorithm works iteratively. It selects the most relevant item first, then for each subsequent pick it scores candidates by weighing their query similarity against their maximum similarity to any already-selected result. The `balance` parameter (λ) controls the trade-off: -- **λ = 0.0**: Pure diversity — maximizes difference from already-selected items -- **λ = 0.5**: Balanced — each result must be both relevant and distinct -- **λ = 1.0**: Pure relevance — equivalent to standard vector search +- **λ = 0.0**: Pure diversity (maximizes difference from already-selected items) +- **λ = 0.5**: Balanced (each result must be both relevant and distinct) +- **λ = 1.0**: Pure relevance (equivalent to standard vector search) MMR is applied at query time as a reranking step on top of standard search. No reindexing is needed. The typical pattern is to retrieve a larger candidate set via regular vector search, then rerank a subset using MMR. diff --git a/docs/weaviate/concepts/storage.md b/docs/weaviate/concepts/storage.md index 55846463a..a673277e7 100644 --- a/docs/weaviate/concepts/storage.md +++ b/docs/weaviate/concepts/storage.md @@ -81,7 +81,7 @@ This change improves reliability during rolling restarts and upgrades. Eager loa The [`HNSW_STARTUP_WAIT_FOR_VECTOR_CACHE`](/deploy/configuration/env-vars#HNSW_STARTUP_WAIT_FOR_VECTOR_CACHE) environment variable controls whether vector cache prefill is synchronous (blocking) or asynchronous (background) at startup. Its default changed to `true` in v1.36.6. -For collections where lazy shard loading is active, vector cache prefill is always **asynchronous** — the `HNSW_STARTUP_WAIT_FOR_VECTOR_CACHE` value is overridden to `false` regardless of the configured value. For eagerly-loaded collections, the configured value applies (default: `true`, meaning synchronous prefill). +For collections where lazy shard loading is active, vector cache prefill is always **asynchronous**: the `HNSW_STARTUP_WAIT_FOR_VECTOR_CACHE` value is overridden to `false` regardless of the configured value. For eagerly-loaded collections, the configured value applies (default: `true`, meaning synchronous prefill). :::note Behavior change from v1.36.6 Prior to v1.36.6, lazy shard loading was enabled by default for all collections. From v1.36.6 onward, shards are **eagerly loaded by default** until a multi-tenant collection crosses the count or size threshold. This may increase startup time for smaller deployments but provides better reliability during rollouts. diff --git a/docs/weaviate/concepts/vector-quantization.md b/docs/weaviate/concepts/vector-quantization.md index c1f949274..8ca7994fb 100644 --- a/docs/weaviate/concepts/vector-quantization.md +++ b/docs/weaviate/concepts/vector-quantization.md @@ -197,7 +197,7 @@ You might be also interested in our blog post [HNSW+PQ - Exploring ANN algorithm ### With a flat index -[RQ](#rotational-quantization) and [BQ](#binary-quantization) can be applied to a [flat index](./indexing/inverted-index.md). As a flat index search is a brute-force method, compression reduces the amount of data Weaviate has to read and increases speed. +[RQ](#rotational-quantization) and [BQ](#binary-quantization) can be applied to a [flat index](./indexing/vector-index.md#flat-index). As a flat index search is a brute-force method, compression reduces the amount of data Weaviate has to read and increases speed. ## Rescoring diff --git a/docs/weaviate/config-refs/collections.mdx b/docs/weaviate/config-refs/collections.mdx index 752db36a6..e97108ebd 100644 --- a/docs/weaviate/config-refs/collections.mdx +++ b/docs/weaviate/config-refs/collections.mdx @@ -263,6 +263,23 @@ Additionally, we strongly recommend that you do not use the following words as p - `vector` - `_vector` +##### Reserved suffixes + +A property name may also not *end* in one of the following suffixes, because each would collide with an internal index that Weaviate derives from another property: + +- `_searchable` +- `_rangeable` +- `_temp` +- `__meta_count` +- `_propertyLength` +- `_nullState` + +A property whose name ends in one of these suffixes, such as `comments_temp`, is rejected with a validation error: `'comments_temp' is not a valid property name: suffix '_temp' is reserved for internal indices`. + +This check runs when you create a collection or add a property to an existing collection. It is not applied when an existing collection definition is loaded, so a collection created before the check was introduced continues to work, and a backup that contains such a property still restores. + +The check was added in `v1.38.0`, and backported to `v1.35.20`, `v1.36.15`, and `v1.37.5`. + #### `tokenization` You can customize how `text` data is tokenized and indexed in the inverted index. Tokenization influences the results returned by the [`bm25`](../api/graphql/search-operators.md#bm25) and [`hybrid`](../api/graphql/search-operators.md#hybrid) operators, and [`where` filters](../api/graphql/filters.md). @@ -722,17 +739,23 @@ Some `asyncConfig` parameters have different defaults depending on whether the c | :--- | :--- | :--- | :--- | :--- | | `hashtreeHeight` | Integer | Height of the hash tree used for data comparison between nodes. Min: `0`, Max: `20` | `16` | `10` | | `frequency` | Integer | Frequency of periodic data comparison between nodes, in milliseconds. | `30000` | `30000` | -| `frequencyWhilePropagating` | Integer | Frequency of data comparison while propagation is active, in milliseconds. | `3000` | `3000` | +| `frequencyWhilePropagating` | Integer | Frequency of data comparison while propagation is active, in milliseconds. | `5000` | `5000` | | `loggingFrequency` | Integer | How often the async replication process logs its activity, in seconds. | `60` | `60` | | `diffBatchSize` | Integer | Number of object keys fetched per request during comparison. Min: `1`, Max: `10000` | `1000` | `1000` | | `diffPerNodeTimeout` | Integer | Timeout for a comparison response from a remote node, in seconds. | `10` | `10` | | `prePropagationTimeout` | Integer | Overall timeout for the pre-propagation phase, in seconds. | `300` | `300` | | `propagationTimeout` | Integer | Timeout for a propagation request to a remote node, in seconds. | `60` | `60` | -| `propagationLimit` | Integer | Maximum number of objects propagated in a single iteration. Min: `1`, Max: `1000000` | `10000` | `10000` | +| `propagationLimit` | Integer | Maximum number of objects propagated in a single iteration. Min: `1`, Max: `100000` | `1000` | `1000` | | `propagationDelay` | Integer | Delay before considering an object for propagation, in milliseconds. | `30000` | `30000` | -| `propagationConcurrency` | Integer | Number of concurrent workers for propagation. Min: `1`, Max: `20` | `5` | `5` | +| `propagationConcurrency` | Integer | Number of concurrent workers for propagation. Min: `1`, Max: `20` | `1` | `1` | | `propagationBatchSize` | Integer | Maximum number of objects per propagation batch. Min: `1`, Max: `1000` | `100` | `100` | +:::note Values changed in `v1.34.19`, `v1.35.14`, `v1.36.4` and `v1.37.0` +Three of the defaults above were changed in the patch releases `v1.34.19`, `v1.35.14` and `v1.36.4`, and apply to every release from `v1.37.0` onwards. On earlier releases of each of those lines, `frequencyWhilePropagating` defaults to `3000`, `propagationLimit` defaults to `10000`, and `propagationConcurrency` defaults to `5`. + +The maximum value for `propagationLimit` was lowered from `1000000` to `100000` one patch later, in `v1.34.20`, `v1.35.15` and `v1.36.6`. +::: + #### Code example - How to configure replication This code example shows how to configure the replication parameters through a client library: diff --git a/docs/weaviate/config-refs/datatypes.md b/docs/weaviate/config-refs/datatypes.md index eafcf0828..90cbd82f2 100644 --- a/docs/weaviate/config-refs/datatypes.md +++ b/docs/weaviate/config-refs/datatypes.md @@ -526,7 +526,7 @@ The `blobHash` data type accepts base64-encoded data (same as [`blob`](#blob)) b } ``` -Use `blobHash` when you need a vectorizer to see the raw media at import time but don't need to retrieve the original bytes afterwards — only the hash is stored. +Use `blobHash` when you need a vectorizer to see the raw media at import time but don't need to retrieve the original bytes afterwards: only the hash is stored. ## `object` @@ -536,7 +536,7 @@ For example, a `Person` collection could have an `address` property as an object :::note Indexing and filtering -`object` and `object[]` properties are not vectorized — only their leaf scalars are stored in the inverted index. From Weaviate `v1.38` (preview), you can filter on nested-object leaves using a dotted path syntax. See [Filter on nested object properties](../search/filters.md#filter-on-nested-object-properties). +`object` and `object[]` properties are not vectorized by default, and only their leaf scalars are stored in the inverted index. If you list an object property in the vector configuration's [`properties` field](indexing/vector-index.mdx#specify-which-properties-to-vectorize), it is converted to a string (its JSON representation) and concatenated into the vectorizer's input text. From Weaviate `v1.38` (preview), you can filter on nested-object leaves using a dotted path syntax. See [Filter on nested object properties](../search/filters.md#filter-on-nested-object-properties). ::: diff --git a/docs/weaviate/config-refs/indexing/inverted-index.mdx b/docs/weaviate/config-refs/indexing/inverted-index.mdx index a9da3d3a3..e82d4bbfc 100644 --- a/docs/weaviate/config-refs/indexing/inverted-index.mdx +++ b/docs/weaviate/config-refs/indexing/inverted-index.mdx @@ -191,19 +191,19 @@ A preset name that matches a built-in (`"en"`, `"none"`) fully replaces the buil -The existing [`stopwords`](#stopwords) configuration remains as the default for properties that do not specify a `textAnalyzer.stopwordPreset` override. For extending a built-in preset with `additions`/`removals`, use [`stopwords`](#stopwords) instead — it is the only stopword config that accepts that object form. +The existing [`stopwords`](#stopwords) configuration remains as the default for properties that do not specify a `textAnalyzer.stopwordPreset` override. For extending a built-in preset with `additions`/`removals`, use [`stopwords`](#stopwords) instead. It is the only stopword config that accepts that object form. #### `textAnalyzer` {#textanalyzer} -Part of a **property definition** (not `invertedIndexConfig`). Configures text analysis behavior for individual `text` properties. The accent-folding options (`asciiFold`, `asciiFoldIgnore`) are supported on properties with tokenization `word`, `lowercase`, `whitespace`, `field`, or `trigram` — not on the language-specific tokenizers (`gse`, `gse_ch`, `kagome_ja`, `kagome_kr`). The `stopwordPreset` option is only supported on properties with `tokenization: "word"`. +Part of a **property definition** (not `invertedIndexConfig`). Configures text analysis behavior for individual `text` properties. The accent-folding options (`asciiFold`, `asciiFoldIgnore`) are supported on properties with tokenization `word`, `lowercase`, `whitespace`, `field`, or `trigram`. They are not supported on the language-specific tokenizers (`gse`, `gse_ch`, `kagome_ja`, and `kagome_kr`). The `stopwordPreset` option is only supported on properties with `tokenization: "word"`. | Parameter | Type | Default | Details | | :---------------- | :--------- | :------ | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `asciiFold` | `boolean` | `false` | Normalizes accented Latin characters to ASCII equivalents during indexing and querying. Uses Unicode NFD decomposition. **Immutable** after the property is created. | | `asciiFoldIgnore` | `string[]` | `[]` | Characters exempt from ASCII folding. Each entry must be a single character. **Immutable** after the property is created. | -| `stopwordPreset` | `string` | (none) | Name of a built-in (`en`, `none`) or collection-level stopword preset to use for this property, overriding the default `stopwords` config. **Only supported on properties with `tokenization: "word"`** — schema validation rejects it on other tokenizers. | +| `stopwordPreset` | `string` | (none) | Name of a built-in (`en`, `none`) or collection-level stopword preset to use for this property, overriding the default `stopwords` config. **Only supported on properties with `tokenization: "word"`**. Schema validation rejects it on other tokenizers. |
Example textAnalyzer configuration - JSON object @@ -225,7 +225,7 @@ Part of a **property definition** (not `invertedIndexConfig`). Configures text a :::note -`asciiFoldIgnore` changes which tokens are written to disk. It cannot be modified after the property is created — schema updates that change the ignore list are rejected. To change it, create a new property and reindex. +`asciiFoldIgnore` changes which tokens are written to disk. It cannot be modified after the property is created. Schema updates that change the ignore list are rejected. To change it, create a new property and reindex. ::: @@ -253,7 +253,7 @@ Using these features requires more resources, as the additional inverted indexes ## Drop an inverted index -You can drop (delete) an inverted index from a property. This is a destructive operation — the index data is removed from disk. To use the index again, it must be regenerated. +You can drop (delete) an inverted index from a property. This is a destructive operation: the index data is removed from disk. To use the index again, it must be regenerated. The following index types can be dropped: `searchable`, `filterable`, `rangeFilters`. @@ -306,7 +306,7 @@ Two REST endpoints let you test tokenization without modifying your schema. :::note -`stopwords` and `stopwordPresets` are mutually exclusive — pass one or the other, not both. Use `stopwords` for a single preset optionally tweaked with additions/removals; use `stopwordPresets` to define named presets and select one via `analyzerConfig.stopwordPreset`. +`stopwords` and `stopwordPresets` are mutually exclusive. Pass one or the other, not both. Use `stopwords` for a single preset optionally tweaked with additions/removals; use `stopwordPresets` to define named presets and select one via `analyzerConfig.stopwordPreset`. ::: @@ -386,7 +386,7 @@ curl -X POST http://localhost:8080/v1/tokenize -d '{ ### Property-based tokenization -`POST /v1/schema/{className}/properties/{propertyName}/tokenize` resolves the full analyzer config from an existing property. The property's tokenization method, `textAnalyzer` settings, and the collection's stopword configuration are applied automatically — nothing else needs to be passed. +`POST /v1/schema/{className}/properties/{propertyName}/tokenize` resolves the full analyzer config from an existing property. The property's tokenization method, `textAnalyzer` settings, and the collection's stopword configuration are applied automatically. Nothing else needs to be passed. **Request body:** diff --git a/docs/weaviate/config-refs/indexing/vector-index.mdx b/docs/weaviate/config-refs/indexing/vector-index.mdx index 8b9a230f5..195d27716 100644 --- a/docs/weaviate/config-refs/indexing/vector-index.mdx +++ b/docs/weaviate/config-refs/indexing/vector-index.mdx @@ -156,7 +156,7 @@ Using the `dynamic` index will initially create a flat index and once the number This is only a one-way switch that converts a flat index to a HNSW, the index does not support changing back to a flat index even if the object count goes below the threshold due to deletion. -The goal of `dynamic` indexing is to shorten latencies during query time at the cost of a larger memory footprint. If your priority is the opposite — keeping memory low — consider the [HFresh index](#hfresh-index) instead. +The goal of `dynamic` indexing is to shorten latencies during query time at the cost of a larger memory footprint. If your priority is the opposite (keeping memory low), consider the [HFresh index](#hfresh-index) instead. ### Dynamic index parameters @@ -184,10 +184,10 @@ HFresh only supports `cosine` and `l2-squared` distance metrics. Dot product is | Parameter | Type | Default | Mutable | Details | | :----------------- | :------ | :------- | :------ | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | `distance` | string | `cosine` | No | Distance metric. Only `cosine` and `l2-squared` are supported. | -| `maxPostingSizeKB` | integer | `48` | Yes | Maximum size in KB for a posting list. Weaviate uses this value along with the vector dimensions to calculate the maximum number of vectors per posting. Min: `8`, Max: `1024`. Best set when you create the collection: an update is accepted but only affects newly-indexed data — data that is already indexed is not re-partitioned. | +| `maxPostingSizeKB` | integer | `48` | Yes | Maximum size in KB for a posting list. Weaviate uses this value along with the vector dimensions to calculate the maximum number of vectors per posting. Min: `8`, Max: `1024`. Best set when you create the collection: an update is accepted but only affects newly-indexed data. Data that is already indexed is not re-partitioned. | | `replicas` | integer | `4` | No | Number of posting lists in which a vector is added. Min: `1`, Max: `10`. | -| `searchProbe` | integer | `64` | Yes | Number of posting lists to search during a query. | -| `rq` | object | -- | Partial | Rotational quantization (RQ) compression configuration. RQ is mandatory for HFresh and cannot be turned off. Its `rescoreLimit` (default `350`) — the number of candidates rescored against uncompressed vectors — is mutable at runtime. | +| `searchProbe` | integer | `256` | Yes | Number of posting lists to search during a query. The default is `256` in `v1.36.20`, `v1.37.10`, `v1.38.2` and later. Earlier releases on each of those lines default to `64`. | +| `rq` | object | -- | Partial | Rotational quantization (RQ) compression configuration. RQ is mandatory for HFresh and cannot be turned off. Its `rescoreLimit` (default `350`), the number of candidates rescored against uncompressed vectors, is mutable at runtime. | :::tip Tuning HFresh recall Start with the defaults. If recall is too low, increase `searchProbe` (search more posting lists per query) or the RQ `rescoreLimit` (rescore more candidates with full-precision vectors). Both are mutable at runtime and take effect **without reindexing**. @@ -245,7 +245,7 @@ For instance, text embedding integrations (e.g. `text2vec-cohere` for Cohere, or Unless specified otherwise in the collection definition, the default behavior is to: -- Only vectorize properties with a string value — `text`, `text[]`, and `blob` (a base64-encoded string) — unless [skipped](../../manage-collections/vector-config.mdx#property-level-settings). Other data types (such as `number`, `int`, `boolean`, `date`, and `object`) are not vectorized unless they are listed in `source_properties` (see [below](#specify-which-properties-to-vectorize)). +- Only vectorize properties with a string value (`text`, `text[]`, and `blob`, which is a base64-encoded string) unless [skipped](../../manage-collections/vector-config.mdx#property-level-settings). Other data types (such as `number`, `int`, `boolean`, `date`, and `object`) are not vectorized unless they are listed in `source_properties` (see [below](#specify-which-properties-to-vectorize)). - Sort properties in alphabetical (a-z) order before concatenating values - If `vectorizePropertyName` is `true` (`false` by default) prepend the property name to each property value - Join the (prepended) property values with spaces @@ -277,10 +277,10 @@ To configure vectorization on a per-property basis, use `skip` and `vectorizePro To vectorize only a specific set of properties, set `source_properties` (the `properties` field of the vector configuration). Only the listed properties are then vectorized, in the order given. -When `source_properties` is set, listed properties that are **not** text are also vectorized: `number`, `int`, `boolean`, `date`, `object`, and their array variants are converted to a string and concatenated into the input text. (Without `source_properties`, only string-valued properties — `text`, `text[]`, and `blob` — are vectorized; `uuid`, geo-coordinates, and phone-number properties are never vectorized.) +When `source_properties` is set, listed properties that are **not** text are also vectorized: `number`, `int`, `boolean`, `date`, `object`, and their array variants are converted to a string and concatenated into the input text. (Without `source_properties`, only `text`, `text[]`, and `blob` properties are vectorized. `uuid`, geo-coordinates, and phone-number properties are never vectorized.) :::caution `blob` properties are vectorized as text -A `blob` value is a base64-encoded string, so an indexed `blob` property is vectorized like text — even without `source_properties`. To avoid sending a blob's base64 data to a text vectorizer, exclude it with `source_properties` or [`skip`](../../manage-collections/vector-config.mdx#property-level-settings). +A `blob` value is a base64-encoded string, so an indexed `blob` property is vectorized like text, even without `source_properties`. To avoid sending a blob's base64 data to a text vectorizer, exclude it with `source_properties` or [`skip`](../../manage-collections/vector-config.mdx#property-level-settings). ::: ## Asynchronous indexing diff --git a/docs/weaviate/configuration/authz-authn.md b/docs/weaviate/configuration/authz-authn.md index 767b86aa7..3ad1f36fd 100644 --- a/docs/weaviate/configuration/authz-authn.md +++ b/docs/weaviate/configuration/authz-authn.md @@ -113,7 +113,7 @@ With [undifferentiated access](../../deploy/configuration/authorization.md#undif - [Configuration: OIDC](/deploy/configuration/oidc.md) - [Configuration: RBAC](/weaviate/configuration/rbac/index.mdx) - [Configuration: Environment variables - Authentication and Authorization](/deploy/configuration/env-vars/index.md#authentication-and-authorization) -- [Weaviate MCP server](/weaviate/configuration/mcp-server.mdx) — authenticates via API key and respects RBAC permissions. +- [Weaviate MCP server](/weaviate/configuration/mcp-server.mdx) (authenticates via API key and respects RBAC permissions) ## Questions and feedback diff --git a/docs/weaviate/configuration/compression/multi-vectors.md b/docs/weaviate/configuration/compression/multi-vectors.md index be4537281..549cf4940 100644 --- a/docs/weaviate/configuration/compression/multi-vectors.md +++ b/docs/weaviate/configuration/compression/multi-vectors.md @@ -69,12 +69,12 @@ The [Weaviate Embeddings multimodal model](/weaviate/model-providers/weaviate/em The final dimensionality of the MUVERA encoded vector will be -`repetition * 2^ksim * dprojections`. Carefully tuning these parameters +`repetitions * 2^ksim * dprojections`. Carefully tuning these parameters is crucial to balance memory usage and retrieval accuracy. These parameters can be used to fine-tune MUVERA: -- **`ksim`** (`int`): +- **`ksim`** (`int`, default: `4`): The number of Gaussian vectors sampled for the SimHash partitioning function. This parameter determines the number of bits in the hash, and consequently, the number of buckets created in the space partitioning step. The total @@ -83,7 +83,7 @@ These parameters can be used to fine-tune MUVERA: the accuracy of the approximation but also increasing the dimensionality of the intermediate encoded vectors. -- **`dprojections`** (`int`): +- **`dprojections`** (`int`, default: `16`): The dimensionality of the sub-vectors after the random linear projection in the dimensionality reduction step. After partitioning the multi-vector embedding into buckets, each bucket's aggregated vector is projected down @@ -92,9 +92,9 @@ These parameters can be used to fine-tune MUVERA: fixed-dimensional encoding, leading to lower memory consumption but potentially at the cost of some information loss and retrieval accuracy. -- **`repetition`** (`int`): +- **`repetitions`** (`int`, default: `10`): The number of times the space partitioning and dimensionality reduction - steps are repeated. This repetition allows for capturing different perspectives + steps are repeated. Each repetition captures a different perspective of the multi-vector embedding and can improve the robustness and accuracy of the final fixed-dimensional encoding. The resulting single vectors from each repetition are concatenated. A higher number of repetitions increases diff --git a/docs/weaviate/configuration/compression/sq-compression.md b/docs/weaviate/configuration/compression/sq-compression.md index e512bdec1..bdeb0cfd6 100644 --- a/docs/weaviate/configuration/compression/sq-compression.md +++ b/docs/weaviate/configuration/compression/sq-compression.md @@ -35,6 +35,14 @@ SQ can be enabled at collection creation time through the collection definition: language="py" /> + + + + + + + + + - - No — it's built into the Weaviate Server binary, not a separate package you install. Setting MCP_SERVER_ENABLED=true exposes the MCP endpoint on the same port as the REST API; nothing extra to run or deploy. The separate "Weaviate Docs MCP server" (Kapa-powered, serves documentation to LLMs) is a distinct product. + No. It's built into the Weaviate Server binary, not a separate package you install. Setting MCP_SERVER_ENABLED=true exposes the MCP endpoint on the same port as the REST API; nothing extra to run or deploy. The separate "Weaviate Docs MCP server" (Kapa-powered, serves documentation to LLMs) is a distinct product. - question: Which tools does the Weaviate MCP server expose? answer: >- - Four tools, gated by RBAC permissions — weaviate-collections-get-config (inspect collection schemas), weaviate-tenants-list (list tenants in a multi-tenant collection), weaviate-query-hybrid (run hybrid vector + keyword searches), and weaviate-objects-upsert (create or update objects, requires write access). + Four tools, gated by RBAC permissions. They are weaviate-collections-get-config (inspect collection schemas), weaviate-tenants-list (list tenants in a multi-tenant collection), weaviate-query-hybrid (run hybrid vector + keyword searches), and weaviate-objects-upsert (create or update objects, requires write access). - question: How do I request a new MCP tool or feature? answer: >- Open a feature request on the Weaviate GitHub repo at https://github.com/weaviate/weaviate/issues/new/choose. Pick the "Feature request" template and describe the tool, parameter, or capability you'd like the MCP server to expose, with a concrete use case. @@ -244,7 +244,7 @@ Lists tenants for multi-tenant collections. - `collection_name` (string, required): The collection to inspect. -**Returns:** List of tenants and their activity status (HOT/COLD). +**Returns:** List of tenants and their activity status (`ACTIVE` or `INACTIVE`, and `OFFLOADED` for tenants that have been offloaded to cold storage). ### `weaviate-query-hybrid` @@ -260,7 +260,10 @@ Performs a hybrid search combining vector similarity and keyword matching (BM25) - `target_vectors` (array, optional): Named vectors to use for vector search. - `target_properties` (array, optional): Properties to search with BM25. If omitted, searches all text properties. - `return_properties` (array, optional): Properties to include in results. -- `return_metadata` (array, optional): Metadata fields to return (e.g., `id`, `distance`, `score`, `creationTimeUnix`). +- `return_metadata` (array, optional): Metadata fields to return (e.g., `id`, `vector`, `distance`, `score`, `creationTimeUnix`, `lastUpdateTimeUnix`). +- `filters` (object, optional): A [where filter](/weaviate/api/graphql/filters.md) applied before scoring. + +A leaf filter is an object with `path` (an array of property names), `operator`, and a typed value field. The typed value field is one of `valueText`, `valueInt`, `valueNumber`, `valueBoolean`, `valueDate`, the corresponding `value*Array` field for a `Contains*` operator, or `valueGeoRange` for `WithinGeoRange`. Combine leaves with `{"operator": "And" | "Or", "operands": [ ... ]}`, nested to any depth. The supported operators are `And`, `Or`, `Not`, `Equal`, `NotEqual`, `Like`, `GreaterThan`, `GreaterThanEqual`, `LessThan`, `LessThanEqual`, `ContainsAny`, `ContainsAll`, `ContainsNone`, `WithinGeoRange`, and `IsNull`. See [Concepts: Filtering](/weaviate/concepts/filtering.md) for how filters interact with search. **Returns:** Ranked objects with similarity scores and distances. diff --git a/docs/weaviate/configuration/modules.md b/docs/weaviate/configuration/modules.md index c4264ba89..805b2b8f8 100644 --- a/docs/weaviate/configuration/modules.md +++ b/docs/weaviate/configuration/modules.md @@ -57,16 +57,17 @@ services: The list of API-based modules can be found on the [model provider integrations page](../model-providers/index.md#api-based). You can also inspect the [source code](https://github.com/weaviate/weaviate/blob/main/adapters/handlers/rest/configure_api.go) where the list is defined. -This can be combined with enabling individual modules. For example, the example below enables all API-based modules, Ollama modules and the `backup-s3` module. +Enabling individual modules can be combined with the API-based modules. For example, since API-based modules are enabled by default from `v1.33`, the example below enables the Ollama modules and the `backup-s3` module alongside them. ```yaml services: weaviate: environment: - ENABLE_API_BASED_MODULES: 'true' ENABLE_MODULES: 'text2vec-ollama,generative-ollama,backup-s3' ``` +To opt out of the API-based modules from `v1.33` onwards, set `API_BASED_MODULES_DISABLED` to `true`. The older `ENABLE_API_BASED_MODULES` variable is no longer read. + Note that enabling multiple vectorizer (e.g. `text2vec`, `multi2vec`) modules will disable the [`Explore` functionality](../api/graphql/explore.md). If you need to use `Explore`, you should only enable one vectorizer module. ### Module-specific variables diff --git a/docs/weaviate/configuration/rbac/manage-groups.mdx b/docs/weaviate/configuration/rbac/manage-groups.mdx index f2400971f..2a66b8ee0 100644 --- a/docs/weaviate/configuration/rbac/manage-groups.mdx +++ b/docs/weaviate/configuration/rbac/manage-groups.mdx @@ -303,4 +303,3 @@ Groups assigned to role 'testRole': import DocsFeedback from "/_includes/docs-feedback.mdx"; -```` diff --git a/docs/weaviate/connections/connect-cloud.mdx b/docs/weaviate/connections/connect-cloud.mdx index 41b14e48c..5c65bbd5c 100644 --- a/docs/weaviate/connections/connect-cloud.mdx +++ b/docs/weaviate/connections/connect-cloud.mdx @@ -83,7 +83,7 @@ import HostnameWarning from "/_includes/wcs/hostname-warning.mdx"; text={GoCode} startMarker="// START APIKeyWCD" endMarker="// END APIKeyWCD" - language="py" + language="goraw" /> @@ -139,7 +139,7 @@ If you use API-based models for vectorization or RAG, you must provide an API ke text={GoCode} startMarker="// START ThirdPartyAPIKeys" endMarker="// END ThirdPartyAPIKeys" - language="py" + language="goraw" /> diff --git a/docs/weaviate/connections/connect-custom.mdx b/docs/weaviate/connections/connect-custom.mdx index 31058ba43..6a75f6998 100644 --- a/docs/weaviate/connections/connect-custom.mdx +++ b/docs/weaviate/connections/connect-custom.mdx @@ -15,7 +15,7 @@ import TsCodeV3 from "!!raw-loader!/_includes/code/connections/connect-ts-v3.ts" import JavaV6Code from "!!raw-loader!/_includes/code/java-v6/src/test/java/ConnectionTest.java"; import CSharpCode from "!!raw-loader!/_includes/code/csharp/ConnectionTest.cs"; -The [Python Client v4](/weaviate/client-libraries/python) and the [TypeScript Client v3](../client-libraries/typescript/index.mdx) provide helper methods for common connection types. They also provide custom methods for when you need additional connection configuration. +The [Python client](/weaviate/client-libraries/python) and the [TypeScript client](../client-libraries/typescript/index.mdx) provide helper methods for common connection types. They also provide custom methods for when you need additional connection configuration. If you are using one of the other clients, the standard connection methods are configurable for all connections. @@ -121,11 +121,11 @@ import WCDOIDCWarning from "/_includes/wcd-oidc.mdx"; The examples below assume you already have a bearer access token. Set the following environment variables before running them: -- `WEAVIATE_HTTP_HOST` — host:port of the Weaviate REST endpoint (e.g., `localhost:8080`) -- `WEAVIATE_GRPC_HOST` — host:port of the Weaviate gRPC endpoint (e.g., `localhost:50051`) -- `WEAVIATE_OIDC_ACCESS_TOKEN` — the access token from your IdP -- `WEAVIATE_OIDC_REFRESH_TOKEN` — *(optional)* refresh token for automatic renewal -- `WEAVIATE_OIDC_EXPIRES_IN` — *(optional)* token lifetime in seconds +- `WEAVIATE_HTTP_HOST`: host:port of the Weaviate REST endpoint (e.g., `localhost:8080`) +- `WEAVIATE_GRPC_HOST`: host:port of the Weaviate gRPC endpoint (e.g., `localhost:50051`) +- `WEAVIATE_OIDC_ACCESS_TOKEN`: the access token from your IdP +- `WEAVIATE_OIDC_REFRESH_TOKEN`: *(optional)* refresh token for automatic renewal +- `WEAVIATE_OIDC_EXPIRES_IN`: *(optional)* token lifetime in seconds import OIDCExamples from "/_includes/code/connections/oidc-connect.mdx"; diff --git a/docs/weaviate/connections/connect-query.mdx b/docs/weaviate/connections/connect-query.mdx index 7e7852f0f..ef5e7d34f 100644 --- a/docs/weaviate/connections/connect-query.mdx +++ b/docs/weaviate/connections/connect-query.mdx @@ -9,7 +9,7 @@ import WCDQueryConsoleLocation from "/docs/cloud/img/wcs-query-console-location. -The fastest way to query data in [Weaviate Cloud (WCD)](/cloud) without writing code is the [Query Agent](/docs/cloud/tools/query-agent.mdx) — a natural-language interface that translates a prompt into a multi-step search against your collections and returns a grounded answer. +The fastest way to query data in [Weaviate Cloud (WCD)](/cloud) without writing code is the [Query Agent](/docs/cloud/tools/query-agent.mdx), a natural-language interface that translates a prompt into a multi-step search against your collections and returns a grounded answer. If you'd rather write GraphQL by hand, the [Query tool](#query-tool) is still available further down on this page. @@ -53,7 +53,7 @@ Beyond picking collections, you can also provide API keys for external model pro ### Generate client code -After running a query in the console, you can copy a Python or TypeScript snippet that reproduces the same call through the [`weaviate-agents`](/query-agent/index.md) client libraries — useful for moving from exploration into application code. +After running a query in the console, you can copy a Python or TypeScript snippet that reproduces the same call through the [`weaviate-agents`](/query-agent/index.md) client libraries. This makes it easy to move from exploration into application code. ### Limitations diff --git a/docs/weaviate/manage-collections/inverted-index.mdx b/docs/weaviate/manage-collections/inverted-index.mdx index 76cd11557..9c697a5f7 100644 --- a/docs/weaviate/manage-collections/inverted-index.mdx +++ b/docs/weaviate/manage-collections/inverted-index.mdx @@ -32,13 +32,13 @@ You can [enable inverted indexes](#enable-inverted-index-for-keyword-searches-an
Enabling inverted index -The inverted index in Weaviate can be enabled through parameters at the property level: +The inverted index in Weaviate can be enabled through parameters at the property level. The names below are the REST and camelCase client spellings; the Python client uses the snake_case equivalent, so `indexFilterable` is `index_filterable`, `indexSearchable` is `index_searchable` and `indexRangeFilters` is `index_range_filters`. -**`index_filterable`** - Controls whether a property can be used in where filters. When set to `true`, the property values are indexed for efficient filtering operations. Disable this for properties that don't need filtering to save storage space. +**`indexFilterable`** - Controls whether a property can be used in where filters. When set to `true`, the property values are indexed for efficient filtering operations. Disable this for properties that don't need filtering to save storage space. -**`index_searchable`** - Determines whether a property participates in keyword search queries. When `true`, the property's text content is tokenized and indexed for search. Set to `false` for properties that shouldn't be searchable to improve performance. +**`indexSearchable`** - Determines whether a property participates in keyword search queries. When `true`, the property's text content is tokenized and indexed for search. Set to `false` for properties that shouldn't be searchable to improve performance. -**`index_range_filters`** - Enables range filtering capabilities (greater than, less than, etc.) for numerical and date properties. When enabled, additional indexing structures are created to support efficient range queries. +**`indexRangeFilters`** - Enables range filtering capabilities (greater than, less than, etc.) for numerical and date properties. When enabled, additional indexing structures are created to support efficient range queries.
@@ -92,17 +92,17 @@ The inverted index in Weaviate can be enabled through parameters at the property
Inverted index parameters -The inverted index in Weaviate can be configured through various parameters at the collection level: +The inverted index in Weaviate can be configured through various parameters at the collection level. The names below are the REST and camelCase client spellings. In REST, `b` and `k1` are members of the `bm25` object. The Python client uses the snake_case equivalent, so they are `bm25_b`, `bm25_k1`, `index_null_state`, `index_property_length` and `index_timestamps`. -**`bm25_b`** - Controls the degree of normalization by document length in the BM25 ranking algorithm. Values range from 0 to 1, where 0 means no length normalization and 1 means full normalization. Higher values favor shorter documents. +**`bm25`: `b`** - Controls the degree of normalization by document length in the BM25 ranking algorithm. Values range from 0 to 1, where 0 means no length normalization and 1 means full normalization. Higher values favor shorter documents. -**`bm25_k1`** - Controls term frequency saturation in BM25. Higher values make term frequency more important, while lower values reduce the impact of term frequency on scoring. +**`bm25`: `k1`** - Controls term frequency saturation in BM25. Higher values make term frequency more important, while lower values reduce the impact of term frequency on scoring. -**`index_null_state`** - Determines whether null values are indexed. When enabled, you can filter for objects that have null values in specific properties. +**`indexNullState`** - Determines whether null values are indexed. When enabled, you can filter for objects that have null values in specific properties. -**`index_property_length`** - Controls whether the length of text properties is indexed. When enabled, allows filtering based on text length and can improve certain ranking algorithms. +**`indexPropertyLength`** - Controls whether the length of text properties is indexed. When enabled, allows filtering based on text length and can improve certain ranking algorithms. -**`index_timestamps`** - Enables indexing of creation and update timestamps for objects, allowing filtering and sorting operations. +**`indexTimestamps`** - Enables indexing of creation and update timestamps for objects, allowing filtering and sorting operations.
@@ -151,7 +151,7 @@ The inverted index in Weaviate can be configured through various parameters at t ## Drop an inverted index -Drop (delete) an inverted index from a property. This is a destructive operation — the index data is removed from disk. To use the index again, it must be regenerated. +Drop (delete) an inverted index from a property. This is a destructive operation: the index data is removed from disk. To use the index again, it must be regenerated. The following index types can be dropped: `searchable`, `filterable`, `rangeFilters`. @@ -201,7 +201,7 @@ Tokenization determines how text content is broken down into individual terms th **`word`** - The default tokenization that splits text on whitespace and punctuation, converting to lowercase. Best for general text search where you want to match individual words. -**`lowercase`** - Splits text on whitespace only, then lowercases each token. Preserves symbols (like `&`, `@`, `_`) that `word` tokenization would strip. Good for case-insensitive matching where punctuation is meaningful — e.g. code snippets or email addresses. +**`lowercase`** - Splits text on whitespace only, then lowercases each token. Preserves symbols (like `&`, `@`, `_`) that `word` tokenization would strip. Good for case-insensitive matching where punctuation is meaningful, such as code snippets or email addresses. **`whitespace`** - Splits text only on whitespace characters, preserving punctuation and case. Good when punctuation is meaningful for search. @@ -211,7 +211,7 @@ Tokenization determines how text content is broken down into individual terms th **`gse`** - Language-aware tokenization for Chinese and Japanese text. Disabled by default. Enable with the `ENABLE_TOKENIZER_GSE` environment variable. For Korean text, see the `kagome_kr` option. -For the full list of supported tokenizers — including `kagome_ja`, `kagome_kr`, and the per-property text-analyzer options — see the [tokenization reference](../config-refs/collections.mdx#tokenization). +For the full list of supported tokenizers (including `kagome_ja`, `kagome_kr`, and the per-property text-analyzer options), see the [tokenization reference](../config-refs/collections.mdx#tokenization).
diff --git a/docs/weaviate/manage-collections/multi-tenancy.mdx b/docs/weaviate/manage-collections/multi-tenancy.mdx index d0580fee3..7a0ba94e3 100644 --- a/docs/weaviate/manage-collections/multi-tenancy.mdx +++ b/docs/weaviate/manage-collections/multi-tenancy.mdx @@ -99,7 +99,7 @@ import AutoTenant from "/_includes/auto-tenant.mdx"; text={GoCodeAuto} startMarker="// START enable autoMT" endMarker="// END enable autoMT" - language="bash" + language="goraw" /> @@ -123,7 +123,7 @@ import AutoTenant from "/_includes/auto-tenant.mdx"; text={CurlCode} startMarker="# START CreateWithAMT" endMarker="# END CreateWithAMT" - language="py" + language="bash" /> @@ -154,7 +154,7 @@ Use the client to update the auto-tenant creation setting. Auto-tenant is only a text={GoCodeAuto} startMarker="// Start update autoMT" endMarker="// END update autoMT" - language="bash" + language="gonew" /> diff --git a/docs/weaviate/manage-objects/create.mdx b/docs/weaviate/manage-objects/create.mdx index ac9e4864e..445cf69b0 100644 --- a/docs/weaviate/manage-objects/create.mdx +++ b/docs/weaviate/manage-objects/create.mdx @@ -288,7 +288,7 @@ import CrossReferencePerformanceNote from "/_includes/cross-reference-performanc import XrefPyCode from "!!raw-loader!/_includes/code/howto/manage-data.cross-refs.py"; -import XrefTSCode from "!!raw-loader!/_includes/code/howto/manage-data.cross-refs"; +import XrefTSCode from "!!raw-loader!/_includes/code/howto/manage-data.cross-refs.ts"; import XrefJavaV6Code from "!!raw-loader!/_includes/code/java-v6/src/test/java/ManageCollectionsCrossReferencesTest.java"; You can create an object with cross-references to other objects. diff --git a/docs/weaviate/manage-objects/import.mdx b/docs/weaviate/manage-objects/import.mdx index 0fd7b1823..84944569a 100644 --- a/docs/weaviate/manage-objects/import.mdx +++ b/docs/weaviate/manage-objects/import.mdx @@ -9,15 +9,13 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; import PyCode from '!!raw-loader!/_includes/code/howto/manage-data.import.py'; -import PySuppCode from '!!raw-loader!/_includes/code/howto/sample-data.py'; import TSCode from '!!raw-loader!/_includes/code/howto/manage-data.import.ts'; -import TsSuppCode from '!!raw-loader!/_includes/code/howto/sample-data.ts'; import JavaV6Code from '!!raw-loader!/_includes/code/java-v6/src/test/java/ManageObjectsImportTest.java'; import CSharpCode from '!!raw-loader!/_includes/code/csharp/ManageObjectsImportTest.cs'; import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/manage-data.import_test.go'; import SkipLink from '/src/components/SkipValidationLink' -[Batch imports](../tutorials/import.mdx) are an efficient way to add multiple data objects and cross-references. For most use cases, we recommend **server-side batching** as the starting point: the server tells the client how much data to send next, so you don't have to tune batch parameters yourself. When you need manual control over the batch size and concurrency — or you are using a client that does not yet support server-side batching — use [manual batching](#manual-batching) instead. +[Batch imports](../tutorials/import.mdx) are an efficient way to add multiple data objects and cross-references. For most use cases, we recommend **server-side batching** as the starting point: the server tells the client how much data to send next, so you don't have to tune batch parameters yourself. When you need manual control over the batch size and concurrency, or you are using a client that does not yet support server-side batching, use [manual batching](#manual-batching) instead. ## Server-side batching @@ -25,20 +23,40 @@ import SsbStatus from '/_includes/feature-notes/ssb-status.mdx'; -With [server-side batch imports](../concepts/data-import.mdx#server-side-batching) (also called "automatic" batching), the client sends data in batch sizes determined by feedback from the server. This simplifies your code and helps the server manage its own load. The following example adds objects to a collection named `MyCollection`. +With [server-side batch imports](../concepts/data-import.mdx#server-side-batching) (also called "automatic" batching), the client sends data in batch sizes determined by feedback from the server. This simplifies your code and helps the server manage its own load. Server-side batching offers two entry points: -Server-side batching uses the [gRPC API](#use-the-grpc-api), which current client versions enable by default. +- **Stream from a data source** (recommended for large datasets): Add objects to the import one at a time as you read them from the source, so the full dataset never has to fit in memory. +- **[Ingest an in-memory list](#ingest-an-in-memory-list)**: Import a list of objects that you already hold in memory with a single call. + +Server-side batching uses the [gRPC API](../api/index.mdx), which current client versions enable by default. + +The following example adds objects to a collection named `MyCollection`. + +Open the `batch.stream()` context manager and add objects one at a time; the client sends them at the pace the server requests. The [async Python client](../client-libraries/python/async.md#bulk-data-insertion) also supports server-side batching through the `stream()` method and the one-shot `ingest()` method. + + +You can also stream from a data source with `data.ingest()`. It accepts any iterable, so you can pass a generator that reads a source file record by record. Objects go to the server as the generator produces them, so the source never has to fit in memory. To import objects that you already hold in a list, see [Ingest an in-memory list](#ingest-an-in-memory-list). + + + +In TypeScript, `data.ingest()` is the server-side batching API, with no separate streaming context. It accepts any `Iterable`, so passing a generator streams objects to the server without building the full list in memory. + + +Open a streaming context with `collection.batch.start()` and add objects one at a time. The batch is flushed and closed automatically when the try-with-resources block exits. + + +Open a streaming batch with `collection.Batch.StartBatch()` and add objects one at a time with `Add`. Call `Close` to flush the batch. + -## Manual batching +### Ingest an in-memory list -Use manual (client-side) batching when you want to control the batch size and concurrency yourself, or when using a client that does not yet support server-side batching (such as the Go client). The following example adds objects to the `MyCollection` collection. +If your objects are already in memory, you can import the whole list with a single call. The client sends the list using server-side batching, so the import is safe for large lists that would exceed the server's [`GRPC_MAX_MESSAGE_SIZE`](/deploy/configuration/env-vars/index.md#GRPC_MAX_MESSAGE_SIZE) limit if sent as one request. -
- Additional information + + -To create a bulk import job manually, follow these steps: +`data.ingest()` is the safe replacement for passing a large list to `insert_many`, which sends all objects in a single request. `ingest` accepts plain property dicts or `DataObject` instances (to set object IDs, vectors, or references) and returns the same return object as `insert_many`. -1. Initialize a batch object. -1. Add items to the batch object. -1. Ensure that the last batch is sent (flushed). + + + + +When your objects are already in an array, pass the array directly to `data.ingest()` to import the whole list in a single call. + + + + + +The Go client does not support server-side batching; use [manual batching](#manual-batching) instead. + + + + +The Java client does not provide a one-shot ingest method. Use the streaming context `collection.batch.start()` shown in the [server-side batching example](#server-side-batching) above. Note that `collection.data.insertMany(...)` sends all objects in a single request and does not use server-side batching. + + + -
+In the C# client, `Batch.InsertMany` uses server-side batching under the hood. + + +
+ + +## Manual batching + +Use manual (client-side) batching when you want to control the batch size and concurrency yourself, or when using a client that does not yet support server-side batching (such as the Go client). The following example adds objects to the `MyCollection` collection. @@ -92,17 +155,6 @@ To create a bulk import job manually, follow these steps: endMarker="# END BasicBatchImportExample" language="py" /> - -### Error handling - - - -During a batch import, any failed objects or references will be stored and can be obtained through `batch.failed_objects` and `batch.failed_references`. -Additionally, a running count of failed objects and references is maintained and can be accessed through `batch.number_errors` within the context manager. -This counter can be used to stop the import process in order to investigate the failed objects or references. - -Find out more about error handling on the Python client [reference page](/weaviate/client-libraries/python). - + +Configure the Go client's gRPC connection parameters as described on the [connection configuration](../connections/connect-custom.mdx) page. + -## Use the gRPC API +## Error handling + + -The [gRPC API](../api/index.mdx) is faster than the REST API. Use the gRPC API to improve import speeds. +Batch imports report failures per object: a problem with one object does not abort the rest of the import. Errors are reported the same way in server-side and manual batching. Inspect the failed items during and after the import to catch data issues early. - - - - + -The Python client uses gRPC by default. +- Within a batching context manager, `batch.number_errors` holds a running count of failed objects and references. You can use this counter to stop the import process and investigate the failures. +- After the context closes, `collection.batch.failed_objects` and `collection.batch.failed_references` contain the failed items. +- The one-shot `data.ingest()` method returns the same result object as `insert_many`: its `errors` dict maps the original index of each failed object to its error. -
-The legacy Python client does not support gRPC. +Find out more about error handling on the Python client [reference page](/weaviate/client-libraries/python). -
+
-The TypeScript client v3 uses gRPC by default. - -
-The legacy TypeScript client does not support gRPC. +`data.ingest()` returns a result object. Inspect its `errors` field for the objects that failed to import. -
- - -The Java client v6 uses gRPC by default. - -To use the gRPC API with the Go client, add the `GrpcConfig` field to your client connection code. Update `Secured` if you use an encrypted connection.

- -```go -cfg := weaviate.Config{ - Host: fmt.Sprintf("localhost:%v", "8080"), - Scheme: "http", - // highlight-start - GrpcConfig: &grpc.Config{ - Host: "localhost:50051", - Secured: false, - }, - // highlight-end -} - -client, err := weaviate.NewClient(cfg) -if err != nil { - require.Nil(t, err) - } -``` +Inspect the per-object errors on the result returned by the batcher.
- + -The C# uses gRPC by default. - - - +Within a `batch.start()` streaming context, `batch.numberOfErrors()` reports the number of objects that could not be imported. The response returned by `insertMany` exposes the failed objects through its `errors()` method. -To use the gRPC API with the [Spark connector](https://github.com/weaviate/spark-connector), add the `grpc:host` field to your client connection code. Update `grpc:secured` if you use an encrypted connection.

+
+ -```java - df.write - .format("io.weaviate.spark.Weaviate") - .option("scheme", "http") - .option("host", "localhost:8080") - // highlight-start - .option("grpc:host", "localhost:50051") - .option("grpc:secured", "false") - // highlight-start - .option("className", className) - .mode("append") - .save() -``` +The response returned by `Batch.InsertMany` is a collection of per-object entries. Filter for entries where `Error` is not null to find the failed objects. With `Batch.StartBatch()`, each `Add` returns a handle whose result reports whether the object succeeded.
-## Specify an ID value +## Customize imported objects + +Batch-imported objects support the same parameters as individually created objects, such as custom IDs, vectors, and cross-references. These parameters work the same way in server-side and manual batching. + +### Specify an ID value Weaviate generates an UUID for each object. Object IDs must be unique. If you set object IDs, use one of these deterministic UUID methods to prevent duplicate IDs: @@ -266,7 +285,7 @@ Weaviate generates an UUID for each object. Object IDs must be unique. If you se -## Specify a vector +### Specify a vector Use the `vector` property to specify a vector for each object. @@ -313,7 +332,7 @@ Use the `vector` property to specify a vector for each object. -## Specify named vectors +### Specify named vectors When you create an object, you can specify named vectors (if [configured in your collection](../manage-collections/vector-config.mdx#define-named-vectors)). @@ -352,15 +371,15 @@ When you create an object, you can specify named vectors (if [configured in your -## Import with references +### Import with references -You can batch create links from an object to another other object through cross-references. +You can batch create links from an object to another object through cross-references. @@ -383,142 +402,15 @@ You can batch create links from an object to another other object through cross- -## Python-specific considerations - -The Python clients have built-in batching methods to help you optimize import speed. For details, see the client documentation: - - -- [Python client](../client-libraries/python/notes-best-practices.mdx) - -### Async Python client and batching - -The [async Python client](../client-libraries/python/async.md#bulk-data-insertion) supports server-side batching through the `stream()` method. For client-side batching, use the sync Python client. - ## Stream data from large files -If your dataset is large, consider streaming the import to avoid out-of-memory issues. +If your dataset does not fit in memory, do not load it all at once. Instead, read the source file lazily and add objects to the import as you go: -To try the example code, download the sample data and create the sample input files. +- With the [server-side streaming context](#server-side-batching), add each object as you read it from the file. The client sends data at the pace the server requests, so memory usage stays flat. +- In Python and TypeScript, the [one-shot import method](#server-side-batching) accepts any iterable, so you can pass a lazy source, such as a generator that reads the file record by record, instead of a fully loaded list. +- With manual batching, apply the same pattern: add objects to the batch as you read them. -
- Get the sample data - - - - - - - - - - - - - - - - -
- -
- Stream JSON files example code - - - - - - - - - - - - - - - - -
- -
- Stream CSV files example code - - - - - - - - - - - - - - - - -
+For JSON files, use a streaming parser that yields one object at a time (such as `ijson` in Python). For CSV files, read the file in chunks (such as `pandas` with the `chunksize` parameter) rather than loading it whole. ## Batch vectorization @@ -541,22 +433,6 @@ Note that each provider exposes different configuration options. language="py" /> - - - - - - ## Additional considerations @@ -565,22 +441,7 @@ Data imports can be resource intensive. Consider the following when you import l ### Asynchronous imports -:::caution Experimental -Available starting in `v1.22`. This is an experimental feature. Use with caution. -::: - -To maximize import speed, enable [asynchronous indexing](/weaviate/config-refs/indexing/vector-index.mdx#asynchronous-indexing). - -To enable asynchronous indexing, set the `ASYNC_INDEXING` environment variable to `true` in your Weaviate configuration file. - -```yaml -weaviate: - image: cr.weaviate.io/semitechnologies/weaviate:||site.weaviate_version|| - ... - environment: - ASYNC_INDEXING: 'true' - ... -``` +To maximize import speed, enable [asynchronous indexing](/weaviate/config-refs/indexing/vector-index.mdx#asynchronous-indexing) by setting the `ASYNC_INDEXING` environment variable to `true` in your Weaviate configuration. This decouples vector index construction from object creation, so imports are not slowed down by index building. ### Automatically add new tenants @@ -594,6 +455,8 @@ For details, see [auto-tenant](/weaviate/manage-collections/multi-tenancy#automa - [Connect to Weaviate](/weaviate/connections/index.mdx) - [How-to: Create objects](./create.mdx) +- [Python client: batch import notes and best practices](../client-libraries/python/notes-best-practices.mdx) +- [Blog: Data import best practices](https://weaviate.io/blog/data-import-best-practices) - References: REST - /v1/batch diff --git a/docs/weaviate/manage-objects/read-all-objects.mdx b/docs/weaviate/manage-objects/read-all-objects.mdx index 87b0f61ff..94c01d270 100644 --- a/docs/weaviate/manage-objects/read-all-objects.mdx +++ b/docs/weaviate/manage-objects/read-all-objects.mdx @@ -20,7 +20,7 @@ Weaviate provides the necessary APIs to iterate through all your data. This is u This is done with the help of the `after` operator, also called the [cursor API](../api/graphql/additional-operators.md#cursor-with-after). :::info Iterator -The new API clients (currently supported by the Python Client v4), encapsulate this functionality as an `Iterator`. +Some clients, such as the Python client, encapsulate this functionality as an `Iterator`. ::: ## Read object properties and ids @@ -47,8 +47,8 @@ The following code iterates through all objects, providing the properties and id @@ -95,14 +95,6 @@ Read through all data including the vectors. (Also applicable where [named vecto language="ts" /> - - - - - - ', }), // highlight-end }); diff --git a/docs/weaviate/model-providers/_includes/provider.vectorizer.py b/docs/weaviate/model-providers/_includes/provider.vectorizer.py index c928a9968..bc0b2400d 100644 --- a/docs/weaviate/model-providers/_includes/provider.vectorizer.py +++ b/docs/weaviate/model-providers/_includes/provider.vectorizer.py @@ -25,7 +25,7 @@ region="us-east-1", source_properties=["title"], service="bedrock", - model="titan-embed-text-v2:0", + model="amazon.titan-embed-text-v2:0", ) ], # highlight-end @@ -62,17 +62,40 @@ # START FullVectorizerAWS from weaviate.classes.config import Configure +# For Bedrock client.collections.create( "DemoCollection", # highlight-start vector_config=[ - Configure.Vectors.text2vec_aws( + Configure.Vectors.text2vec_aws_bedrock( name="title_vector", region="us-east-1", source_properties=["title"], - service="bedrock", # `bedrock` or `sagemaker` - model="titan-embed-text-v2:0", # If using `bedrock`, this is required - # endpoint="", # If using `sagemaker`, this is required + model="amazon.titan-embed-text-v2:0", # Required + # Further options + # dimensions=512, # Amazon models only + ) + ], + # highlight-end + # Additional parameters not shown +) + +# clean up +client.collections.delete("DemoCollection") + +# For SageMaker +client.collections.create( + "DemoCollection", + # highlight-start + vector_config=[ + Configure.Vectors.text2vec_aws_sagemaker( + name="title_vector", + region="us-east-1", + source_properties=["title"], + endpoint="", # Required + # Further options + # target_model="", + # target_variant="", ) ], # highlight-end @@ -294,12 +317,13 @@ client.collections.create( "DemoCollection", # highlight-start - vector_config=Configure.Vectors.text2vec_google( + vector_config=Configure.Vectors.text2vec_google_vertex( name="title_vector", source_properties=["title"], project_id="", # Required for Vertex AI # Further options # model="", + # location="", # api_endpoint="", ), # highlight-end @@ -317,7 +341,7 @@ name="title_vector", source_properties=["title"], # Further options - model_id="gemini-embedding-2", + model="gemini-embedding-2", ), # highlight-end # Additional parameters not shown @@ -422,10 +446,9 @@ Configure.Vectors.text2vec_huggingface( name="title_vector", source_properties=["title"], - # NOTE: Use only one of (`model`), (`passage_model` and `query_model`), or (`endpoint_url`) + # NOTE: Use only one of (`model`), (`passage_model`), or (`endpoint_url`) model="sentence-transformers/all-MiniLM-L6-v2", - # passage_model="sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base", # Required if using `query_model` - # query_model="sentence-transformers/facebook-dpr-question_encoder-single-nq-base", # Required if using `passage_model` + # passage_model="sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base", # endpoint_url="", # # wait_for_model=True, @@ -690,7 +713,9 @@ Configure.Vectors.text2vec_mistral( name="title_vector", source_properties=["title"], - model="mistral-embed" + model="mistral-embed", + # Further options + # base_url="", ) ], # highlight-end @@ -967,7 +992,7 @@ # highlight-start vector_config=[ Configure.Vectors.text2vec_digitalocean( - model="qwen3-embedding-0.6b", # Required — choose from the DigitalOcean Serverless Inference catalogue + model="qwen3-embedding-0.6b", # Required. Choose from the DigitalOcean Serverless Inference catalogue name="title_vector", source_properties=["title"], ) diff --git a/docs/weaviate/model-providers/_includes/provider.vectorizer.ts b/docs/weaviate/model-providers/_includes/provider.vectorizer.ts index 26b482a94..11f5bef7a 100644 --- a/docs/weaviate/model-providers/_includes/provider.vectorizer.ts +++ b/docs/weaviate/model-providers/_includes/provider.vectorizer.ts @@ -327,7 +327,7 @@ await client.collections.create({ // highlight-start vectorizers: [ weaviate.configure.vectors.text2VecDigitalOcean({ - model: 'qwen3-embedding-0.6b', // Required — choose from the DigitalOcean Serverless Inference catalogue + model: 'qwen3-embedding-0.6b', // Required. Choose from the DigitalOcean Serverless Inference catalogue name: 'title_vector', sourceProperties: ['title'], }) @@ -378,7 +378,7 @@ await client.collections.create({ ], // highlight-start vectorizers: [ - weaviate.configure.vectors.text2VecGoogle({ + weaviate.configure.vectors.text2VecGoogleGemini({ name: 'title_vector', sourceProperties: ['title'], // (Optional) To manually set the model ID @@ -547,10 +547,10 @@ await client.collections.create({ weaviate.configure.vectors.text2VecHuggingFace({ name: 'title_vector', sourceProperties: ['title'], + // NOTE: Use only one of `model`, `passageModel`, or `endpointURL` model: 'sentence-transformers/all-MiniLM-L6-v2', // endpointURL: , - // passageModel: 'sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base', // Required if using `query_model` - // queryModel: 'sentence-transformers/facebook-dpr-question_encoder-single-nq-base', // Required if using `passage_model` + // passageModel: 'sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base', // waitForModel: true, // useCache: true, // useGPU: true, @@ -838,7 +838,9 @@ await client.collections.create({ weaviate.configure.vectors.text2VecMistral({ name: 'title_vector', sourceProperties: ['title'], - model: 'mistral-embed' + model: 'mistral-embed', + // Further options + // baseURL: '', }, ), ], diff --git a/docs/weaviate/model-providers/anthropic/generative.md b/docs/weaviate/model-providers/anthropic/generative.md index 37a0ef79b..889be8b1f 100644 --- a/docs/weaviate/model-providers/anthropic/generative.md +++ b/docs/weaviate/model-providers/anthropic/generative.md @@ -51,7 +51,7 @@ You must provide a valid Anthropic API key to Weaviate for this integration. Go Provide the API key to Weaviate using one of the following methods: -- Set the `ANTHROPIC_API_KEY` environment variable that is available to Weaviate. +- Set the `ANTHROPIC_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. @@ -301,12 +301,7 @@ The default base URL is `https://api.anthropic.com`. ### Available models -Any model available in the Anthropic API can be used with Weaviate. As of July 2024, the following models are available: - -- `claude-3-5-sonnet-20240620` (default) -- `claude-3-opus-20240229` -- `claude-3-sonnet-20240229` -- `claude-3-haiku-20240307` +Any model available in the Anthropic API can be used with Weaviate. If you do not specify a model, Weaviate uses `claude-haiku-4-5` by default. That default was set in `v1.34.0`, and backported to `v1.31.20`, `v1.32.17`, and `v1.33.5`. Earlier releases on each of those lines default to `claude-3-5-sonnet-20240620`. See the [Anthropic API documentation](https://docs.anthropic.com/en/docs/about-claude/models#model-names) for the most up-to-date list of available models. diff --git a/docs/weaviate/model-providers/anyscale/generative.md b/docs/weaviate/model-providers/anyscale/generative.md index 142064a97..af46de106 100644 --- a/docs/weaviate/model-providers/anyscale/generative.md +++ b/docs/weaviate/model-providers/anyscale/generative.md @@ -51,7 +51,7 @@ You must provide a valid Anyscale API key to Weaviate for this integration. Go t Provide the API key to Weaviate using one of the following methods: -- Set the `ANYSCALE_API_KEY` environment variable that is available to Weaviate. +- Set the `ANYSCALE_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. diff --git a/docs/weaviate/model-providers/aws/embeddings.md b/docs/weaviate/model-providers/aws/embeddings.md index 3fafe5b4d..49fec0529 100644 --- a/docs/weaviate/model-providers/aws/embeddings.md +++ b/docs/weaviate/model-providers/aws/embeddings.md @@ -185,12 +185,34 @@ import VectorizationBehavior from '/_includes/vectorization.behavior.mdx'; ### Vectorizer parameters -The following examples show how to configure AWS-specific options. +**Common parameters:** +- `service` (Optional): The AWS service to use, either `bedrock` or `sagemaker`. Defaults to `bedrock`. +- `region` (Required): The AWS region to send requests to, e.g. `us-east-1`. -The AWS region setting is required for all AWS integrations. +**Bedrock parameters:** +- `model` (Required): The full Bedrock model identifier, e.g. `amazon.titan-embed-text-v2:0`. +- `dimensions` (Optional): The size of the embedding to request from the model, e.g. `512`. -- Bedrock users must set `service` to `bedrock` and provide the `model` name. -- SageMaker users must set `service` to `sagemaker` and provide the `endpoint` address. +**SageMaker parameters:** +- `endpoint` (Required): The name of the SageMaker endpoint to invoke, e.g. `tei-xxx`. +- `targetModel` (Optional): The model to target on a multi-model endpoint. +- `targetVariant` (Optional): The production variant to target on the endpoint. + +#### `service` + +Weaviate falls back to `bedrock` when `service` is not set, so a SageMaker configuration has to set it to `sagemaker`. Any value other than `bedrock` or `sagemaker` is rejected when the collection is created. + +Some clients offer service-specific constructors, such as `text2vec_aws_bedrock` and `text2vec_aws_sagemaker`, which select the service for you. Examples built on those constructors do not pass `service` at all. Every other example on this page sets it explicitly, as does any configuration written directly against the collection definition. + +#### `dimensions` + +`dimensions` was added in `v1.36.19`, and is also available from `v1.37.10`, `v1.38.2`, and `v1.39.0` onward. + +Weaviate only forwards `dimensions` to Amazon models on Bedrock, such as the Titan and Nova embedding families. Cohere models on Bedrock, and SageMaker endpoints, accept the setting in the collection configuration but ignore it when embeddings are generated, so their vectors keep the model's default size. Check the model's documentation for the sizes it supports. + +#### Example configuration + +The following examples show how to configure AWS-specific options for each service. diff --git a/docs/weaviate/model-providers/aws/generative.md b/docs/weaviate/model-providers/aws/generative.md index df4d028e2..515bdea8c 100644 --- a/docs/weaviate/model-providers/aws/generative.md +++ b/docs/weaviate/model-providers/aws/generative.md @@ -147,7 +147,7 @@ For SageMaker, you must provide the endpoint address in the generative AI config -You can [specify](#generative-parameters) one of the [available models](#available-models) for Weaviate to use. The [default model](#available-models) is used if no model is specified. +You can [specify](#generative-parameters) which [model](#available-models) Weaviate uses. ### Generative parameters @@ -272,31 +272,11 @@ You can also supply images as a part of the input when performing retrieval augm #### Bedrock -- `ai21.j2-ultra-v1` -- `ai21.j2-mid-v1` -- `amazon.titan-text-lite-v1` -- `amazon.titan-text-express-v1` -- `amazon.titan-text-premier-v1:0` -- `anthropic.claude-v2` -- `anthropic.claude-v2:1` -- `anthropic.claude-instant-v1` -- `anthropic.claude-3-sonnet-20240229-v1:0` -- `anthropic.claude-3-haiku-20240307-v1:0` -- `cohere.command-text-v14` -- `cohere.command-light-text-v14` -- `cohere.command-r-v1:0` -- `cohere.command-r-plus-v1:0` -- `meta.llama3-8b-instruct-v1:0` -- `meta.llama3-70b-instruct-v1:0` -- `meta.llama2-13b-chat-v1` -- `meta.llama2-70b-chat-v1` -- `mistral.mistral-7b-instruct-v0:2` -- `mistral.mixtral-8x7b-instruct-v0:1` -- `mistral.mistral-large-2402-v1:0` - -Refer to the [this document](https://docs.aws.amazon.com/bedrock/latest/userguide/model-usage.html) to find out how request access to a model. +Weaviate passes the `model` value through to Amazon Bedrock, so any Bedrock text generation model that your AWS account and region has access to can be used. Weaviate recognizes the model families offered by AI21 Labs, Amazon (Titan and Nova), Anthropic, Cohere, Meta, and Mistral AI, including their cross-region inference profile IDs. -### SageMaker +For the current model IDs, see the [Amazon Bedrock supported foundation models](https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html) documentation. Refer to [this document](https://docs.aws.amazon.com/bedrock/latest/userguide/model-usage.html) to find out how to request access to a model. + +#### SageMaker Any custom SageMaker URL can be used as an endpoint. diff --git a/docs/weaviate/model-providers/cohere/embeddings-multimodal.md b/docs/weaviate/model-providers/cohere/embeddings-multimodal.md index 9c2b723c5..e7126e2d0 100644 --- a/docs/weaviate/model-providers/cohere/embeddings-multimodal.md +++ b/docs/weaviate/model-providers/cohere/embeddings-multimodal.md @@ -51,7 +51,7 @@ You must provide a valid Cohere API key to Weaviate for this integration. Go to Provide the API key to Weaviate using one of the following methods: -- Set the `COHERE_API_KEY` environment variable that is available to Weaviate. +- Set the `COHERE_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. @@ -294,7 +294,8 @@ The query below returns the `n` most similar objects to the input image from the ### Available models -- `embed-multilingual-v3.0` (Default) +- `embed-v4.0` +- `embed-multilingual-v3.0` (server default) - `embed-multilingual-light-v3.0` - `embed-english-v3.0` - `embed-english-light-v3.0` diff --git a/docs/weaviate/model-providers/cohere/embeddings.md b/docs/weaviate/model-providers/cohere/embeddings.md index e3000c344..6464cb67f 100644 --- a/docs/weaviate/model-providers/cohere/embeddings.md +++ b/docs/weaviate/model-providers/cohere/embeddings.md @@ -54,7 +54,7 @@ You must provide a valid Cohere API key to Weaviate for this integration. Go to Provide the API key to Weaviate using one of the following methods: -- Set the `COHERE_API_KEY` environment variable that is available to Weaviate. +- Set the `COHERE_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. @@ -356,7 +356,7 @@ The query below returns the `n` best scoring objects from the database, set by ` ### Available models - `embed-v4.0` -- `embed-multilingual-v3.0` (Default) +- `embed-multilingual-v3.0` (server default) - `embed-multilingual-light-v3.0` - `embed-multilingual-v2.0` (previously `embed-multilingual-22-12`) - `embed-english-v3.0` @@ -379,7 +379,7 @@ The following models are available, but deprecated: ### Other integrations -- [Cohere multimodal embedding embeddings models + Weaviate](./embeddings-multimodal.md) +- [Cohere multimodal embedding models + Weaviate](./embeddings-multimodal.md) - [Cohere generative models + Weaviate](./generative.md) - [Cohere reranker models + Weaviate](./reranker.md) diff --git a/docs/weaviate/model-providers/cohere/generative.md b/docs/weaviate/model-providers/cohere/generative.md index d04059380..15ba44867 100644 --- a/docs/weaviate/model-providers/cohere/generative.md +++ b/docs/weaviate/model-providers/cohere/generative.md @@ -52,7 +52,7 @@ You must provide a valid Cohere API key to Weaviate for this integration. Go to Provide the API key to Weaviate using one of the following methods: -- Set the `COHERE_API_KEY` environment variable that is available to Weaviate. +- Set the `COHERE_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. @@ -264,8 +264,15 @@ In other words, when you have `n` search results, the generative model generates ### Available models +Weaviate does not validate the model name, so you can set any model that your Cohere account can reach. Name validation was removed in `v1.33.0`, and backported to `v1.31.17` and `v1.32.10`. + +The server default is `command-a-03-2025`. It changed in `v1.33.0`, and was backported to `v1.31.17` and `v1.32.10`. Earlier releases on each of those lines default to `command-r`. + +The following models are commonly used: + +- `command-a-03-2025` (server default) - `command-r-plus` -- `command-r` (default) +- `command-r` (previous server default) - `command-xlarge` - `command-xlarge-beta` - `command-xlarge-nightly` @@ -281,7 +288,7 @@ In other words, when you have `n` search results, the generative model generates ### Other integrations - [Cohere text embedding models + Weaviate](./embeddings.md). -- [Cohere multimodal embedding embeddings models + Weaviate](./embeddings-multimodal.md) +- [Cohere multimodal embedding models + Weaviate](./embeddings-multimodal.md). - [Cohere reranker models + Weaviate](./reranker.md). ### Code examples diff --git a/docs/weaviate/model-providers/cohere/reranker.md b/docs/weaviate/model-providers/cohere/reranker.md index dcfccc4d4..5f3565b52 100644 --- a/docs/weaviate/model-providers/cohere/reranker.md +++ b/docs/weaviate/model-providers/cohere/reranker.md @@ -52,7 +52,7 @@ You must provide a valid Cohere API key to Weaviate for this integration. Go to Provide the API key to Weaviate using one of the following methods: -- Set the `COHERE_API_KEY` environment variable that is available to Weaviate. +- Set the `COHERE_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. @@ -180,7 +180,7 @@ Any search in Weaviate can be combined with a reranker to perform reranking oper ### Available models -- `rerank-v3.5` (default) +- `rerank-v3.5` (server default) - `rerank-english-v3.0` - `rerank-multilingual-v3.0` - `rerank-english-v2.0` @@ -199,7 +199,7 @@ For further details on model parameters, see the [Cohere API documentation](http ### Other integrations - [Cohere text embedding models + Weaviate](./embeddings.md). -- [Cohere multimodal embedding embeddings models + Weaviate](./embeddings-multimodal.md) +- [Cohere multimodal embedding models + Weaviate](./embeddings-multimodal.md). - [Cohere generative models + Weaviate](./generative.md). ### Code examples diff --git a/docs/weaviate/model-providers/contextualai/generative.md b/docs/weaviate/model-providers/contextualai/generative.md index dd591044a..654f1030d 100644 --- a/docs/weaviate/model-providers/contextualai/generative.md +++ b/docs/weaviate/model-providers/contextualai/generative.md @@ -55,7 +55,7 @@ You must provide a valid Contextual AI API key to Weaviate for this integration. Provide the API key to Weaviate using one of the following methods: -- Set the `CONTEXTUAL_API_KEY` environment variable that is available to Weaviate. +- Set the `CONTEXTUALAI_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. @@ -163,7 +163,7 @@ Configure the following generative parameters to customize the model behavior. For further details on model parameters, see the [Contextual AI API documentation](https://docs.contextual.ai/api-reference/generate/generate). -If a parameter is not specified, Weaviate uses the server-side default for that parameter. They are: +If a parameter is not specified, Weaviate uses the server default for that parameter. They are: - model = `"v2"` - temperature = `0.0` diff --git a/docs/weaviate/model-providers/contextualai/reranker.md b/docs/weaviate/model-providers/contextualai/reranker.md index dfed9ddc4..bf4224f23 100644 --- a/docs/weaviate/model-providers/contextualai/reranker.md +++ b/docs/weaviate/model-providers/contextualai/reranker.md @@ -55,7 +55,7 @@ You must provide a valid Contextual AI API key to Weaviate for this integration. Provide the API key to Weaviate using one of the following methods: -- Set the `CONTEXTUAL_API_KEY` environment variable that is available to Weaviate. +- Set the `CONTEXTUALAI_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. diff --git a/docs/weaviate/model-providers/databricks/embeddings.md b/docs/weaviate/model-providers/databricks/embeddings.md index 6207c2041..7060f1b45 100644 --- a/docs/weaviate/model-providers/databricks/embeddings.md +++ b/docs/weaviate/model-providers/databricks/embeddings.md @@ -50,7 +50,7 @@ This integration is enabled by default on Weaviate Cloud (WCD) instances. You must provide a valid Databricks Personal Access Token (PAT) to Weaviate for this integration. Refer to the [Databricks documentation](https://docs.databricks.com/en/dev-tools/auth/pat.html) for instructions on generating your PAT in your workspace. -Provide the Dataricks token to Weaviate using one of the following methods: +Provide the Databricks token to Weaviate using one of the following methods: - Set the `DATABRICKS_TOKEN` environment variable that is available to Weaviate. - Provide the token at runtime, as shown in the examples below. @@ -125,13 +125,13 @@ This will configure Weaviate to use the vectorizer served through the endpoint y ### Vectorizer parameters - `endpoint`: The URL of the embedding model hosted on Databricks. -- `instruction`:An optional instruction to pass to the embedding model. +- `instruction`: An optional instruction to pass to the embedding model. For further details on model parameters, see the [Databricks documentation](https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html#embedding-request). ## Header parameters -You can provide the API key as well as some optional parameters at runtime through additional headers in the request. The following headers are available: +You can provide the token as well as some optional parameters at runtime through additional headers in the request. The following headers are available: - `X-Databricks-Token`: The Databricks API token. - `X-Databricks-Endpoint`: The endpoint to use for the Databricks model. diff --git a/docs/weaviate/model-providers/databricks/generative.md b/docs/weaviate/model-providers/databricks/generative.md index 8f9a52780..22c953e60 100644 --- a/docs/weaviate/model-providers/databricks/generative.md +++ b/docs/weaviate/model-providers/databricks/generative.md @@ -48,10 +48,10 @@ This integration is enabled by default on Weaviate Cloud (WCD) instances. You must provide a valid Databricks Personal Access Token (PAT) to Weaviate for this integration. Refer to the [Databricks documentation](https://docs.databricks.com/en/dev-tools/auth/pat.html) for instructions on generating your PAT in your workspace. -Provide the Dataricks token to Weaviate using one of the following methods: +Provide the Databricks token to Weaviate using one of the following methods: - Set the `DATABRICKS_TOKEN` environment variable that is available to Weaviate. -- Provide the API key at runtime, as shown in the examples below. +- Provide the token at runtime, as shown in the examples below. @@ -156,7 +156,7 @@ Aside from setting the default model provider when creating the collection, you ## Header parameters -You can provide the API key as well as some optional parameters at runtime through additional headers in the request. The following headers are available: +You can provide the token as well as some optional parameters at runtime through additional headers in the request. The following headers are available: - `X-Databricks-Token`: The Databricks API token. - `X-Databricks-Endpoint`: The endpoint to use for the Databricks model. diff --git a/docs/weaviate/model-providers/databricks/index.md b/docs/weaviate/model-providers/databricks/index.md index 15feb7118..7fff6036d 100644 --- a/docs/weaviate/model-providers/databricks/index.md +++ b/docs/weaviate/model-providers/databricks/index.md @@ -25,11 +25,11 @@ Databricks' embedding models transform text data into vector embeddings, capturi ### Generative AI models for RAG -![Single prompt RAG integration generates individual outputs per search result](../_includes/integration_openai_rag_single.png) +![Single prompt RAG integration generates individual outputs per search result](../_includes/integration_databricks_rag_single.png) -Databrick' generative AI models can generate human-like text based on given prompts and contexts. +Databricks' generative AI models can generate human-like text based on given prompts and contexts. -[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly from the Weaviate Database. This combines Weaviate' efficient storage and fast retrieval capabilities with Databrick' generative AI models to generate personalized and context-aware responses. +[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly from the Weaviate Database. This combines Weaviate's efficient storage and fast retrieval capabilities with Databricks' generative AI models to generate personalized and context-aware responses. [Databricks generative AI integration page](./generative.md) @@ -43,7 +43,7 @@ In turn, they simplify the process of building AI-driven applications to speed u You must provide a valid Databricks personal access token to Weaviate for these integrations. Refer to the [Databricks documentation](https://docs.databricks.com/en/dev-tools/auth/pat.html) for instructions on generating your personal access token in your workspace. -Then, go to the relevant integration page to learn how to configure Weaviate with the OpenAI models and start using them in your applications. +Then, go to the relevant integration page to learn how to configure Weaviate with the Databricks models and start using them in your applications. - [Text Embeddings](./embeddings.md) - [Generative AI](./generative.md) diff --git a/docs/weaviate/model-providers/digitalocean/embeddings.md b/docs/weaviate/model-providers/digitalocean/embeddings.md index 256f70f8e..403d011b9 100644 --- a/docs/weaviate/model-providers/digitalocean/embeddings.md +++ b/docs/weaviate/model-providers/digitalocean/embeddings.md @@ -164,7 +164,7 @@ When you perform a [hybrid search](../../search/hybrid.md), Weaviate fuses keywo ### Available models -DigitalOcean's Serverless Inference catalogue includes several embedding-capable models. See the [DigitalOcean Serverless Inference docs](https://docs.digitalocean.com/products/inference/how-to/use-serverless-inference/) for the live list — model availability and dimensions can change. +DigitalOcean's Serverless Inference catalogue includes several embedding-capable models. See the [DigitalOcean Serverless Inference docs](https://docs.digitalocean.com/products/inference/how-to/use-serverless-inference/) for the live list, as model availability and dimensions can change. :::note Dimensions parameter currently not supported @@ -180,7 +180,7 @@ DigitalOcean's `/v1/embeddings` endpoint does not accept a `dimensions` request ### Code examples -Once the vectorizer is configured, Weaviate handles model inference transparently. The standard [client library how-tos](../../client-libraries/index.mdx) apply unchanged — no DigitalOcean-specific code is required at query or import time beyond the configuration shown above. +Once the vectorizer is configured, Weaviate handles model inference transparently. The standard [client library how-tos](../../client-libraries/index.mdx) apply unchanged. No DigitalOcean-specific code is required at query or import time beyond the configuration shown above. ## Questions and feedback diff --git a/docs/weaviate/model-providers/friendliai/_category_.json b/docs/weaviate/model-providers/friendliai/_category_.json index c00e1815e..7aa7693ab 100644 --- a/docs/weaviate/model-providers/friendliai/_category_.json +++ b/docs/weaviate/model-providers/friendliai/_category_.json @@ -1,4 +1,4 @@ { "label": "FriendliAI", - "position": 226 + "position": 227 } diff --git a/docs/weaviate/model-providers/google/embeddings-multimodal.md b/docs/weaviate/model-providers/google/embeddings-multimodal.md index 7a4a6fa32..efff5c8d9 100644 --- a/docs/weaviate/model-providers/google/embeddings-multimodal.md +++ b/docs/weaviate/model-providers/google/embeddings-multimodal.md @@ -144,9 +144,7 @@ import ApiKeyNote from '../_includes/google-api-key-note.md'; -You can [specify](#vectorizer-parameters) one of the [available models](#available-models) for the vectorizer to use. - - +You can [specify](#vectorizer-parameters) one of the [available models](#available-models) for the vectorizer to use. The [default model](#available-models) is used if no model is specified. import VectorizationBehavior from '/_includes/vectorization.behavior.mdx'; @@ -326,8 +324,8 @@ The query below returns the `n` most similar objects to the input image from the ### Available models -- `gemini-embedding-2` (Vertex AI and Gemini API, added in 1.36.13) — Vertex AI and Gemini API; supports text, images, PDFs, and audio (Gemini API only, up to 180 seconds); `3072` dimensions -- `multimodalembedding@001` (Vertex AI only) — supports text, images, and video; dimensions: `128`, `256`, `512`, `1408` +- `gemini-embedding-2` (Vertex AI and Gemini API, added in 1.36.13). Supports text, images, PDFs, and audio (Gemini API only, up to 180 seconds); `3072` dimensions +- `multimodalembedding@001` (default, Vertex AI only). Supports text, images, and video; dimensions: `128`, `256`, `512`, `1408` ## Further resources diff --git a/docs/weaviate/model-providers/google/embeddings.md b/docs/weaviate/model-providers/google/embeddings.md index 802b5d0d8..5101bad00 100644 --- a/docs/weaviate/model-providers/google/embeddings.md +++ b/docs/weaviate/model-providers/google/embeddings.md @@ -150,7 +150,7 @@ You can [specify](#vectorizer-parameters) one of the [available models](#availab ### Google AI Studio (Gemini API) -For Google AI Studio, use the `text2vec_google_gemini()` vectorizer. No `project_id` or `api_endpoint` is required. +For Google AI Studio, use the Gemini-specific vectorizer. A Google Cloud project ID is not required. The Python and TypeScript clients set the Gemini API endpoint for you. @@ -234,9 +234,12 @@ The following examples show how to configure Google-specific options. **Vertex AI parameters:** - `projectId` (Required): Your Google Cloud project ID, e.g. `cloud-large-language-models` +- `location` (Optional): The Google Cloud region to send requests to, e.g. `europe-west1`. - `apiEndpoint` (Optional): Regional endpoint, e.g. `us-central1-aiplatform.googleapis.com` - `modelId` (Optional): e.g. `gemini-embedding-001`, `text-embedding-005` +Set `location` together with a matching `apiEndpoint` to keep data in a specific region. + Deprecated models diff --git a/docs/weaviate/model-providers/google/generative.md b/docs/weaviate/model-providers/google/generative.md index 426e568ec..a63783d05 100644 --- a/docs/weaviate/model-providers/google/generative.md +++ b/docs/weaviate/model-providers/google/generative.md @@ -320,15 +320,19 @@ You can also supply images as a part of the input when performing retrieval augm ### Available models +:::caution Always set the model explicitly +If no model is specified, the server falls back to a legacy PaLM model that Google has deprecated. Set the model explicitly when you configure a collection or when you run a query. +::: + Vertex AI: - `gemini-2.5-pro` -- `gemini-2.5-flash` (default) +- `gemini-2.5-flash` - `gemini-2.0-flash` - `gemini-1.5-pro` - `gemini-1.5-flash` Gemini API: -- `gemini-2.5-flash` (default) +- `gemini-2.5-flash` - `gemini-2.5-pro` - `gemini-2.0-flash` - `gemini-1.5-pro` diff --git a/docs/weaviate/model-providers/google/index.md b/docs/weaviate/model-providers/google/index.md index 5e11e7c96..7b0d72ff7 100644 --- a/docs/weaviate/model-providers/google/index.md +++ b/docs/weaviate/model-providers/google/index.md @@ -22,6 +22,7 @@ Google's embedding models transform text data into vector embeddings, capturing [Weaviate integrates with Google's embedding models](./embeddings.md) to enable seamless vectorization of data. This integration allows users to perform semantic and hybrid search operations without the need for additional preprocessing or data transformation steps. [Google embedding integration page](./embeddings.md) + [Google multimodal embedding integration page](./embeddings-multimodal.md) ### Generative AI models for RAG @@ -42,11 +43,11 @@ In turn, they simplify the process of building AI-driven applications to speed u ## Credentials -You must provide a valid Googles API credentials to Weaviate for these integrations. +You must provide valid Google API credentials to Weaviate for these integrations. ### Vertex AI -##### Automatic token generation +#### Automatic token generation import UseGoogleAuthInstructions from './_includes/use_google_auth_instructions.mdx'; @@ -54,7 +55,7 @@ import UseGoogleAuthInstructions from './_includes/use_google_auth_instructions. ## Get started -Weaviate integrates with both [Google Gemini API](https://aistudio.google.com/app/apikey/?utm_source=weaviate&utm_medium=referral&utm_campaign=partnerships&utm_content=) or [Google Vertex AI](https://cloud.google.com/vertex-ai). +Weaviate integrates with both the [Google Gemini API](https://aistudio.google.com/app/apikey/?utm_source=weaviate&utm_medium=referral&utm_campaign=partnerships&utm_content=) and [Google Vertex AI](https://cloud.google.com/vertex-ai). Go to the relevant integration page to learn how to configure Weaviate with the Google models and start using them in your applications. diff --git a/docs/weaviate/model-providers/huggingface/embeddings.md b/docs/weaviate/model-providers/huggingface/embeddings.md index 3c24822fd..75644f6ff 100644 --- a/docs/weaviate/model-providers/huggingface/embeddings.md +++ b/docs/weaviate/model-providers/huggingface/embeddings.md @@ -53,7 +53,7 @@ You must provide a valid Hugging Face API key to Weaviate for this integration. Provide the API key to Weaviate using one of the following methods: -- Set the `HUGGINGFACE_API_KEY` environment variable that is available to Weaviate. +- Set the `HUGGINGFACE_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. @@ -119,7 +119,7 @@ Provide the API key to Weaviate using one of the following methods: -You must specify one of the [available models](#available-models) for the vectorizer to use. +You can specify one of the [available models](#available-models) for the vectorizer to use. If you do not specify a model, Weaviate uses the server default, `sentence-transformers/msmarco-bert-base-dot-v5`. import VectorizationBehavior from '/_includes/vectorization.behavior.mdx'; @@ -169,13 +169,17 @@ The following examples show how to configure Hugging Face-specific options. Only select one of the following parameters to specify the model: - `model`, -- `passageModel` and `queryModel`, or +- `passageModel`, or - `endpointURL` -:::note Differences between `model`, `passageModel`/`queryModel` and `endpointURL` -The `passageModel` and `queryModel` parameters are used together to specify a [DPR](https://huggingface.co/docs/transformers/en/model_doc/dpr) passage and query model. +:::note Differences between `model`, `passageModel` and `endpointURL` +`model` and `passageModel` name the same thing. Weaviate reads `model` first and falls back to `passageModel`, then uses that model for both object and query vectorization. Setting both raises a validation error. -The `endpointURL` parameter is used to specify a [custom Hugging Face Inference Endpoint](https://huggingface.co/inference-endpoints). This parameter overrides the `model`, `passageModel`, and `queryModel` parameters. +The `endpointURL` parameter is used to specify a [custom Hugging Face Inference Endpoint](https://huggingface.co/inference-endpoints). This parameter overrides the `model` and `passageModel` parameters. +::: + +:::caution `queryModel` no longer has any effect +Weaviate no longer reads the `queryModel` parameter. Support for a separate [DPR](https://huggingface.co/docs/transformers/en/model_doc/dpr) query model was removed in `v1.26.9`, `v1.27.2` and `v1.28.0`. The client libraries still accept `queryModel` (`query_model` in Python) and send it to the server without an error, so any value you set there is silently ignored. Remove it from your collection configuration and use `model` or `passageModel` instead. ::: #### Additional parameters diff --git a/docs/weaviate/model-providers/index.md b/docs/weaviate/model-providers/index.md index d6c85263b..20c49da2b 100644 --- a/docs/weaviate/model-providers/index.md +++ b/docs/weaviate/model-providers/index.md @@ -40,7 +40,11 @@ This enables an enhanced developed experience, such as the ability to: #### Enable all API-based modules -All API-based model integrations are available by default starting with Weaviate `v1.33`. For older versions, you can enable them all by setting the [`ENABLE_API_BASED_MODULES` environment variable](/deploy/configuration/env-vars#ENABLE_API_BASED_MODULES) to `true`. +All API-based model integrations are available by default starting with Weaviate `v1.33`. + +To opt out, for example in an air-gapped or otherwise restricted deployment, set the [`API_BASED_MODULES_DISABLED` environment variable](/deploy/configuration/env-vars#API_BASED_MODULES_DISABLED) to `true`. Weaviate then loads only the modules that you list in [`ENABLE_MODULES`](/deploy/configuration/env-vars#ENABLE_MODULES). This variable was added in `v1.33`. + +For releases before `v1.33`, enable all API-based modules by setting the [`ENABLE_API_BASED_MODULES` environment variable](/deploy/configuration/env-vars#ENABLE_API_BASED_MODULES) to `true`. Weaviate stopped reading that variable in `v1.33`. ### Locally hosted diff --git a/docs/weaviate/model-providers/jinaai/embeddings-colbert.md b/docs/weaviate/model-providers/jinaai/embeddings-colbert.md index 73897c343..e9af8d70f 100644 --- a/docs/weaviate/model-providers/jinaai/embeddings-colbert.md +++ b/docs/weaviate/model-providers/jinaai/embeddings-colbert.md @@ -52,7 +52,7 @@ You must provide a valid Jina AI API key to Weaviate for this integration. Go to Provide the API key to Weaviate using one of the following methods: -- Set the `JINAAI_API_KEY` environment variable that is available to Weaviate. +- Set the `JINAAI_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. @@ -328,7 +328,7 @@ The query below returns the `n` best scoring objects from the database, set by ` ### Available models -- `jina-colbert-v2` +- `jina-colbert-v2` (server default) - By default, Weaviate uses `128` dimensions - `jina-colbert-v1` diff --git a/docs/weaviate/model-providers/jinaai/embeddings-multimodal.md b/docs/weaviate/model-providers/jinaai/embeddings-multimodal.md index 4bc35c135..a9850dec3 100644 --- a/docs/weaviate/model-providers/jinaai/embeddings-multimodal.md +++ b/docs/weaviate/model-providers/jinaai/embeddings-multimodal.md @@ -50,7 +50,7 @@ You must provide a valid Jina AI API key to Weaviate for this integration. Go to Provide the API key to Weaviate using one of the following methods: -- Set the `JINAAI_API_KEY` environment variable that is available to Weaviate. +- Set the `JINAAI_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. @@ -125,7 +125,7 @@ You can specify one of the [available models](#available-models) for the vectori -You can [specify](#vectorizer-parameters) one of the [available models](#available-models) for Weaviate to use. The [default model](#available-models) is used if no model is specified. +The [default model](#available-models) is used if you do not specify one. import VectorizationBehavior from '/_includes/vectorization.behavior.mdx'; @@ -140,6 +140,10 @@ import VectorizationBehavior from '/_includes/vectorization.behavior.mdx'; The following examples show how to configure Jina AI-specific options. +- `model`: The model name. +- `dimensions`: The number of dimensions for the model. + - Note that [not all models](#available-models) support this parameter. + -### Vectorizer parameters - -- `model`: The model name. -- `dimensions`: The number of dimensions for the model. - - Note that [not all models](#available-models) support this parameter. - ## Data import After configuring the vectorizer, [import data](../../manage-objects/import.mdx) into Weaviate. Weaviate generates embeddings for text objects using the specified model. @@ -297,7 +295,7 @@ The query below returns the `n` most similar objects to the input image from the ### Available models -- `jina-clip-v2` +- `jina-clip-v2` (server default) - This model is a multilingual, multimodal model using [Matryoshka Representation Learning](https://arxiv.org/abs/2205.13147). - It will accept a `dimensions` parameter, which can be any integer between (and including) 64 and 1024. The default value is 1024. - `jina-clip-v1` diff --git a/docs/weaviate/model-providers/jinaai/embeddings.md b/docs/weaviate/model-providers/jinaai/embeddings.md index 1aed40823..460aca635 100644 --- a/docs/weaviate/model-providers/jinaai/embeddings.md +++ b/docs/weaviate/model-providers/jinaai/embeddings.md @@ -53,7 +53,7 @@ You must provide a valid Jina AI API key to Weaviate for this integration. Go to Provide the API key to Weaviate using one of the following methods: -- Set the `JINAAI_API_KEY` environment variable that is available to Weaviate. +- Set the `JINAAI_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. @@ -329,13 +329,19 @@ The query below returns the `n` best scoring objects from the database, set by ` ### Available models +The server default changed in `v1.32.0`, and was backported to `v1.31.6`. Earlier releases on each of those lines default to `jina-embeddings-v2-base-en`. + +- `jina-embeddings-v4` (server default) + - When using this model, Weaviate will automatically use the appropriate `task` type, applying `retrieval.passage` for embedding entries and `retrieval.query` for queries. - `jina-embeddings-v3` - When using this model, Weaviate will automatically use the appropriate `task` type, applying `retrieval.passage` for embedding entries and `retrieval.query` for queries. - - By default, Weaviate uses `1024` dimensions -- `jina-embeddings-v2-base-en` (Default) +- `jina-embeddings-v2-base-en` (previous server default) - `jina-embeddings-v2-small-en` +- `jina-embeddings-v2-base-zh` +- `jina-embeddings-v2-base-es` +- `jina-embeddings-v2-base-code` -Note that `dimensions` is not applicable for the `jina-embeddings-v2` models. +If you do not set `dimensions`, Weaviate does not send a dimension count and the Jina AI API applies its own default for the model. Note that `dimensions` is not applicable for the `jina-embeddings-v2` models. ## Further resources diff --git a/docs/weaviate/model-providers/jinaai/index.md b/docs/weaviate/model-providers/jinaai/index.md index bf78e3e72..9d1b6144b 100644 --- a/docs/weaviate/model-providers/jinaai/index.md +++ b/docs/weaviate/model-providers/jinaai/index.md @@ -25,6 +25,16 @@ Jina AI's embedding models transform text data into vector embeddings, capturing [Jina AI ColBERT embedding integration page](./embeddings-colbert.md) [Jina AI multimodal embedding integration page](./embeddings-multimodal.md) +### Reranker models + +![Reranker integration illustration](../_includes/integration_jinaai_reranker.png) + +Jina AI's reranker models are designed to improve the relevance and ranking of search results. + +[The Weaviate reranker integration](./reranker.md) allows users to easily refine their search results by leveraging Jina AI's reranker models. + +[Jina AI reranker integration page](./reranker.md) + ## Summary These integrations enable developers to leverage Jina AI's powerful models directly within Weaviate. @@ -40,7 +50,7 @@ Then, go to the relevant integration page to learn how to configure Weaviate wit - [Text Embeddings](./embeddings.md) - [ColBERT embeddings](./embeddings-colbert.md) - [Multimodal embeddings](./embeddings-multimodal.md) -- [Rerankers](./reranker.md) +- [Reranker](./reranker.md) ## Questions and feedback diff --git a/docs/weaviate/model-providers/jinaai/reranker.md b/docs/weaviate/model-providers/jinaai/reranker.md index 3a2343fa0..a1f78b60f 100644 --- a/docs/weaviate/model-providers/jinaai/reranker.md +++ b/docs/weaviate/model-providers/jinaai/reranker.md @@ -50,7 +50,7 @@ You must provide a valid JinaAI API key to Weaviate for this integration. Go to Provide the API key to Weaviate using one of the following methods: -- Set the `JINAAI_API_KEY` environment variable that is available to Weaviate. +- Set the `JINAAI_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. @@ -167,7 +167,7 @@ Any search in Weaviate can be combined with a reranker to perform reranking oper ### Available models -- `jina-reranker-v2-base-multilingual` (default) +- `jina-reranker-v2-base-multilingual` (server default) - `jina-reranker-v1-base-en` - `jina-reranker-v1-turbo-en` - `jina-reranker-v1-tiny-en` diff --git a/docs/weaviate/model-providers/kubeai/_category_.json b/docs/weaviate/model-providers/kubeai/_category_.json index a2c5ceb93..b319d23e4 100644 --- a/docs/weaviate/model-providers/kubeai/_category_.json +++ b/docs/weaviate/model-providers/kubeai/_category_.json @@ -1,4 +1,4 @@ { - "label": "KubeAI (Locally hosted)", - "position": 320 + "label": "KubeAI (locally hosted)", + "position": 330 } diff --git a/docs/weaviate/model-providers/kubeai/embeddings.md b/docs/weaviate/model-providers/kubeai/embeddings.md index 188567a0a..2cac62fef 100644 --- a/docs/weaviate/model-providers/kubeai/embeddings.md +++ b/docs/weaviate/model-providers/kubeai/embeddings.md @@ -55,7 +55,7 @@ The OpenAI integration requires an API key value. To use KubeAI, provide any val Provide the API key to Weaviate using one of the following methods: -- Set the `OPENAI_API_KEY` environment variable that is available to Weaviate. +- Set the `OPENAI_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. diff --git a/docs/weaviate/model-providers/kubeai/generative.md b/docs/weaviate/model-providers/kubeai/generative.md index b3c867bd9..1278907ec 100644 --- a/docs/weaviate/model-providers/kubeai/generative.md +++ b/docs/weaviate/model-providers/kubeai/generative.md @@ -55,7 +55,7 @@ The OpenAI integration requires an API key value. To use KubeAI, provide any val Provide the API key to Weaviate using one of the following methods: -- Set the `OPENAI_API_KEY` environment variable that is available to Weaviate. +- Set the `OPENAI_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. diff --git a/docs/weaviate/model-providers/kubeai/index.md b/docs/weaviate/model-providers/kubeai/index.md index 8f8200197..e1d500803 100644 --- a/docs/weaviate/model-providers/kubeai/index.md +++ b/docs/weaviate/model-providers/kubeai/index.md @@ -7,7 +7,7 @@ image: og/docs/integrations/provider_integrations_kubeai.jpg -[KubeAI](https://github.com/substratusai/kubeai) provides offers a wide range of models for natural language processing and generation through OpenAI-style API endpoints. Weaviate seamlessly integrates with KubeAI's APIs, allowing users to leverage any KubeAI models directly from the Weaviate Database. +[KubeAI](https://github.com/substratusai/kubeai) offers a wide range of models for natural language processing and generation through OpenAI-style API endpoints. Weaviate seamlessly integrates with KubeAI's APIs, allowing users to leverage any KubeAI models directly from the Weaviate Database. These integrations empower developers to build sophisticated AI-driven applications with ease. diff --git a/docs/weaviate/model-providers/mistral/embeddings.md b/docs/weaviate/model-providers/mistral/embeddings.md index 6b3102f70..92cd583fe 100644 --- a/docs/weaviate/model-providers/mistral/embeddings.md +++ b/docs/weaviate/model-providers/mistral/embeddings.md @@ -53,7 +53,7 @@ You must provide a valid Mistral API key to Weaviate for this integration. Go to Provide the API key to Weaviate using one of the following methods: -- Set the `MISTRAL_API_KEY` environment variable that is available to Weaviate. +- Set the `MISTRAL_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. diff --git a/docs/weaviate/model-providers/mistral/generative.md b/docs/weaviate/model-providers/mistral/generative.md index 97aee3971..8ede2a11e 100644 --- a/docs/weaviate/model-providers/mistral/generative.md +++ b/docs/weaviate/model-providers/mistral/generative.md @@ -52,7 +52,7 @@ You must provide a valid Mistral API key to Weaviate for this integration. Go to Provide the API key to Weaviate using one of the following methods: -- Set the `MISTRAL_API_KEY` environment variable that is available to Weaviate. +- Set the `MISTRAL_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. diff --git a/docs/weaviate/model-providers/model2vec/_category_.json b/docs/weaviate/model-providers/model2vec/_category_.json index 8f8d4ff1e..88b85a984 100644 --- a/docs/weaviate/model-providers/model2vec/_category_.json +++ b/docs/weaviate/model-providers/model2vec/_category_.json @@ -1,4 +1,4 @@ { - "label": "GPT4All (locally hosted)", - "position": 320 + "label": "Model2Vec (locally hosted)", + "position": 340 } diff --git a/docs/weaviate/model-providers/nvidia/embeddings-multimodal.md b/docs/weaviate/model-providers/nvidia/embeddings-multimodal.md index 8d10ef361..a15e386e5 100644 --- a/docs/weaviate/model-providers/nvidia/embeddings-multimodal.md +++ b/docs/weaviate/model-providers/nvidia/embeddings-multimodal.md @@ -51,7 +51,7 @@ You must provide a valid NVIDIA NIM API key to Weaviate for this integration. Go Provide the API key to Weaviate using one of the following methods: -- Set the `NVIDIA_API_KEY` environment variable that is available to Weaviate. +- Set the `NVIDIA_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. diff --git a/docs/weaviate/model-providers/nvidia/embeddings.md b/docs/weaviate/model-providers/nvidia/embeddings.md index ff5e0ba1f..377ecdafc 100644 --- a/docs/weaviate/model-providers/nvidia/embeddings.md +++ b/docs/weaviate/model-providers/nvidia/embeddings.md @@ -53,7 +53,7 @@ You must provide a valid NVIDIA NIM API key to Weaviate for this integration. Go Provide the API key to Weaviate using one of the following methods: -- Set the `NVIDIA_API_KEY` environment variable that is available to Weaviate. +- Set the `NVIDIA_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. @@ -342,7 +342,7 @@ The default model is `nvidia/nv-embed-v1`. ### Other integrations -- [NVIDIA multimodal embedding embeddings models + Weaviate](./embeddings-multimodal.md) +- [NVIDIA multimodal embedding models + Weaviate](./embeddings-multimodal.md) - [NVIDIA generative models + Weaviate](./generative.md) - [NVIDIA reranker models + Weaviate](./reranker.md) diff --git a/docs/weaviate/model-providers/nvidia/generative.md b/docs/weaviate/model-providers/nvidia/generative.md index 9d4675e8a..59473b3fe 100644 --- a/docs/weaviate/model-providers/nvidia/generative.md +++ b/docs/weaviate/model-providers/nvidia/generative.md @@ -50,7 +50,7 @@ You must provide a valid API key to Weaviate for this integration. Go to [NVIDIA Provide the API key to Weaviate using one of the following methods: -- Set the `NVIDIA_API_KEY` environment variable that is available to Weaviate. +- Set the `NVIDIA_APIKEY` environment variable that is available to Weaviate. - Provide the token at runtime, as shown in the examples below. @@ -260,7 +260,7 @@ The default model is `nvidia/llama-3.1-nemotron-51b-instruct`. ### Other integrations - [NVIDIA text embedding models + Weaviate](./embeddings.md). -- [NVIDIA multimodal embedding embeddings models + Weaviate](./embeddings-multimodal.md) +- [NVIDIA multimodal embedding models + Weaviate](./embeddings-multimodal.md). - [NVIDIA reranker models + Weaviate](./reranker.md). ### Code examples diff --git a/docs/weaviate/model-providers/nvidia/index.md b/docs/weaviate/model-providers/nvidia/index.md index 9adbfd246..75fa0f487 100644 --- a/docs/weaviate/model-providers/nvidia/index.md +++ b/docs/weaviate/model-providers/nvidia/index.md @@ -54,7 +54,7 @@ In turn, it simplifies the process of building AI-driven applications to speed u You must provide a valid NVIDIA API key to Weaviate for this integration. Go to [NVIDIA](https://build.nvidia.com/) to sign up and obtain an API key. -Then, go to the relevant integration page to learn how to configure Weaviate with the Cohere models and start using them in your applications. +Then, go to the relevant integration page to learn how to configure Weaviate with the NVIDIA models and start using them in your applications. - [Text Embeddings](./embeddings.md) - [Multimodal Embeddings](./embeddings-multimodal.md) diff --git a/docs/weaviate/model-providers/nvidia/reranker.md b/docs/weaviate/model-providers/nvidia/reranker.md index e21d9a52d..497b6dfa6 100644 --- a/docs/weaviate/model-providers/nvidia/reranker.md +++ b/docs/weaviate/model-providers/nvidia/reranker.md @@ -51,7 +51,7 @@ You must provide a valid NVIDIA NIM API key to Weaviate for this integration. Go Provide the API key to Weaviate using one of the following methods: -- Set the `NVIDIA_API_KEY` environment variable that is available to Weaviate. +- Set the `NVIDIA_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. @@ -170,14 +170,14 @@ Any search in Weaviate can be combined with a reranker to perform reranking oper You can use any reranker model [on NVIDIA NIM APIs](https://build.nvidia.com/models) with Weaviate. -The default model is `nnvidia/rerank-qa-mistral-4b`. +The default model is `nvidia/rerank-qa-mistral-4b`. ## Further resources ### Other integrations - [NVIDIA text embedding models + Weaviate](./embeddings.md). -- [NVIDIA multimodal embedding embeddings models + Weaviate](./embeddings-multimodal.md) +- [NVIDIA multimodal embedding models + Weaviate](./embeddings-multimodal.md). - [NVIDIA generative models + Weaviate](./generative.md). ### Code examples diff --git a/docs/weaviate/model-providers/octoai/_includes/octoai_deprecation.md b/docs/weaviate/model-providers/octoai/_includes/octoai_deprecation.md index 681cad6b3..c2e4de674 100644 --- a/docs/weaviate/model-providers/octoai/_includes/octoai_deprecation.md +++ b/docs/weaviate/model-providers/octoai/_includes/octoai_deprecation.md @@ -7,6 +7,9 @@ OctoAI announced that they are winding down the commercial availability of its services by **31 October 2024**. Accordingly, the Weaviate OctoAI integrations are deprecated. Do not use these integrations for new projects.
+From Weaviate `v1.25.22`, `v1.26.8`, and `v1.27.1`, the `text2vec-octoai` and `generative-octoai` modules are inactive. Every vectorization and generation request they receive fails server-side with the error `OctoAI is permanently shut down`. The configuration, import, search, and RAG examples on these pages therefore no longer run against OctoAI, and are kept only as a record of how the integrations used to be configured. Of the options below, only "bring your own vectors" (Option 1) keeps an existing OctoAI collection usable. +
+ If you have a collection that is using an OctoAI integration, consider your options depending on whether you are using OctoAI's embedding models ([your options](#for-collections-with-octoai-embedding-integrations)) or generative models ([your options](#for-collections-with-octoai-generative-ai-integrations)). #### For collections with OctoAI embedding integrations diff --git a/docs/weaviate/model-providers/octoai/embeddings.md b/docs/weaviate/model-providers/octoai/embeddings.md index 1df992d6a..5822ab176 100644 --- a/docs/weaviate/model-providers/octoai/embeddings.md +++ b/docs/weaviate/model-providers/octoai/embeddings.md @@ -54,7 +54,7 @@ You must provide a valid OctoAI API key to Weaviate for this integration. Go to Provide the API key to Weaviate using one of the following methods: -- Set the `OCTOAI_API_KEY` environment variable that is available to Weaviate. +- Set the `OCTOAI_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. diff --git a/docs/weaviate/model-providers/octoai/generative.md b/docs/weaviate/model-providers/octoai/generative.md index 54c6b914a..5637813c8 100644 --- a/docs/weaviate/model-providers/octoai/generative.md +++ b/docs/weaviate/model-providers/octoai/generative.md @@ -54,7 +54,7 @@ You must provide a valid OctoAI API key to Weaviate for this integration. Go to Provide the API key to Weaviate using one of the following methods: -- Set the `OCTOAI_API_KEY` environment variable that is available to Weaviate. +- Set the `OCTOAI_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. diff --git a/docs/weaviate/model-providers/openai-azure/embeddings.md b/docs/weaviate/model-providers/openai-azure/embeddings.md index 614cd6f9d..1bbc7480e 100644 --- a/docs/weaviate/model-providers/openai-azure/embeddings.md +++ b/docs/weaviate/model-providers/openai-azure/embeddings.md @@ -54,7 +54,7 @@ You must provide a valid Azure OpenAI API key to Weaviate for this integration. Provide the API key to Weaviate using one of the following methods: -- Set the `AZURE_API_KEY` environment variable that is available to Weaviate. +- Set the `AZURE_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. diff --git a/docs/weaviate/model-providers/openai-azure/generative.md b/docs/weaviate/model-providers/openai-azure/generative.md index b034457c3..c942f84eb 100644 --- a/docs/weaviate/model-providers/openai-azure/generative.md +++ b/docs/weaviate/model-providers/openai-azure/generative.md @@ -52,7 +52,7 @@ You must provide a valid Azure OpenAI API key to Weaviate for this integration. Provide the API key to Weaviate using one of the following methods: -- Set the `AZURE_API_KEY` environment variable that is available to Weaviate. +- Set the `AZURE_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. diff --git a/docs/weaviate/model-providers/openai/embeddings.md b/docs/weaviate/model-providers/openai/embeddings.md index e73187ac1..51f6cc470 100644 --- a/docs/weaviate/model-providers/openai/embeddings.md +++ b/docs/weaviate/model-providers/openai/embeddings.md @@ -59,7 +59,7 @@ You must provide a valid OpenAI API key to Weaviate for this integration. Go to Provide the API key to Weaviate using one of the following methods: -- Set the `OPENAI_API_KEY` environment variable that is available to Weaviate. +- Set the `OPENAI_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. @@ -217,6 +217,9 @@ import VectorizationBehavior from '/_includes/vectorization.behavior.mdx'; - `modelVersion`: The version string for the model. - `type`: The model type, either `text` or `code`. - `baseURL`: The URL to use (e.g. a proxy) instead of the default OpenAI URL. +- `endpoint`: The API path that Weaviate appends to the base URL. Defaults to `/v1/embeddings`. Set it if an OpenAI-compatible service uses a different path. + +For how Weaviate combines `baseURL` and `endpoint` into a request URL, see [Header parameters](#header-parameters). #### (`model` & `dimensions`) or (`model` & `modelVersion`) @@ -272,9 +275,13 @@ Any additional headers provided at runtime will override the existing Weaviate c Provide the headers as shown in the [API credentials examples](#api-credentials) above. -:::note +:::note How Weaviate builds the request URL + +Use the `X-OpenAI-Baseurl` header, or the `baseURL` parameter, to target an OpenAI-compatible service. Weaviate builds the request URL by appending the `endpoint` path (`/v1/embeddings` by default) to the base URL. + +If your provider uses a different path, set [`endpoint`](#vectorizer-parameters) to that path, or rewrite the path with a proxy. -By passing the `X-OpenAI-Baseurl`, you can use an endpoint compatible with the OpenAI API. Weaviate appends `/v1/embeddings` to this base URL. If this doesn't match your endpoint, you can rewrite the path with a proxy (e.g., `your.domain.com/v1/embeddings` -> `api.deepinfra.com/v1/openai/embeddings`). +The [Azure OpenAI integration](../openai-azure/embeddings.md) builds a deployment-specific path and ignores `endpoint`. ::: diff --git a/docs/weaviate/model-providers/openai/generative.md b/docs/weaviate/model-providers/openai/generative.md index e40782ba8..b305241c9 100644 --- a/docs/weaviate/model-providers/openai/generative.md +++ b/docs/weaviate/model-providers/openai/generative.md @@ -57,7 +57,7 @@ You must provide a valid OpenAI API key to Weaviate for this integration. Go to Provide the API key to Weaviate using one of the following methods: -- Set the `OPENAI_API_KEY` environment variable that is available to Weaviate. +- Set the `OPENAI_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. @@ -163,6 +163,13 @@ Configure the following generative parameters to customize the model behavior. +Two additional parameters are available for reasoning models such as the `gpt-5` family. They were added in `v1.33.0`, and backported to `v1.31.15` and `v1.32.9`: + +- `reasoningEffort`: How much reasoning the model does before it answers. One of `minimal`, `low`, `medium`, or `high`. If not set, the model provider default applies. +- `verbosity`: How detailed the generated answer is. One of `low`, `medium`, or `high`. If not set, the model provider default applies. + +The Python client exposes these as the `reasoning_effort` and `verbosity` arguments, both when you configure the collection and when you [select a model at runtime](#select-a-model-at-runtime). The TypeScript client does not expose them yet, so set them with another client or through the REST collection configuration API. + For further details on model parameters, see the [OpenAI API documentation](https://platform.openai.com/docs/api-reference/chat). ## Select a model at runtime @@ -200,6 +207,16 @@ Any additional headers provided at runtime will override the existing Weaviate c Provide the headers as shown in the [API credentials examples](#api-credentials) above. +:::note How Weaviate builds the request URL + +Use the `X-OpenAI-Baseurl` header, or the `baseURL` parameter, to target an OpenAI-compatible service. Weaviate builds the request URL by appending `/v1/chat/completions` to the base URL, or `/v1/completions` for the legacy `text-davinci-002` and `text-davinci-003` models. + +The generative integration has no `endpoint` parameter to override this path, unlike the [OpenAI embeddings integration](./embeddings.md#header-parameters). If your provider expects a different path, point the base URL at a proxy that rewrites `/v1/chat/completions` to your provider's path. + +The [Azure OpenAI integration](../openai-azure/generative.md) builds a deployment-specific path and ignores the paths above. + +::: + ## Retrieval augmented generation After configuring the generative AI integration, perform RAG operations, either with the [single prompt](#single-prompt) or [grouped task](#grouped-task) method. @@ -294,7 +311,16 @@ You can also supply images as a part of the input when performing retrieval augm ### Available models -* [gpt-3.5-turbo](https://platform.openai.com/docs/models/gpt-3-5) (default) +Weaviate does not validate the model name, so you can set any model that your OpenAI account can reach. Name validation was removed in `v1.33.0`, and backported to `v1.31.17` and `v1.32.10`. + +The server default is `gpt-5-mini`. It changed in `v1.32.3`, and was backported to `v1.30.16` and `v1.31.10`. Earlier releases on each of those lines default to `gpt-3.5-turbo`. + +The following models are recognized by Weaviate's token limit table: + +* [gpt-5](https://platform.openai.com/docs/models/gpt-5) +* [gpt-5-mini](https://platform.openai.com/docs/models/gpt-5-mini) (server default) +* [gpt-5-nano](https://platform.openai.com/docs/models/gpt-5-nano) +* [gpt-3.5-turbo](https://platform.openai.com/docs/models/gpt-3-5) (previous server default) * [gpt-3.5-turbo-16k](https://platform.openai.com/docs/models/gpt-3-5) * [gpt-3.5-turbo-1106](https://platform.openai.com/docs/models/gpt-3-5) * [gpt-4](https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo) diff --git a/docs/weaviate/model-providers/transformers/embeddings-custom-image.md b/docs/weaviate/model-providers/transformers/embeddings-custom-image.md index 7c2be7f1b..424eaa188 100644 --- a/docs/weaviate/model-providers/transformers/embeddings-custom-image.md +++ b/docs/weaviate/model-providers/transformers/embeddings-custom-image.md @@ -70,7 +70,7 @@ To build an image with a local, custom model, create a new `Dockerfile` similar Save the `Dockerfile` as `my-inference-image.Dockerfile`. (You can name it anything you like.)
-This will creates a custom image for a model stored in a local folder `my-model` on your machine. +This will create a custom image for a model stored in a local folder `my-model` on your machine.
```yaml diff --git a/docs/weaviate/model-providers/transformers/embeddings-multimodal-custom-image.md b/docs/weaviate/model-providers/transformers/embeddings-multimodal-custom-image.md index 0773b08bd..e2e6ec499 100644 --- a/docs/weaviate/model-providers/transformers/embeddings-multimodal-custom-image.md +++ b/docs/weaviate/model-providers/transformers/embeddings-multimodal-custom-image.md @@ -61,16 +61,16 @@ RUN CLIP_MODEL_NAME=clip-ViT-B-32 TEXT_MODEL_NAME=clip-ViT-B-32 ./download.py
-You can also build a custom image with any model that is compatible with the Transformer library's `SentenceTransformers` and `ClIPModel` classes. To ensure that text embeddings will output compatible vectors to image embeddings, you must only use models that have been specifically trained for use with CLIP models. (Note that a CLIP model is in reality two models: one for text and one for images.) +You can also build a custom image with models compatible with the `SentenceTransformer` class from the Sentence Transformers library and the `CLIPModel` class from the Transformers library. To ensure that text embeddings will output compatible vectors to image embeddings, you must only use models that have been specifically trained for use with CLIP models. (Note that a CLIP model is in reality two models: one for text and one for images.)
-To build an image with a local, custom model, create a new `Dockerfile` similar to the following, replacing `./my-test-model` and `./my-clip-model` with the path to your model folder. +To build an image with a local, custom model, create a new `Dockerfile` similar to the following, replacing `./my-text-model` and `./my-clip-model` with the paths to your model folders.
Save the `Dockerfile` as `my-inference-image.Dockerfile`. (You can name it anything you like.)
-This will creates a custom image for a model stored in a local folder `my-model` on your machine. +This will create a custom image for the models stored in the local folders `my-text-model` and `my-clip-model` on your machine.
```yaml diff --git a/docs/weaviate/model-providers/transformers/embeddings-multimodal.md b/docs/weaviate/model-providers/transformers/embeddings-multimodal.md index cc14c4ebe..5c1640c31 100644 --- a/docs/weaviate/model-providers/transformers/embeddings-multimodal.md +++ b/docs/weaviate/model-providers/transformers/embeddings-multimodal.md @@ -156,8 +156,8 @@ As this integration runs a local container with the CLIP model, no additional cr
-:::note Chose a container image to select a model -To chose a model, select the [container image](#configure-the-integration) that hosts it. +:::note Choose a container image to select a model +To choose a model, select the [container image](#configure-the-integration) that hosts it. ::: import VectorizationBehavior from '/_includes/vectorization.behavior.mdx'; diff --git a/docs/weaviate/model-providers/transformers/embeddings.md b/docs/weaviate/model-providers/transformers/embeddings.md index 409b7636b..a0fd1ebf7 100644 --- a/docs/weaviate/model-providers/transformers/embeddings.md +++ b/docs/weaviate/model-providers/transformers/embeddings.md @@ -159,8 +159,8 @@ As this integration runs a local container with the Transformers model, no addit
-:::note Chose a container image to select a model -To chose a model, select the [container image](#configure-the-integration) that hosts it. +:::note Choose a container image to select a model +To choose a model, select the [container image](#configure-the-integration) that hosts it. ::: import VectorizationBehavior from '/_includes/vectorization.behavior.mdx'; @@ -188,7 +188,7 @@ Specify `passageInferenceUrl` and `queryInferenceUrl` if using a [DPR](https://h #### Additional parameters -- `poolingStrategy` – the pooling strategy to use when the input exceeds the model's context window. +- `poolingStrategy`: the pooling strategy to use when the input exceeds the model's context window. - Default: `masked_mean`. Allowed values: `masked_mean` or `cls`. ([Read more on this topic.](https://arxiv.org/abs/1908.10084)) diff --git a/docs/weaviate/model-providers/transformers/index.md b/docs/weaviate/model-providers/transformers/index.md index bf3be8a5c..efaeadf2c 100644 --- a/docs/weaviate/model-providers/transformers/index.md +++ b/docs/weaviate/model-providers/transformers/index.md @@ -25,6 +25,16 @@ Transformers-compatible embedding models transform text data into vector embeddi [Hugging Face Transformers embedding integration page](./embeddings.md) +### Reranker models + +![Reranker integration illustration](../_includes/integration_transformers_reranker.png) + +Transformers-compatible reranker models are designed to improve the relevance and ranking of search results. + +[The Weaviate reranker integration](./reranker.md) allows users to easily refine their search results with a locally hosted Hugging Face Transformers reranker model. + +[Hugging Face Transformers reranker integration page](./reranker.md) + ## Summary These integrations enable developers to leverage powerful Hugging Face Transformers models from directly within Weaviate. @@ -41,6 +51,7 @@ Go to the relevant integration page to learn how to configure Weaviate with the - [Text Embeddings (custom image)](./embeddings-custom-image.md) - [Multimodal Embeddings](./embeddings-multimodal.md) - [Multimodal Embeddings (custom image)](./embeddings-multimodal-custom-image.md) +- [Reranker](./reranker.md) ## Questions and feedback diff --git a/docs/weaviate/model-providers/transformers/reranker.md b/docs/weaviate/model-providers/transformers/reranker.md index cff1fba80..b47029ee0 100644 --- a/docs/weaviate/model-providers/transformers/reranker.md +++ b/docs/weaviate/model-providers/transformers/reranker.md @@ -160,8 +160,8 @@ Configure a Weaviate collection to use a Transformer reranker model as follows: -:::note Chose a container image to select a model -To chose a model, select the [container image](#configure-the-integration) that hosts it. +:::note Choose a container image to select a model +To choose a model, select the [container image](#configure-the-integration) that hosts it. ::: ## Reranking query diff --git a/docs/weaviate/model-providers/voyageai/embeddings-multimodal.md b/docs/weaviate/model-providers/voyageai/embeddings-multimodal.md index 5e9b91790..2ebc51c23 100644 --- a/docs/weaviate/model-providers/voyageai/embeddings-multimodal.md +++ b/docs/weaviate/model-providers/voyageai/embeddings-multimodal.md @@ -53,7 +53,7 @@ You must provide a valid VoyageAI API key to Weaviate for this integration. Go t Provide the API key to Weaviate using one of the following methods: -- Set the `VOYAGEAI_API_KEY` environment variable that is available to Weaviate. +- Set the `VOYAGEAI_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. diff --git a/docs/weaviate/model-providers/voyageai/embeddings.md b/docs/weaviate/model-providers/voyageai/embeddings.md index 49b709857..d45101c6a 100644 --- a/docs/weaviate/model-providers/voyageai/embeddings.md +++ b/docs/weaviate/model-providers/voyageai/embeddings.md @@ -54,7 +54,7 @@ You must provide a valid Voyage AI API key to Weaviate for this integration. Go Provide the API key to Weaviate using one of the following methods: -- Set the `VOYAGEAI_API_KEY` environment variable that is available to Weaviate. +- Set the `VOYAGEAI_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. @@ -387,8 +387,8 @@ The `voyage-context-3` model uses Voyage AI's [contextual embeddings API](https: ### Other integrations -- [Voyage AI multimodal embedding embeddings models + Weaviate](./embeddings-multimodal.md) -- [Voyage AI reranker models + Weaviate](./embeddings.md). +- [Voyage AI multimodal embedding models + Weaviate](./embeddings-multimodal.md). +- [Voyage AI reranker models + Weaviate](./reranker.md). ### Code examples diff --git a/docs/weaviate/model-providers/voyageai/reranker.md b/docs/weaviate/model-providers/voyageai/reranker.md index e8a903445..d85b75e5b 100644 --- a/docs/weaviate/model-providers/voyageai/reranker.md +++ b/docs/weaviate/model-providers/voyageai/reranker.md @@ -52,7 +52,7 @@ You must provide a valid Voyage AI API key to Weaviate for this integration. Go Provide the API key to Weaviate using one of the following methods: -- Set the `VOYAGEAI_API_KEY` environment variable that is available to Weaviate. +- Set the `VOYAGEAI_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. @@ -184,7 +184,7 @@ Any search in Weaviate can be combined with a reranker to perform reranking oper ### Other integrations - [Voyage AI embedding models + Weaviate](./embeddings.md). -- [Voyage AI multimodal embedding embeddings models + Weaviate](./embeddings-multimodal.md) +- [Voyage AI multimodal embedding models + Weaviate](./embeddings-multimodal.md). ### Code examples diff --git a/docs/weaviate/model-providers/xai/generative.md b/docs/weaviate/model-providers/xai/generative.md index b4fb417f6..ebf28042a 100644 --- a/docs/weaviate/model-providers/xai/generative.md +++ b/docs/weaviate/model-providers/xai/generative.md @@ -54,7 +54,7 @@ You must provide a valid API key to Weaviate for this integration. Go to [xAI](h Provide the API key to Weaviate using one of the following methods: -- Set the `XAI_API_KEY` environment variable that is available to Weaviate. +- Set the `XAI_APIKEY` environment variable that is available to Weaviate. - Provide the API key at runtime, as shown in the examples below. diff --git a/docs/weaviate/modules/index.md b/docs/weaviate/modules/index.md index b21f43ea5..48734d7dd 100644 --- a/docs/weaviate/modules/index.md +++ b/docs/weaviate/modules/index.md @@ -9,7 +9,7 @@ image: og/docs/modules/_title.jpg This section describes Weaviate's individual modules, including their capabilities and how to use them. :::tip Looking for vectorizer, generative AI, or reranker integration docs? -They have moved to our [model provider integrations](../model-providers/index.md) section, for a more focussed, user-centric look at these integrations. +They have moved to our [model provider integrations](../model-providers/index.md) section, for a more focused, user-centric look at these integrations. ::: ## General @@ -25,7 +25,7 @@ Weaviate modules can be divided into the following categories: - [Generative AI](#vectorizer-reranker-and-generative-ai-integrations): Integrate generative AI models for retrieval augmented generation (RAG). - [Backup](#backup-modules): Facilitate backup and restore operations in Weaviate. - [Offloading](#offloading-modules): Facilitate offloading of tenant data to external storage. -- [Others]: Modules that provide additional functionalities. +- [Others](#other-modules): Modules that provide additional functionalities. #### Vectorizer, reranker, and generative AI integrations @@ -113,9 +113,18 @@ In addition to the above, there are other modules such as: - [qna-transformers](./qna-transformers.md): Question-answering (answer extraction) capability using transformers models. - [qna-openai](./qna-openai.md): Question-answering (answer extraction) capability using OpenAI models. - [ner-transformers](./ner-transformers.md): Named entity recognition capability using transformers models. -- [text-spellcheck](./ner-transformers.md): Spell checking capability for GraphQL queries. +- [text-spellcheck](./spellcheck.md): Spell checking capability for GraphQL queries. - [sum-transformers](./sum-transformers.md): Summarize text using transformer models. - [usage-modules](./usage-modules.md): Collect and upload usage analytics to GCS or S3 for the purposes of billing. +- [custom-modules](./custom-modules.md): Attach your own machine learning model to Weaviate as a module. + +### Other vectorizer modules + +The following vectorizer modules are not covered by the [model provider integration](../model-providers/index.md) pages: + +- [text2vec-contextionary](./text2vec-contextionary.md) (deprecated): Vectorize text locally with the lightweight Contextionary model. +- [img2vec-neural](./img2vec-neural.md): Vectorize images locally with a `resnet50` model. +- [ref2vec-centroid](./ref2vec-centroid.md): Calculate an object's vector from the centroid of its referenced objects' vectors. ## Related pages diff --git a/docs/weaviate/modules/ner-transformers.md b/docs/weaviate/modules/ner-transformers.md index a215172ce..aa185fe04 100644 --- a/docs/weaviate/modules/ner-transformers.md +++ b/docs/weaviate/modules/ner-transformers.md @@ -57,7 +57,7 @@ services: EXTENSIONS_STORAGE_ORIGIN: http://weaviate:8080 NEIGHBOR_OCCURRENCE_IGNORE_PERCENTILE: 5 ENABLE_COMPOUND_SPLITTING: 'false' - image: cr.weaviate.io/semitechnologies/contextionary:en0.16.0-v1.0.2 + image: cr.weaviate.io/semitechnologies/contextionary:en0.16.0-v1.2.1 ports: - 9999:9999 ner-transformers: diff --git a/docs/weaviate/modules/qna-openai.md b/docs/weaviate/modules/qna-openai.md index 166c3ade3..1986a1a2c 100644 --- a/docs/weaviate/modules/qna-openai.md +++ b/docs/weaviate/modules/qna-openai.md @@ -47,12 +47,12 @@ For requests that require the OpenAI organization name, you can provide it at qu You can provide your API key in two ways: -1. During the **configuration** of your Docker instance, by adding `OPENAI_API_KEY` or `AZURE_API_KEY` as appropriate under `environment` to your `Docker Compose` file, like this: +1. During the **configuration** of your Docker instance, by adding `OPENAI_APIKEY` or `AZURE_APIKEY` as appropriate under `environment` to your `Docker Compose` file, like this: ```yaml environment: - OPENAI_API_KEY: 'your-key-goes-here' # For use with OpenAI. Setting this parameter is optional; you can also provide the key at runtime. - AZURE_API_KEY: 'your-key-goes-here' # For use with Azure OpenAI. Setting this parameter is optional; you can also provide the key at runtime. + OPENAI_APIKEY: 'your-key-goes-here' # For use with OpenAI. Setting this parameter is optional; you can also provide the key at runtime. + AZURE_APIKEY: 'your-key-goes-here' # For use with Azure OpenAI. Setting this parameter is optional; you can also provide the key at runtime. ... ``` @@ -100,9 +100,9 @@ services: AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' PERSISTENCE_DATA_PATH: '/var/lib/weaviate' ENABLE_MODULES: 'text2vec-openai,qna-openai' - OPENAI_API_KEY: sk-foobar # For use with OpenAI. Setting this parameter is optional; you can also provide the key at runtime. + OPENAI_APIKEY: sk-foobar # For use with OpenAI. Setting this parameter is optional; you can also provide the key at runtime. OPENAI_ORGANIZATION: your-orgname # For use with OpenAI. Setting this parameter is optional; you can also provide the key at runtime. - AZURE_API_KEY: sk-foobar # For use with Azure OpenAI. Setting this parameter is optional; you can also provide the key at runtime. + AZURE_APIKEY: sk-foobar # For use with Azure OpenAI. Setting this parameter is optional; you can also provide the key at runtime. CLUSTER_HOSTNAME: 'node1' ``` diff --git a/docs/weaviate/modules/ref2vec-centroid.md b/docs/weaviate/modules/ref2vec-centroid.md index dddbe55c5..83b68b250 100644 --- a/docs/weaviate/modules/ref2vec-centroid.md +++ b/docs/weaviate/modules/ref2vec-centroid.md @@ -23,7 +23,7 @@ Which modules to use in a Weaviate instance can be specified in the `Docker Comp ```yaml --- -services:html +services: weaviate: command: - --host diff --git a/docs/weaviate/modules/spellcheck.md b/docs/weaviate/modules/spellcheck.md index 36fbf5cff..ba00222ea 100644 --- a/docs/weaviate/modules/spellcheck.md +++ b/docs/weaviate/modules/spellcheck.md @@ -55,7 +55,7 @@ services: EXTENSIONS_STORAGE_ORIGIN: http://weaviate:8080 NEIGHBOR_OCCURRENCE_IGNORE_PERCENTILE: 5 ENABLE_COMPOUND_SPLITTING: 'false' - image: cr.weaviate.io/semitechnologies/contextionary:en0.16.0-v1.0.2 + image: cr.weaviate.io/semitechnologies/contextionary:en0.16.0-v1.2.1 ports: - 9999:9999 text-spellcheck: diff --git a/docs/weaviate/modules/sum-transformers.md b/docs/weaviate/modules/sum-transformers.md index 62bdbf407..7b9f587b8 100644 --- a/docs/weaviate/modules/sum-transformers.md +++ b/docs/weaviate/modules/sum-transformers.md @@ -76,7 +76,7 @@ services: EXTENSIONS_STORAGE_ORIGIN: http://weaviate:8080 NEIGHBOR_OCCURRENCE_IGNORE_PERCENTILE: 5 ENABLE_COMPOUND_SPLITTING: 'false' - image: cr.weaviate.io/semitechnologies/contextionary:en0.16.0-v1.0.2 + image: cr.weaviate.io/semitechnologies/contextionary:en0.16.0-v1.2.1 ports: - 9999:9999 sum-transformers: diff --git a/docs/weaviate/more-resources/example-datasets.md b/docs/weaviate/more-resources/example-datasets.md index aabf0fdbc..0d7a6d6bb 100644 --- a/docs/weaviate/more-resources/example-datasets.md +++ b/docs/weaviate/more-resources/example-datasets.md @@ -105,7 +105,7 @@ export CACHE_DIR= # Optionally you can set the batch size (if not specified by default 200) export BATCH_SIZE= # Make sure to replace WEAVIATE_ORIGIN with the Weaviate origin as mentioned in the basics above -docker run -it -e weaviate_host=$WEAVIATE_ORIGIN -e cache_dir-$CACHE_DIR -e batch_size=$BATCH_SIZE semitechnologies/weaviate-demo-newspublications:latest +docker run -it -e weaviate_host=$WEAVIATE_ORIGIN -e cache_dir=$CACHE_DIR -e batch_size=$BATCH_SIZE semitechnologies/weaviate-demo-newspublications:latest ``` @@ -125,7 +125,7 @@ export CACHE_DIR= # Optionally you can set the batch size (if not specified by default 200) export BATCH_SIZE= # Run docker -docker run -it --network=$WEAVIATE_NETWORK -e weaviate_host=$WEAVIATE_ORIGIN -e cache_dir-$CACHE_DIR -e batch_size=$BATCH_SIZE semitechnologies/weaviate-demo-newspublications:latest +docker run -it --network=$WEAVIATE_NETWORK -e weaviate_host=$WEAVIATE_ORIGIN -e cache_dir=$CACHE_DIR -e batch_size=$BATCH_SIZE semitechnologies/weaviate-demo-newspublications:latest ``` ## Questions and feedback diff --git a/docs/weaviate/more-resources/example-use-cases.md b/docs/weaviate/more-resources/example-use-cases.md index e68cad09a..35f046e32 100644 --- a/docs/weaviate/more-resources/example-use-cases.md +++ b/docs/weaviate/more-resources/example-use-cases.md @@ -37,7 +37,6 @@ Vector databases help to address some of large language models (LLMs) limitation |Title | Description | Modality | Code | | --- | --- | --- | --- | - | Verba, the golden RAGtriever ([Video](https://www.youtube.com/watch?v=OSt3sFT1i18)) | Retrieval-Augmented Generation (RAG) system to chat with Weaviate documentation and blog posts. | Text | [Python](https://github.com/weaviate/Verba) | | HealthSearch ([Blog](https://weaviate.io/blog/healthsearch-demo)) | Recommendation system of health products based on symptoms. | Text | [Python](https://github.com/weaviate/healthsearch-demo) | | Magic Chat | Search through Magic The Gathering cards | Text | [Python](https://github.com/weaviate/st-weaviate-connection/tree/main) | @@ -54,7 +53,7 @@ Weaviate can leverage its vectorization capabilities to enable automatic, real-t |Title | Description | Modality | Code | | --- | --- | --- | --- | | Toxic Comment Classification | Classify whether a comment is toxic or non-toxic. | Text | [Python](https://github.com/weaviate-tutorials/DEMO-classification-toxic-comment) | -| Audio Genre Classification | Classify the music genre of an audio file. | Image | [Python](https://github.com/weaviate-tutorials/DEMO-classification-audio-genre/) | +| Audio Genre Classification | Classify the music genre of an audio file. | Audio | [Python](https://github.com/weaviate-tutorials/DEMO-classification-audio-genre/) | ## Other use cases @@ -62,7 +61,7 @@ Weaviate's [modular ecosystem](../modules/index.md) unlocks many other use cases |Title | Description | Code | | --- | --- | --- | -| Named Entity Recognition (NER)| tbd | [Python](https://github.com/weaviate/weaviate-examples/tree/main/example-with-NER-module) | +| Named Entity Recognition (NER)| Extract named entities, such as people, organizations and locations, from text stored in Weaviate. | [Python](https://github.com/weaviate/weaviate-examples/tree/main/example-with-NER-module) | ## Questions and feedback diff --git a/docs/weaviate/more-resources/faq.md b/docs/weaviate/more-resources/faq.md index 01fd0cb98..9e385c719 100644 --- a/docs/weaviate/more-resources/faq.md +++ b/docs/weaviate/more-resources/faq.md @@ -541,6 +541,31 @@ client.collections.create( +#### Q: Why does `insert_many` fail with a "message larger than max" (`RESOURCE_EXHAUSTED`) error? + +
+ Answer + +`insert_many` (Python) and `insertMany` (TypeScript, Java) send all objects in a **single gRPC request**. Requests larger than the server's [`GRPC_MAX_MESSAGE_SIZE`](/deploy/configuration/env-vars/index.md#GRPC_MAX_MESSAGE_SIZE) limit are rejected with the gRPC status `RESOURCE_EXHAUSTED`, so a sufficiently large list fails as a whole with an error similar to: + +```text +WeaviateBatchError: Query call with protocol GRPC batch failed with message +CLIENT: Sent message larger than max (3002340 vs. 1000000). +``` + +The two numbers are the size of your request and the server's limit (the values shown here are from a test with a 1 MB limit). Recent Python clients read the server's limit at connect time and reject oversized requests before sending them. With older clients, the server-side variant `grpc: received message larger than max` may appear instead. + +For large lists, use [server-side batching](../manage-objects/import.mdx#server-side-batching) instead, which splits the data into server-paced batches: + +- **Python**: Use `collection.data.ingest(objects)`, a drop-in replacement for `insert_many` that returns the same return object. Alternatively, use the `collection.batch.stream()` context manager. +- **TypeScript**: `collection.data.ingest(objects)`. +- **Java** (`v6`): the `collection.batch.start()` streaming context. +- **C#**: `collection.Batch.InsertMany(items)` (already uses server-side batching). + +Alternatively, you can raise the `GRPC_MAX_MESSAGE_SIZE` [environment variable](/deploy/configuration/env-vars/index.md#GRPC_MAX_MESSAGE_SIZE) on the server, but batching is the recommended solution. + +
+ ## Miscellaneous #### Q: Can I request a feature in Weaviate? @@ -650,7 +675,7 @@ If you need resources from the previous version of Weaviate Academy, check out t
Answer -> The Weaviate Community Slack has been decommissioned. We've moved community discussions to the [Weaviate Community Forum](https://forum.weaviate.io/), which offers better long-term discoverability — conversations are indexed and searchable, so valuable answers don't get lost over time. +> The Weaviate Community Slack has been decommissioned. We've moved community discussions to the [Weaviate Community Forum](https://forum.weaviate.io/), which offers better long-term discoverability: conversations are indexed and searchable, so valuable answers don't get lost over time. > > Join us at [forum.weaviate.io](https://forum.weaviate.io/) to ask questions, share ideas, and connect with the community. For private support inquiries, you can reach us at [support@weaviate.io](mailto:support@weaviate.io). @@ -663,8 +688,8 @@ If you need resources from the previous version of Weaviate Academy, check out t > Yes, Weaviate provides two MCP (Model Context Protocol) servers: > -> - **[Weaviate MCP server](/weaviate/configuration/mcp-server.mdx)** — Built into Weaviate itself. Exposes tools for inspecting schemas, searching data (vector/hybrid), and modifying objects. Runs on the same port as the REST API at `/v1/mcp`. Disabled by default — enable with `MCP_SERVER_ENABLED=true`. -> - **[Weaviate Docs MCP server](/weaviate/mcp/docs-mcp-server.mdx)** — A standalone server that gives LLMs access to Weaviate's documentation. Useful for AI-assisted development with Weaviate. +> - **[Weaviate MCP server](/weaviate/configuration/mcp-server.mdx)**: Built into Weaviate itself. Exposes tools for inspecting schemas, searching data (vector/hybrid), and modifying objects. Runs on the same port as the REST API at `/v1/mcp`. Disabled by default. Enable it with `MCP_SERVER_ENABLED=true`. +> - **[Weaviate Docs MCP server](/weaviate/mcp/docs-mcp-server.mdx)**: A standalone server that gives LLMs access to Weaviate's documentation. Useful for AI-assisted development with Weaviate. > > Both servers use the Streamable HTTP transport and work with MCP clients like Claude Code, Claude Desktop, Cursor, and VS Code. diff --git a/docs/weaviate/more-resources/performance.md b/docs/weaviate/more-resources/performance.md index e7ba6e1d7..fa89d89ce 100644 --- a/docs/weaviate/more-resources/performance.md +++ b/docs/weaviate/more-resources/performance.md @@ -26,7 +26,7 @@ Inverted indexes are used often in document retrieval systems and search engines The inverted index currently does not do any weighing (e.g. tf-idf) for sorting, since the vector index is used for these features like sorting. The inverted index is thus, at the moment, rather a binary operation: including or excluding data objects from the query result list, which results in an 'allow list'. ## Vector index -Everything that has a vector, thus every data object in Weaviate, is also indexed in the vector index. Weaviate currently supports [HNSW](https://arxiv.org/abs/1603.09320) and flat vector indexes. +Everything that has a vector, thus every data object in Weaviate, is also indexed in the vector index. Weaviate supports several vector index types, which trade off search speed, recall and resource use in different ways. The default is [HNSW](https://arxiv.org/abs/1603.09320). See [Concepts: vector index](../concepts/indexing/vector-index.md) for more information about the vector index. @@ -54,7 +54,7 @@ A tip is to avoid deeply nested filters in the queries. Additionally, try to mak ## Profiling query performance -To diagnose slow queries, use [query profiling](/weaviate/search/query-profile.md) to get per-shard timing breakdowns. This shows exactly how long each phase takes — vector search, keyword scoring, filter evaluation, object retrieval — broken down by shard and cluster node. +To diagnose slow queries, use [query profiling](/weaviate/search/query-profile.md) to get per-shard timing breakdowns. This shows exactly how long each phase takes (vector search, keyword scoring, filter evaluation, and object retrieval), broken down by shard and cluster node. ## Questions and feedback diff --git a/docs/weaviate/recipes.mdx b/docs/weaviate/recipes.mdx index c5c4bdcd6..75e4345f9 100644 --- a/docs/weaviate/recipes.mdx +++ b/docs/weaviate/recipes.mdx @@ -3,7 +3,7 @@ title: Weaviate Recipes (code examples) hide_table_of_contents: true --- -This page contains recipes for common Weaviate operations. You can also check out the original [Jupyter Notebooks](https://github.com/weaviate/recipes) the recipes where created from. +This page contains recipes for common Weaviate operations. You can also check out the original [Jupyter Notebooks](https://github.com/weaviate/recipes) from which the recipes were created. import RecipesCards from "@site/src/components/RecipesCards"; diff --git a/docs/weaviate/recipes/multi-vector-colipali-rag.md b/docs/weaviate/recipes/multi-vector-colipali-rag.md index e2189ed69..1468bd07d 100644 --- a/docs/weaviate/recipes/multi-vector-colipali-rag.md +++ b/docs/weaviate/recipes/multi-vector-colipali-rag.md @@ -444,7 +444,7 @@ Python output: True ``` -For this tutorial, you will need the Weaviate `v1.29.0` or higher. +This tutorial requires Weaviate `v1.29.0` and later. Let's make sure we have the required version: ```python @@ -578,16 +578,6 @@ As an example of what we are going to build, consider the following actual demo query = "How does DeepSeek-V2 compare against the LLaMA family of LLMs?" ``` -Python output: - -```text -Running cells with 'Python 3.13.5' requires the ipykernel package. - -Install 'ipykernel' into the Python environment. - -Command: '/opt/homebrew/bin/python3 -m pip install ipykernel -U --user --force-reinstall' -``` - By inspecting the first page of the [DeepSeek-V2 paper](https://arxiv.org/abs/2405.04434), we see that it does indeed contain a figure that is relevant for answering our query: diff --git a/docs/weaviate/recipes/rag_titan-text-express-v1_bedrock.md b/docs/weaviate/recipes/rag_titan-text-express-v1_bedrock.md index 9ef947bc9..d735b7d2c 100644 --- a/docs/weaviate/recipes/rag_titan-text-express-v1_bedrock.md +++ b/docs/weaviate/recipes/rag_titan-text-express-v1_bedrock.md @@ -7,7 +7,7 @@ integration: False agent: False tags: ['Generative Search', 'RAG', 'AWS'] --- -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/weaviate/recipes/weaviate-features/model-providers/aws/rag_titan-text-express-v1_bedrock.ipynb) +[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/weaviate/recipes/blob/main/weaviate-features/model-providers/aws/rag_titan-text-express-v1_bedrock.ipynb) ## Dependencies @@ -57,7 +57,7 @@ if (client.collections.exists("JeopardyQuestion")): client.collections.create( name="JeopardyQuestion", - vectorizer_config=wc.Configure.Vectorizer.text2vec_aws( + vector_config=wc.Configure.Vectors.text2vec_aws( service="bedrock", #this is crucial model="cohere.embed-english-v3", # select the model, make sure it is enabled for your account # model="amazon.titan-embed-text-v1", # select the model, make sure it is enabled for your account diff --git a/docs/weaviate/recipes/weaviate_embeddings_service.md b/docs/weaviate/recipes/weaviate_embeddings_service.md index e9f70b66d..2e2b62fc2 100644 --- a/docs/weaviate/recipes/weaviate_embeddings_service.md +++ b/docs/weaviate/recipes/weaviate_embeddings_service.md @@ -8,7 +8,7 @@ agent: False tags: ["Weaviate Embeddings", "Weaviate Cloud"] --- -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/weaviate/recipes/weaviate-services/embedding-service/weaviate_embeddings_service.ipynb) +[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/weaviate/recipes/blob/main/weaviate-services/embedding-service/weaviate_embeddings_service.ipynb) # Weaviate Embedding Service diff --git a/docs/weaviate/release-notes/index.md b/docs/weaviate/release-notes/index.md index ca97aaac2..1bbd63dd4 100644 --- a/docs/weaviate/release-notes/index.md +++ b/docs/weaviate/release-notes/index.md @@ -17,6 +17,11 @@ import QuickLinks from "/src/components/QuickLinks"; export const pythonCardsData = [ { +title: "v1.39", +link: "https://github.com/weaviate/weaviate/releases/tag/v1.39.0", +icon: "fa fa-tags", +}, +{ title: "v1.38", link: "https://github.com/weaviate/weaviate/releases/tag/v1.38.0", icon: "fa fa-tags", @@ -36,11 +41,6 @@ title: "v1.35", link: "https://weaviate.io/blog/weaviate-1-35-release", icon: "fa fa-tags", }, -{ -title: "v1.34", -link: "https://weaviate.io/blog/weaviate-1-34-release", -icon: "fa fa-tags", -}, ]; diff --git a/docs/weaviate/release-notes/known-issues.mdx b/docs/weaviate/release-notes/known-issues.mdx index cd34aaf11..75e193c09 100644 --- a/docs/weaviate/release-notes/known-issues.mdx +++ b/docs/weaviate/release-notes/known-issues.mdx @@ -18,7 +18,7 @@ This page documents significant known issues in Weaviate, their symptoms, and re | Issue | Affected Versions | Resolution | Fixed In | | ------------------------------------------------------------------------------------- | ------------------------------------------------------------------- | ------------- | -------------------------------- | -| [Empty collections panic](#empty-collections-panic) | 1.28 - 1.31 | In Progress | - | +| [Empty collections panic](#empty-collections-panic) | 1.28 (all), 1.29.0-8, 1.30.0-8, 1.31.0, 1.31.1 | Fixed | 1.29.9+, 1.30.9+, 1.31.2+, 1.32.0+ | | [Database restoration blocked](#database-restoration-blocked) | 1.27.23-26, 1.28.14-15, 1.29.5-7, 1.30.3 | Fixed | 1.27.27, 1.28.16, 1.29.8, 1.30.4 | | [RAFT snapshot compatibility on downgrade](#raft-snapshot-compatibility-on-downgrade) | 1.28.13+, 1.29.5+, 1.30.2+ (when downgrading to 1.27.25 or earlier) | Fixed | 1.27.26 | | [RAFT bootstrap timeout](#raft-bootstrap-timeout) | 1.25 - 1.28 | Workaround | - | @@ -426,8 +426,8 @@ echo "vm.max_map_count=8388608" >> /etc/sysctl.conf :::info Impact summary -- **Affected versions:** 1.28, 1.29, 1.30, 1.31 -- **Resolution:** In progress +- **Affected versions:** 1.28 (all patch releases), 1.29.0 - 1.29.8, 1.30.0 - 1.30.8, 1.31.0 and 1.31.1 +- **Resolution:** Fixed in 1.29.9, 1.30.9, 1.31.2 and 1.32.0 ::: @@ -447,12 +447,23 @@ RAFT snapshots with no collections (previously called classes) cause a nil map a #### Resolution -**Option 1: Upgrade (when available)** -Upgrade to patched version containing the fix. +**Option 1: Upgrade (recommended)** + +Upgrade to a release that contains the fix: + +- 1.29.9 and later 1.29 releases +- 1.30.9 and later 1.30 releases +- 1.31.2 and later 1.31 releases +- 1.32.0 and later + +Two points to watch when you choose a target release: + +- 1.31.0 and 1.31.1 do **not** contain the fix. Within the 1.31 line, upgrade to 1.31.2 or later. +- The 1.28 line was never patched. If you run 1.28, upgrade to one of the releases listed above. **Option 2: Remove empty snapshot** -Identify and remove the problematic snapshot: +If you cannot upgrade yet, identify and remove the problematic snapshot: ```bash # Navigate to RAFT directory diff --git a/docs/weaviate/search/basics.md b/docs/weaviate/search/basics.md index 7560d9825..6e2d29d1f 100644 --- a/docs/weaviate/search/basics.md +++ b/docs/weaviate/search/basics.md @@ -644,7 +644,6 @@ import QueryReplication from '/\_includes/code/replication.get.object.by.id.mdx' - [Connect to Weaviate](/weaviate/connections) - [API References: GraphQL: Get](../api/graphql/get.md) -- For tutorials, see [Queries](/weaviate/tutorials/query.md) - For search using the GraphQL API, see [GraphQL API](../api/graphql/get.md) ## Questions and feedback diff --git a/docs/weaviate/search/bm25.md b/docs/weaviate/search/bm25.md index 8acf000c8..0f24fd76d 100644 --- a/docs/weaviate/search/bm25.md +++ b/docs/weaviate/search/bm25.md @@ -99,7 +99,7 @@ import SearchOperators from '/_includes/feature-notes/search-operators.mdx'; -Search operators define the minimum number of query [tokens](#set-tokenization) that must be present within a single searched property for an object to be returned. The options are `and`, or `or` (default). +Search operators define how many of the query [tokens](#set-tokenization) must match, and whether they must all match within a single searched property. The options are `or` (default), `and`, and `and_cross`. ### `or` @@ -179,6 +179,48 @@ With the `and` operator, the search returns objects where all tokens in the sear +### `and_cross` + +:::info Added in `v1.38.8` +::: + +With the `and_cross` operator, every token in the search string must be matched by at least one of the searched properties, but the tokens do not all have to occur in the same property. An object whose title matches one token and whose body matches the rest is a match for `and_cross`, and is not a match for `and`. + +Because it relaxes the single-property requirement, `and_cross` returns every object that `and` returns, and usually more. + +:::caution All searched properties must be configured alike + +`and_cross` requires every searched property to share the same tokenization and the same analyzer settings, which means the tokenizer, accent folding and its exceptions, and the stopword preset. If they differ, the query fails with an error instead of returning fewer results: + +``` +OPERATOR_AND_CROSS requires all searched properties to share the same tokenization and analyzer settings +``` + +If a collection mixes tokenizations, restrict the search to a compatible set of properties. See [Search on selected properties only](#search-on-selected-properties-only). + +::: + +The examples below use Python and GraphQL, because `and_cross` is currently available in the Python client and not yet in the TypeScript, Go, Java, or C# clients. + + + + + + + + + + ## Retrieve BM25F scores You can retrieve the BM25F `score` values for each returned object. @@ -454,8 +496,8 @@ See [Inverted index: Accent folding](../concepts/indexing/inverted-index.md#acce By default, Weaviate filters out common English stopwords (like "a", "the", "is") from BM25 scoring. You can customize this behavior: -- **Custom presets**: Define named stopword lists per collection via `invertedIndexConfig.stopwordPresets` — useful for non-English languages or domain-specific terms. -- **Per-property overrides**: Assign different stopword presets to individual properties via `textAnalyzer.stopwordPreset` — useful for multilingual collections where each property contains text in a different language. +- **Custom presets**: Define named stopword lists per collection via `invertedIndexConfig.stopwordPresets`, useful for non-English languages or domain-specific terms. +- **Per-property overrides**: Assign different stopword presets to individual properties via `textAnalyzer.stopwordPreset`, useful for multilingual collections where each property contains text in a different language. Stopwords are still indexed and only filtered at query time, so changing the configuration does not require reindexing. @@ -769,7 +811,7 @@ Set the tokenization method to `trigram` at the property level when creating you -Keyword (BM25) queries accept an optional `boost` argument that promotes or demotes matching documents without removing them — useful for biasing results by recency, popularity, a soft filter, or another property. Matching documents move up. Everything else stays in the results but ranks lower. +Keyword (BM25) queries accept an optional `boost` argument that promotes or demotes matching documents without removing them. This is useful for biasing results by recency, popularity, a soft filter, or another property. Matching documents move up. Everything else stays in the results but ranks lower. See [Boost](./boost.md) for the supported condition types (filter, property value, time decay, numeric decay), curve choices, blending semantics, and depth tuning. diff --git a/docs/weaviate/search/boost.md b/docs/weaviate/search/boost.md index 3cf663f28..cb6c3404e 100644 --- a/docs/weaviate/search/boost.md +++ b/docs/weaviate/search/boost.md @@ -2,7 +2,7 @@ title: Boost sidebar_position: 76 image: og/docs/howto.jpg -description: Soft-rank vector, hybrid, and BM25 results — promote or demote matching documents without filtering them out. Worked Python examples for filter, property, time-decay, numeric-decay, and blended boosts. +description: Soft-rank vector, hybrid, and BM25 results by promoting or demoting matching documents without filtering them out. Worked Python examples for filter, property, time-decay, numeric-decay, and blended boosts. # tags: ['boost', 'search', 'ranking'] --- @@ -14,16 +14,18 @@ import BoostNote from '/_includes/feature-notes/boost.mdx'; -**Boost** soft-ranks search results — promote or demote matching documents without removing them from the result set. Matching documents move up. Non-matching documents stay in the results but rank lower. +**Boost** soft-ranks search results: it promotes or demotes matching documents without removing them from the result set. Matching documents move up. Non-matching documents stay in the results but rank lower. Apply boost to vector, hybrid, BM25, near-text, near-vector, near-object, near-image, and near-media queries. +The examples on this page are Python only, because `boost` is currently available in the Python client and not yet in the TypeScript, Go, Java, or C# clients. + ## How it works A boost is a **post-retrieval rescorer**: 1. The primary search (vector, hybrid, BM25, ...) fetches `depth` candidate results. Set `depth` higher than `offset + limit` if you want boost to consider candidates beyond the first page. -2. The boost scorer rescores those candidates in memory by evaluating each condition per candidate, normalizing per result set, and blending with the primary score. There are no new index queries — the cost is per-candidate in-memory scoring, not extra shard fan-out. Both primary and boost scores are min-max normalized into `[0, 1]` before blending, and the final score is renormalized to `[0, 1]`. +2. The boost scorer rescores those candidates in memory by evaluating each condition per candidate, normalizing per result set, and blending with the primary score. There are no new index queries. The cost is per-candidate in-memory scoring, not extra shard fan-out. Both primary and boost scores are min-max normalized into `[0, 1]` before blending, and the final score is renormalized to `[0, 1]`. 3. The user's original `offset` and `limit` are applied **after** the re-sort. ## Condition types @@ -96,9 +98,9 @@ Continuous `[0, 1]` score that **decays with distance from an origin time**. The ### Numeric decay {#numeric-decay} -Like time decay but for numeric (`int`, `number`) properties. Use this when "closer to X is better" — prices near a target, distances near a coordinate, ages near a band. Same `scale` / `offset` / `decay` / `curve` semantics as time decay, with all values expressed as numbers. +Like time decay but for numeric (`int`, `number`) properties. Use this when "closer to X is better": prices near a target, distances near a coordinate, ages near a band. Same `scale` / `offset` / `decay` / `curve` semantics as time decay, with all values expressed as numbers. -`scale` must be `> 0` and `decay` (if set) must be in `(0, 1]` — same as time decay. +`scale` must be `> 0` and `decay` (if set) must be in `(0, 1]`, same as time decay. @@ -117,11 +119,11 @@ The three decay curves shape how score falls off with distance. At `distance == | Curve | Shape | When to use | |---|---|---| -| `EXPONENTIAL` (default) | Heavy tail — score halves geometrically every `scale` past the origin. | "Recency matters, but don't aggressively flatten older items to zero." | -| `GAUSSIAN` | Bell curve — sharp falloff past `scale`. | "Items close to the origin are great, items far away are nearly worthless." | -| `LINEAR` | Straight line — score reaches zero at a finite distance past `scale`. | "Predictable falloff with a clear cutoff." | +| `EXPONENTIAL` (default) | Heavy tail: score halves geometrically every `scale` past the origin. | "Recency matters, but don't aggressively flatten older items to zero." | +| `GAUSSIAN` | Bell curve: sharp falloff past `scale`. | "Items close to the origin are great, items far away are nearly worthless." | +| `LINEAR` | Straight line: score reaches zero at a finite distance past `scale`. | "Predictable falloff with a clear cutoff." | -Only these three values are accepted — anything else is rejected at request time. +Only these three values are accepted. Anything else is rejected at request time. ## Blending and weights @@ -131,10 +133,10 @@ A boost must carry **at least one** and **at most 20** conditions. Use `Boost.bl final_score = (1 − weight) · primary_norm + weight · boost_norm ``` -- **`weight`** — the outer blending weight, in `[0, 1]`. Defaults to `0.5`. +- **`weight`**: the outer blending weight, in `[0, 1]`. Defaults to `0.5`. - **`weight: 0`** is a no-op: the boost short-circuits and primary results are returned unchanged. -- **Per-condition `weight`** — a `float` defaulting to `1.0`. Use it to balance multiple boosts ("recency twice as important as popularity"). -- **Negative per-condition `weight`** — *demotes* matching documents. They stay in the result set but rank lower than non-matching ones. +- **Per-condition `weight`**: a `float` defaulting to `1.0`. Use it to balance multiple boosts ("recency twice as important as popularity"). +- **Negative per-condition `weight`**: *demotes* matching documents. They stay in the result set but rank lower than non-matching ones. @@ -149,7 +151,7 @@ final_score = (1 − weight) · primary_norm + weight · boost_norm ### Negative weights demote -A condition with `weight: -1.0` (or `-2.0`, etc.) reverses the effect: documents that match the condition rank below non-matching ones instead of above them. They are not removed. This is useful for deprioritizing — for example, surfacing drafts last without filtering them out. +A condition with `weight: -1.0` (or `-2.0`, etc.) reverses the effect: documents that match the condition rank below non-matching ones instead of above them. They are not removed. This is useful for deprioritizing (for example, surfacing drafts last without filtering them out). @@ -171,12 +173,12 @@ A condition with `weight: -1.0` (or `-2.0`, etc.) reverses the effect: documents | Default | `100` | | Operator override | [`QUERY_BOOST_DEFAULT_DEPTH`](/deploy/configuration/env-vars#QUERY_BOOST_DEFAULT_DEPTH) env var | | Hard cap | `QUERY_MAXIMUM_RESULTS` (cluster-wide limit) | -| Lower bound | At least `offset + limit` — boost always sees enough to fill the page | -| Accepted range | `≥ 0` — `0` means "use the default" | +| Lower bound | At least `offset + limit`, so boost always sees enough to fill the page | +| Accepted range | `≥ 0`, where `0` means "use the default" | :::tip Raise `depth` to reorder beyond the top page -Boost can only reorder what the primary search already retrieved. If your boost should reorder past the default top-100 — for example, to pull a popular older article back to page 1 — pass a higher `depth` on the query. +Boost can only reorder what the primary search already retrieved. If your boost should reorder past the default top-100 (for example, to pull a popular older article back to page 1), pass a higher `depth` on the query. Higher `depth` increases the primary search cost (BM25 / vector index work) and per-candidate scoring cost. Set it as tight as your use case allows. @@ -184,10 +186,10 @@ Higher `depth` increases the primary search cost (BM25 / vector index work) and ## Further resources -- [Rerank](./rerank.md) — second-stage reranking with an external model. -- [Hybrid search](./hybrid.md) — the BM25 / vector `alpha` blend. -- [Filters](./filters.md) — hard filters (remove non-matching docs). -- [BM25](./bm25.md) — keyword search. +- [Rerank](./rerank.md): second-stage reranking with an external model. +- [Hybrid search](./hybrid.md): the BM25 / vector `alpha` blend. +- [Filters](./filters.md): hard filters (remove non-matching docs). +- [BM25](./bm25.md): keyword search. ## Questions and feedback diff --git a/docs/weaviate/search/filters.md b/docs/weaviate/search/filters.md index 29ba28f5e..2c2b7c6c1 100644 --- a/docs/weaviate/search/filters.md +++ b/docs/weaviate/search/filters.md @@ -1069,7 +1069,7 @@ This filter requires the [property null state](../config-refs/indexing/inverted- :::caution Preview feature -Available from Weaviate `v1.38` as a preview, gated by the `WEAVIATE_PREVIEW_NESTED_FILTERING=on` environment variable on the server. The path syntax and operator semantics are stable, but the on-disk encoding may change before GA — don't rely on persistent state from preview clusters carrying over to the GA release. The env var is removed at GA and the feature is enabled unconditionally. +Available from Weaviate `v1.38` as a preview, gated by the `WEAVIATE_PREVIEW_NESTED_FILTERING=on` environment variable on the server. The path syntax and operator semantics are stable, but the on-disk encoding may change before GA. Don't rely on persistent state from preview clusters carrying over to the GA release. The env var is removed at GA and the feature is enabled unconditionally. ::: @@ -1094,7 +1094,7 @@ The filter property is a single dotted path. The dot is the only separator. An o | `cars.tires.width` | Any tire on any car (recursive across two `object[]` levels) | | `cars[1].tires[2].brand` | The second car's third tire's `brand` (positional through nesting) | -`[N]` on a segment requires that segment to be an `object[]` (array). Every intermediate segment must be `object` or `object[]` — you cannot pivot through a scalar. The leaf may be any supported scalar type. +`[N]` on a segment requires that segment to be an `object[]` (array). Every intermediate segment must be `object` or `object[]`. You cannot pivot through a scalar. The leaf may be any supported scalar type. ### Match any element (default) @@ -1128,7 +1128,7 @@ Use `[N]` to pin a path segment to a specific array index. Indices are 0-based. ### Same-element correlation across leaves -Combining two leaf filters with `And` matches when **the same element** in the parent array satisfies both. A document with one car `(Toyota, blue)` and another `(Honda, red)` would not match `cars.make = "Toyota" AND cars.color = "red"` — both conditions must hold on the **same** car. +Combining two leaf filters with `And` matches when **the same element** in the parent array satisfies both. A document with one car `(Toyota, blue)` and another `(Honda, red)` would not match `cars.make = "Toyota" AND cars.color = "red"`. Both conditions must hold on the **same** car. @@ -1176,7 +1176,7 @@ Pointing a path at an `object` or `object[]` segment (rather than a scalar leaf) :::note - **Allowed leaf data types**: `text`, `int`, `number`, `boolean`, `date`, `uuid`, and their array variants. `blob`, `blobHash`, `geoCoordinates`, `phoneNumber`, and cross-references (`cref`) are not allowed inside nested objects for nested filtering. -- **`IndexFilterable` is required**: nested filtering uses the filterable inverted index on each leaf. `IndexRangeFilters` and `IndexSearchable` flags exist on nested-property definitions but are not yet exercised by the nested searcher — range filters on nested numeric leaves currently use the filterable bucket. +- **`IndexFilterable` is required**: nested filtering uses the filterable inverted index on each leaf. `IndexRangeFilters` and `IndexSearchable` flags exist on nested-property definitions but are not yet exercised by the nested searcher. Range filters on nested numeric leaves currently use the filterable bucket. - **Tokenization matters**: nested `text` leaves use the same tokenization options as flat properties. For exact-match filters on names, codes, or identifiers, set `tokenization: field` on the leaf so the value is stored as a single token. - **Reference-path vs nested-path**: a reference-path filter is a multi-element `Path` (`["inCity", "City", "name"]`) traversing cross-references; a nested-path filter is a **single-element** path with dots inside it (`["cars.make"]`). diff --git a/docs/weaviate/search/generative.md b/docs/weaviate/search/generative.md index 025cb56ef..703674388 100644 --- a/docs/weaviate/search/generative.md +++ b/docs/weaviate/search/generative.md @@ -52,7 +52,7 @@ To use RAG with a [generative model integration](../model-providers/index.md): -```ts +```go // Go support coming soon ``` @@ -96,7 +96,7 @@ The second review is for the Stadt Krems 2009 Steinterrassen Riesling from Austr :::tip -For more information on the available modeld and their additional options, see the [model providers section](../model-providers/index.md). +For more information on the available models and their additional options, see the [model providers section](../model-providers/index.md). ::: ## Named vectors @@ -168,7 +168,7 @@ The second review is for the Stadt Krems 2009 Steinterrassen Riesling from Austr ## Single prompt search Single prompt search returns a generated response for each object in the query results.
-Define object `properties` – using `{prop-name}` syntax – to interpolate retrieved content in the prompt.
+Define object `properties` with the `{prop-name}` syntax to interpolate retrieved content in the prompt.
The properties you use in the prompt do not have to be among the properties you retrieve in the query. @@ -268,8 +268,8 @@ You can use *generative parameters* to specify additional options when performin @@ -338,24 +338,24 @@ Grouped task search returns one response that includes all of the query results. diff --git a/docs/weaviate/search/hybrid.md b/docs/weaviate/search/hybrid.md index 1a522821c..7f0f13176 100644 --- a/docs/weaviate/search/hybrid.md +++ b/docs/weaviate/search/hybrid.md @@ -16,6 +16,7 @@ import GoCode from '!!raw-loader!/\_includes/code/howto/go/docs/mainpkg/search-h import JavaV6Code from "!!raw-loader!/\_includes/code/java-v6/src/test/java/SearchHybridTest.java"; import CSharpCode from "!!raw-loader!/\_includes/code/csharp/SearchHybridTest.cs"; import GQLCode from '!!raw-loader!/\_includes/code/howto/search.hybrid.gql.py'; +import MMRPyCode from '!!raw-loader!/\_includes/code/howto/search.similarity.mmr.py'; import BoostNote from '/_includes/feature-notes/boost.mdx'; `Hybrid` search combines the results of a vector search and a keyword (BM25F) search by fusing the two result sets. @@ -232,6 +233,8 @@ Hybrid search results can favor the keyword component or the vector component. T - An `alpha` of `1` is a pure vector search. - An `alpha` of `0` is a pure keyword search. +If you do not set `alpha`, the effective weighting depends on your client. See [Alpha parameter](/weaviate/concepts/search/hybrid-search.md#alpha-parameter). + -For a discussion of fusion methods, see [this blog post](https://weaviate.io/blog/hybrid-search-fusion-algorithms) and [this reference page](../api/graphql/search-operators.md#variables-2) +For a discussion of fusion methods, see [this blog post](https://weaviate.io/blog/hybrid-search-fusion-algorithms) and [this reference page](../api/graphql/search-operators.md#fusion-algorithms).
@@ -384,7 +387,9 @@ import SearchOperators from '/_includes/feature-notes/search-operators.mdx'; -Keyword (BM25) search operators define the minimum number of query [tokens](#tokenization) that must be present within a single searched property for an object to be returned. The options are `and`, or `or` (default). +Keyword (BM25) search operators define how many of the query [tokens](#tokenization) must match, and whether they must all match within a single searched property. The options are `or` (default), `and`, and `and_cross`. + +The keyword leg of a hybrid query accepts the same operators as a standalone keyword search. For `and_cross`, which matches every token across the searched properties combined, see [BM25 search: `and_cross`](./bm25.md#and_cross). ### `or` @@ -1043,18 +1048,49 @@ import TokenizationNote from '/\_includes/tokenization.mdx' -Hybrid queries accept an optional `boost` argument that promotes or demotes matching documents without removing them — useful for biasing results by recency, popularity, a soft filter, or another property. +Hybrid queries accept an optional `boost` argument that promotes or demotes matching documents without removing them. This is useful for biasing results by recency, popularity, a soft filter, or another property. The boost runs once over the **fused** hybrid result. The BM25 and vector sub-search legs do not see the boost themselves. Hybrid's own `alpha` blend runs first, and the boost rescores the fused candidate pool on top. See [Boost](./boost.md) for the supported condition types (filter, property value, time decay, numeric decay), curve choices, blending semantics, and depth tuning. +## Diversity selection (MMR) + +:::info Added in `v1.38.6` +::: + +Hybrid search fuses a keyword result set and a vector result set, which often means the top of the fused list is a cluster of near-duplicates. **Maximum Marginal Relevance (MMR)** reranks that list to balance relevance with diversity, so that each selected object adds something new to the result set. + +Diversity selection runs after fusion. Both search legs run first, their results are fused with the configured `alpha` and fusion method, and the diversity pass then picks a diverse subset of the fused candidates. + +The examples in this section are Python only, because diversity selection is currently available in the Python client and not yet in the TypeScript, Go, Java, or C# clients. There is no GraphQL equivalent. + + + + + + + +Important notes: + +- **Top-level only**: set diversity selection on the hybrid query itself. Setting it on a sub-search is rejected with an error. +- **Two limits**: the query's top-level `limit` is the candidate window that gets diversified, and the diversity `limit` is the number of results returned. The diversity `limit` must be at least `1` and no larger than the query `limit`. +- **Ordering**: results come back in MMR order, not fused-score order. +- **Pagination**: `offset` moves the candidate window, so it must advance by the query `limit`, not by the number of returned objects. Weaviate does not validate this, and getting it wrong silently repeats some objects across pages while skipping others. See [Pagination](./similarity.md#pagination). +- **Not supported**: multi-vector collections. Weaviate rejects these queries with an error. + +For the parameters, the relevance and diversity trade-off, and vector search examples, see [Diversity selection (MMR)](./similarity.md#diversity-selection-mmr). + ## Related pages - [Connect to Weaviate](/weaviate/connections/index.mdx) - [API References: Search operators # Hybrid](../api/graphql/search-operators.md#hybrid) - About [hybrid fusion algorithms](https://weaviate.io/blog/hybrid-search-fusion-algorithms). -- For tutorials, see [Queries](/weaviate/tutorials/query.md) - For search using the GraphQL API, see [GraphQL API](../api/graphql/get.md). ## Questions and feedback diff --git a/docs/weaviate/search/image.md b/docs/weaviate/search/image.md index a574dc99d..16805a0f7 100644 --- a/docs/weaviate/search/image.md +++ b/docs/weaviate/search/image.md @@ -84,7 +84,7 @@ If your query image is stored in a file, you can use the client library to searc Example response -Query profiling provides per-shard timing breakdowns for search queries. Enable it on any search request to see how long each phase takes — vector search, keyword scoring, filter evaluation, object retrieval — broken down by shard and cluster node. +Query profiling provides per-shard timing breakdowns for search queries. Enable it on any search request to see how long each phase takes (vector search, keyword scoring, filter evaluation, object retrieval), broken down by shard and cluster node. Profiling uses the same instrumentation as [slow query logging](/deploy/configuration/logging.md#slow-query-logging). It adds minimal overhead when enabled and zero overhead when disabled. diff --git a/docs/weaviate/search/rerank.md b/docs/weaviate/search/rerank.md index df7b5b54d..c9f9ea311 100644 --- a/docs/weaviate/search/rerank.md +++ b/docs/weaviate/search/rerank.md @@ -201,7 +201,7 @@ The response should look like this: -For lightweight result reordering based on filters, property values, or time / numeric decay — without calling an external rerank model — use [Boost](./boost.md). Rerank and Boost can be used independently. Pick rerank when you need a smarter model to re-rank the top-N, and Boost when you want to bias by simple signals already on the objects. +For lightweight result reordering based on filters, property values, or time / numeric decay (without calling an external rerank model), use [Boost](./boost.md). Rerank and Boost can be used independently. Pick rerank when you need a smarter model to re-rank the top-N, and Boost when you want to bias by simple signals already on the objects. ## Related pages diff --git a/docs/weaviate/search/similarity.md b/docs/weaviate/search/similarity.md index 5a15583ec..ebabf4fcd 100644 --- a/docs/weaviate/search/similarity.md +++ b/docs/weaviate/search/similarity.md @@ -673,9 +673,9 @@ import V137Preview from '/\_includes/feature-notes/v137-preview.mdx'; -Standard vector search returns the closest matches to the query, which often means a cluster of near-duplicate results. **Maximum Marginal Relevance (MMR)** reranks results to balance relevance with diversity — each selected result must add something new to the result set. +Standard vector search returns the closest matches to the query, which often means a cluster of near-duplicate results. **Maximum Marginal Relevance (MMR)** reranks results to balance relevance with diversity: each selected result must add something new to the result set. -Add the `selection` parameter to any vector search query: +Add the `diversity_selection` parameter to any vector search query: + Candidate window, worked example, and deep pages + +A diversified query works with two limits: + +- the query's top-level `limit` is the **candidate window** that gets diversified, and +- the diversity `limit` is the **page size**, or how many objects come back. + +Each page is taken from the slice `[offset, offset + limit)` of the relevance-ranked results, so `offset` must advance by the query `limit`: + + + +The usual pagination idiom, adding the number of returned objects to `offset`, is the failing case here. The page size is always smaller than or equal to the candidate window, so consecutive windows overlap. With a query `limit` of `10` and a diversity `limit` of `3`, advancing `offset` by `3` reads the windows `[0:10]`, `[3:13]`, and `[6:16]`, which is how objects come back twice while others are skipped. + +When diversity selection is combined with a boost or with hybrid search, deep pages are also not stable slices of one fixed ranking. Both the boost pool and the two hybrid search legs fetch more candidates as `offset` grows, so which objects can reach a given page depends on how deep you have paged. + + + ## Soft-rank with Boost -Vector search queries accept an optional `boost` argument that promotes or demotes matching documents without removing them — useful for biasing results by recency, popularity, a soft filter, or another property. Matching documents move up. Everything else stays in the results but ranks lower. +Vector search queries accept an optional `boost` argument that promotes or demotes matching documents without removing them. This is useful for biasing results by recency, popularity, a soft filter, or another property. Matching documents move up. Everything else stays in the results but ranks lower. See [Boost](./boost.md) for the supported condition types (filter, property value, time decay, numeric decay), curve choices, blending semantics, and depth tuning. @@ -729,7 +756,6 @@ See [Boost](./boost.md) for the supported condition types (filter, property valu - [Connect to Weaviate](/weaviate/connections/index.mdx) - For image search, see [Image search](/weaviate/search/image). -- For tutorials, see [Queries](/weaviate/tutorials/query.md). - For search using the GraphQL API, see [GraphQL API](/weaviate/api). ## Questions and feedback diff --git a/docs/weaviate/starter-guides/custom-vectors.mdx b/docs/weaviate/starter-guides/custom-vectors.mdx index a1e5e11c9..b5d590350 100644 --- a/docs/weaviate/starter-guides/custom-vectors.mdx +++ b/docs/weaviate/starter-guides/custom-vectors.mdx @@ -71,13 +71,13 @@ Weaviate is open source. You can [run Weaviate](/deploy/index.mdx) locally, in t ### Client library -[Client libraries](/weaviate/client-libraries) simplify working with Weaviate. Clients are available for multiple programming languages. This guide provides examples in Python, Typescript, and cURL. +[Client libraries](/weaviate/client-libraries) simplify working with Weaviate. Clients are available for multiple programming languages. This guide provides examples in Python, TypeScript, Java, C#, and cURL. To install a client library, use the installer for the client language: -The v4 client requires Weaviate 1.23.7 or higher.

+The Python client requires Weaviate `v1.23.7` and later.

```bash pip install -U weaviate-client @@ -101,6 +101,13 @@ npm install weaviate-client ``` +
+ + +```bash +dotnet add package Weaviate.Client --version ||site.csharp_client_version|| +``` +
diff --git a/docs/weaviate/starter-guides/generative.md b/docs/weaviate/starter-guides/generative.md index 923b20916..f75d97a3a 100644 --- a/docs/weaviate/starter-guides/generative.md +++ b/docs/weaviate/starter-guides/generative.md @@ -349,7 +349,7 @@ For configurable deployments, you can specify enabled modules. For example, in a services: weaviate: environment: - ENABLE_MODULES: "text2vec-cohere,text2vec-huggingface,text2vec-openai,text2vec-google,generative-cohere,generative-openai,generative-googles" + ENABLE_MODULES: "text2vec-cohere,text2vec-huggingface,text2vec-openai,text2vec-google,generative-cohere,generative-openai,generative-google" ``` Check the specific documentation for your deployment method ([Docker](/deploy/installation-guides/docker-installation.md), [Kubernetes](/deploy/installation-guides/k8s-installation.md), [Embedded Weaviate](/deploy/installation-guides/embedded.md)) for more information on how to configure it. @@ -359,38 +359,49 @@ Check the specific documentation for your deployment method ([Docker](/deploy/in
How to configure the language model -Model properties are exposed through the Weaviate module configuration. Accordingly, you can customize them through the `moduleConfig` parameter in the collection definition. - -For example, the `generative-cohere` module has the following properties: - -```json - "moduleConfig": { - "generative-cohere": { - "model": "command-xlarge-nightly", // Optional - Defaults to `command-xlarge-nightly`. Can also use`command-xlarge-beta` and `command-xlarge` - "temperatureProperty": , // Optional - "maxTokensProperty": , // Optional - "kProperty": , // Optional - "stopSequencesProperty": , // Optional - "returnLikelihoodsProperty": , // Optional - } - } +Model parameters are exposed through the generative model provider configuration. You can set them when you create the collection, alongside the generative integration itself. + +For example, the `generative-cohere` integration can be configured as follows: + +```python +from weaviate.classes.config import Configure + +client.collections.create( + "DemoCollection", + generative_config=Configure.Generative.cohere( + # # These parameters are optional + # model="command-a-03-2025", + # temperature=0.7, + # max_tokens=500, + # k=5, + # stop_sequences=["\n\n"], + ) + # Additional parameters not shown +) ``` -And the `generative-openai` module may be configured as follows: - -```json - "moduleConfig": { - "generative-openai": { - "model": "gpt-3.5-turbo", // Optional - Defaults to `gpt-3.5-turbo` - "temperatureProperty": , // Optional, applicable to both OpenAI and Azure OpenAI - "maxTokensProperty": , // Optional, applicable to both OpenAI and Azure OpenAI - "frequencyPenaltyProperty": , // Optional, applicable to both OpenAI and Azure OpenAI - "presencePenaltyProperty": , // Optional, applicable to both OpenAI and Azure OpenAI - "topPProperty": , // Optional, applicable to both OpenAI and Azure OpenAI - }, - } +And the `generative-openai` integration can be configured as follows: + +```python +from weaviate.classes.config import Configure + +client.collections.create( + "DemoCollection", + generative_config=Configure.Generative.openai( + # # These parameters are optional + # model="gpt-5-mini", + # temperature=0.7, + # max_tokens=500, + # frequency_penalty=0, + # presence_penalty=0, + # top_p=0.7, + ) + # Additional parameters not shown +) ``` +Each parameter is optional. If you do not set a parameter, Weaviate applies the server-defined default. For the available models, the default model and the full parameter list, see the model provider pages for [Cohere](../model-providers/cohere/generative.md) and [OpenAI](../model-providers/openai/generative.md). + See the [documentation](../model-providers/index.md) for various model provider integrations.
diff --git a/docs/weaviate/starter-guides/managing-collections/index.mdx b/docs/weaviate/starter-guides/managing-collections/index.mdx index 2c488756f..9346b3f3e 100644 --- a/docs/weaviate/starter-guides/managing-collections/index.mdx +++ b/docs/weaviate/starter-guides/managing-collections/index.mdx @@ -39,104 +39,162 @@ The returned configuration looks similar to this: ```json { - "classes": [ + "class": "Question", + "invertedIndexConfig": { + "bm25": { + "b": 0.75, + "k1": 1.2 + }, + "cleanupIntervalSeconds": 60, + "stopwords": { + "additions": null, + "preset": "en", + "removals": null + }, + "usingBlockMaxWAND": true + }, + "moduleConfig": { + "generative-cohere": {} + }, + "multiTenancyConfig": { + "autoTenantActivation": false, + "autoTenantCreation": false, + "enabled": false + }, + "properties": [ { - "class": "Question", - "description": "Information from a Jeopardy! question", - "invertedIndexConfig": { - "bm25": { - "b": 0.75, - "k1": 1.2 - }, - "cleanupIntervalSeconds": 60, - "stopwords": { - "additions": null, - "preset": "en", - "removals": null + "dataType": [ + "text" + ], + "indexFilterable": true, + "indexRangeFilters": false, + "indexSearchable": true, + "moduleConfig": { + "text2vec-openai": { + "skip": false, + "vectorizePropertyName": false } }, + "name": "question", + "tokenization": "word" + }, + { + "dataType": [ + "text" + ], + "indexFilterable": true, + "indexRangeFilters": false, + "indexSearchable": true, "moduleConfig": { "text2vec-openai": { - "model": "ada", - "modelVersion": "002", - "type": "text", - "vectorizeClassName": true + "skip": false, + "vectorizePropertyName": false } }, - "properties": [ - { - "dataType": ["text"], - "description": "The question", - "moduleConfig": { - "text2vec-openai": { - "skip": false, - "vectorizePropertyName": false - } - }, - "name": "question", - "tokenization": "word" - }, - { - "dataType": ["text"], - "description": "The answer", - "moduleConfig": { - "text2vec-openai": { - "skip": false, - "vectorizePropertyName": false - } - }, - "name": "answer", - "tokenization": "word" - }, - { - "dataType": ["text"], - "description": "The category", - "moduleConfig": { - "text2vec-openai": { - "skip": false, - "vectorizePropertyName": false - } - }, - "name": "category", - "tokenization": "word" - } + "name": "answer", + "tokenization": "word" + }, + { + "dataType": [ + "text" ], - "replicationConfig": { - "factor": 1 - }, - "shardingConfig": { - "virtualPerPhysical": 128, - "desiredCount": 1, - "actualCount": 1, - "desiredVirtualCount": 128, - "actualVirtualCount": 128, - "key": "_id", - "strategy": "hash", - "function": "murmur3" + "indexFilterable": true, + "indexRangeFilters": false, + "indexSearchable": true, + "moduleConfig": { + "text2vec-openai": { + "skip": false, + "vectorizePropertyName": false + } }, + "name": "category", + "tokenization": "word" + } + ], + "shardingConfig": { + "actualCount": 1, + "actualVirtualCount": 128, + "desiredCount": 1, + "desiredVirtualCount": 128, + "function": "murmur3", + "key": "_id", + "strategy": "hash", + "virtualPerPhysical": 128 + }, + "vectorConfig": { + "default": { "vectorIndexConfig": { - "skip": false, + "bq": { + "enabled": false + }, "cleanupIntervalSeconds": 300, - "maxConnections": 32, - "efConstruction": 128, - "ef": -1, - "dynamicEfMin": 100, - "dynamicEfMax": 500, + "distance": "cosine", "dynamicEfFactor": 8, - "vectorCacheMaxObjects": 1000000000000, + "dynamicEfMax": 500, + "dynamicEfMin": 100, + "ef": -1, + "efConstruction": 128, + "filterStrategy": "acorn", "flatSearchCutoff": 40000, - "distance": "cosine" + "maxConnections": 32, + "multivector": { + "aggregation": "maxSim", + "enabled": false, + "muvera": { + "dprojections": 16, + "enabled": false, + "ksim": 4, + "repetitions": 10 + } + }, + "pq": { + "bitCompression": false, + "centroids": 256, + "enabled": false, + "encoder": { + "distribution": "log-normal", + "type": "kmeans" + }, + "segments": 0, + "trainingLimit": 100000 + }, + "rq": { + "bits": 8, + "enabled": false, + "rescoreLimit": 20 + }, + "skip": false, + "skipDefaultQuantization": false, + "sq": { + "enabled": false, + "rescoreLimit": 20, + "trainingLimit": 100000 + }, + "trackDefaultQuantization": false, + "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", - "vectorizer": "text2vec-openai" + "vectorizer": { + "text2vec-openai": { + "baseURL": "https://api.openai.com", + "isAzure": false, + "model": "text-embedding-3-small", + "vectorizeClassName": true + } + } } - ] + }, + "replicationConfig": { + "deletionStrategy": "TimeBasedResolution", + "factor": 1, + "asyncEnabled": false + } } ``` -Although we only specified the collection name and properties, the returned definition includes much more information. +Although we only specified the collection name, its properties, the vectorizer and the generative module, the returned definition includes much more information. This is because Weaviate infers the definition based on the data schema and default settings. Each of these options can be specified manually at collection creation time. diff --git a/docs/weaviate/starter-guides/managing-resources/index.md b/docs/weaviate/starter-guides/managing-resources/index.md index af24ddfbc..83689343f 100644 --- a/docs/weaviate/starter-guides/managing-resources/index.md +++ b/docs/weaviate/starter-guides/managing-resources/index.md @@ -145,7 +145,7 @@ Weaviate supports the following vector compression methods: | Product Quantization (PQ) | HNSW | Yes | Each vector becomes an array of integer-based centroids ([read more](../../concepts/vector-quantization.md#product-quantization)) | | Binary Quantization (BQ) | HNSW, Flat | No | Each vector dimension becomes a bit ([read more](../../concepts/vector-quantization.md#binary-quantization)) | | Scalar Quantization (SQ) | HNSW | Yes | Each vector dimension becomes an integer ([read more](../../concepts/vector-quantization.md#scalar-quantization)) | -| Rotational Quantization (RQ) | HNSW | No | Each vector is rotated then quantized to an integer ([read more](../../concepts/vector-quantization.md#rotational-quantization)) | +| Rotational Quantization (RQ) | All index types | No | Each vector is rotated then quantized to an integer ([read more](../../concepts/vector-quantization.md#rotational-quantization)) | As a starting point, use the following guidelines for selecting a compression method: diff --git a/docs/weaviate/starter-guides/managing-resources/indexing.mdx b/docs/weaviate/starter-guides/managing-resources/indexing.mdx index 450abef8b..8baa226a4 100644 --- a/docs/weaviate/starter-guides/managing-resources/indexing.mdx +++ b/docs/weaviate/starter-guides/managing-resources/indexing.mdx @@ -100,7 +100,7 @@ import HFreshStatus from '/_includes/feature-notes/hfresh_status.mdx'; HFresh indexes are well-suited when memory efficiency is a priority, especially with high-dimensional vectors. They use mandatory 1-bit rotational quantization (RQ) for postings and 8-bit RQ for centroids. Only the compressed centroid index is kept in memory. The posting lists live on disk, so memory usage stays low even as the collection grows. The trade-off is lower peak query throughput than HNSW, so HFresh is a good fit when you can tolerate higher query latency in exchange for smaller memory requirements. :::tip Tuning -Start with the default parameters. If recall is too low, increase `searchProbe` or the RQ `rescoreLimit` — both take effect at runtime. +Start with the default parameters. If recall is too low, increase `searchProbe` or the RQ `rescoreLimit`. Both take effect at runtime. ::: :::note diff --git a/docs/weaviate/tutorials/cross-references.mdx b/docs/weaviate/tutorials/cross-references.mdx index d17ab920c..de2ad6150 100644 --- a/docs/weaviate/tutorials/cross-references.mdx +++ b/docs/weaviate/tutorials/cross-references.mdx @@ -18,7 +18,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; import XRefCrudPythonCode from '!!raw-loader!/_includes/code/howto/manage-data.cross-refs.py'; -import XRefCrudTSCode from '!!raw-loader!/_includes/code/howto/manage-data.cross-refs'; +import XRefCrudTSCode from '!!raw-loader!/_includes/code/howto/manage-data.cross-refs.ts'; import XRefCrudGoCode from '!!raw-loader!/_includes/code/howto/go/docs/manage-data.cross-refs_test.go'; import SearchBasicsPythonCode from '!!raw-loader!/_includes/code/howto/search.basics.py'; @@ -37,7 +37,7 @@ We will refer to the originating object as the **source** object, and the object ### Prerequisites -This tutorial assumes that you have completed the [QuickStart tutorial](docs/weaviate/quickstart/index.md) and have access to a Weaviate instance with write access. +This tutorial assumes that you have completed the [QuickStart tutorial](../quickstart/index.md) and have access to a Weaviate instance with write access. ## When to use cross-references @@ -112,6 +112,14 @@ An example syntax is shown below:
+ + + - End-to-end guide that cover important topics like configuring + End-to-end guide that covers important topics like configuring collections, compression, etc. ), @@ -38,9 +38,9 @@ export const advancedFeaturesData = [ { title: "Zero-downtime collection migration with aliases", description: - "Learn how to migrate Weaviate collections without service interruption using collections aliases.", + "Learn how to migrate Weaviate collections without service interruption using collection aliases.", link: "/weaviate/tutorials/collection-aliases", - icon: "fas fa-share ", + icon: "fas fa-share", }, { title: "Import data in bulk", @@ -82,7 +82,7 @@ export const advancedFeaturesData = [ description: "Learn how to update the embedding model in order to improve performance.", link: "/weaviate/tutorials/vectorizer-migration", - icon: "fas fa-lock", + icon: "fas fa-right-left", }, ]; diff --git a/docs/weaviate/tutorials/multi-vector-embeddings.md b/docs/weaviate/tutorials/multi-vector-embeddings.md index c4c027bd5..da1130896 100644 --- a/docs/weaviate/tutorials/multi-vector-embeddings.md +++ b/docs/weaviate/tutorials/multi-vector-embeddings.md @@ -353,16 +353,16 @@ In all other searches where a vector embedding is to be specifically provided, i @@ -553,16 +553,16 @@ To perform a hybrid search with user-provided embeddings, provide the query vect diff --git a/docs/weaviate/tutorials/query.md b/docs/weaviate/tutorials/query.md deleted file mode 100644 index a75f1eca8..000000000 --- a/docs/weaviate/tutorials/query.md +++ /dev/null @@ -1,294 +0,0 @@ ---- -title: Queries in detail -description: Learn effective query techniques in Weaviate to retrieve accurate results. -sidebar_position: 50 -image: og/docs/tutorials.jpg -# tags: ['basics'] ---- - -import UpdateInProgressNote from '/\_includes/update-in-progress.mdx'; - - - -In this section, we will explore different queries that you can perform with Weaviate. Here, we will expand on the `nearText` queries that you may have seen in the [Quickstart tutorial](docs/weaviate/quickstart/index.md) to show you different query types, filters and metrics that can be used. - -By the end of this section, you will have performed vector and scalar searches separately as well as in combination to retrieve individual objects and aggregations. - -## Prerequisites - -We recommend you complete the [Quickstart tutorial](docs/weaviate/quickstart/index.md) first. - -Before you start this tutorial, you should follow the steps in the Quickstart to have: - -- An instance of Weaviate running (e.g. on the [Weaviate Cloud](/go/console?utm_content=tutorial)), -- An API key for your preferred inference API, such as OpenAI, Cohere, or Hugging Face, -- Installed your preferred Weaviate client library, -- Set up a `Question` class in your schema, and -- Imported the `jeopardy_tiny.json` data. - -## Object retrieval with `Get` - -:::tip GraphQL -Weaviate's queries are built using GraphQL. If this is new to you, don't worry. We will take it step-by-step and build up from the basics. Also, in many cases, the GraphQL syntax is abstracted by the client. - -You can query Weaviate using one or a combination of a semantic (i.e. vector) search and a lexical (i.e. scalar) search. As you've seen, a vector search allows for similarity-based searches, while scalar searches allow filtering by exact matches. -::: - -First, we will start by making queries to Weaviate to retrieve **Question** objects that we imported earlier. - -The Weaviate function for retrieving objects is `Get`. - -This might be familiar for some of you. If you have completed our [Imports in detail tutorial](./import.mdx), you may have performed a `Get` query to confirm that the data import was successful. Here is the same code as a reminder: - -import CodeImportGet from '/\_includes/code/quickstart.import.get.mdx'; - - - -This query simply asks Weaviate for _some_ objects of this (`Question`) class. - -Of course, in most cases we would want to retrieve information on some criteria. Let's build on this query by adding a vector search. - -### `Get` with `nearText` - -This is a vector search using a `Get` query. - -import CodeAutoschemaNeartext from '/\_includes/code/quickstart/neartext.mdx' - - - -This might also look familiar, as it was used in the [Quickstart tutorial](docs/weaviate/quickstart/index.md). But let's break it down a little. - -Here, we are using a `nearText` operator. What we are doing is to provide Weaviate with a query `concept` of `biology`. Weaviate then converts this into a vector through the inference API (OpenAI in this particular example) and uses that vector as the basis for a vector search. - -Also note here that we pass the API key in the header. This is required as the inference API is used to vectorize the input query. - -Additionally, we use the `limit` argument to only fetch a maximum of two (2) objects. - -If you run this query, you should see the entries on _"DNA"_ and _"species"_ returned by Weaviate. - -### `Get` with `nearVector` - -In some cases, you might wish to input a vector directly as a search query. For example, you might be running Weaviate with a custom, external vectorizer. In such a case, you can use the `nearVector` operator to provide the query vector to Weaviate. - -For example, here is an example Python code obtaining an OpenAI embedding manually and providing it through the `nearVector` operator: - -```python -import openai - -openai.api_key = "YOUR-OPENAI-API-KEY" -model="text-embedding-ada-002" -oai_resp = openai.Embedding.create(input = ["biology"], model=model) - -oai_embedding = oai_resp['data'][0]['embedding'] - -result = ( - client.query - .get("Question", ["question", "answer"]) - .with_near_vector({ - "vector": oai_embedding, - "certainty": 0.7 - }) - .with_limit(2) - .do() -) - -print(json.dumps(result, indent=4)) -``` - -And it should return the same results as above. - -Note that we used the same OpenAI embedding model (`text-embedding-ada-002`) here so that the vectors are in the same vector "space". - -You might also have noticed that we have added a `certainty` argument in the `with_near_vector` method. This lets you specify a similarity threshold for objects, and can be very useful for ensuring that no distant objects are returned. - -## Additional properties - -We can ask Weaviate to return `_additional` properties for any returned objects. This allows us to obtain properties such as the `vector` of each returned object as well as the actual `certainty` value, so we can verify how close each object is to our query vector. Here is a query that will return the `certainty` value: - -import CodeQueryNeartextAdditional from '/\_includes/code/quickstart.query.neartext.additional.mdx' - - - -Try it out, and you should see a response like this: - -```json -{ - "data": { - "Get": { - "Question": [ - { - "_additional": { - "certainty": 0.9030631184577942 - }, - "answer": "DNA", - "category": "SCIENCE", - "question": "In 1953 Watson & Crick built a model of the molecular structure of this, the gene-carrying substance" - }, - { - "_additional": { - "certainty": 0.900638073682785 - }, - "answer": "species", - "category": "SCIENCE", - "question": "2000 news: the Gunnison sage grouse isn't just another northern sage grouse, but a new one of this classification" - } - ] - } - } -} -``` - -You can try modifying this query to see if you retrieve the vector (note - it will be a looooong response 😉). - -We encourage you to also try out different queries and see how that changes the results and distances not only with this dataset but also with different datasets, and/or vectorizers. - -## Filters - -As useful as it is, sometimes vector search alone may not be sufficient. For example, you may actually only be interested in **Question** objects in a particular category, for instance. - -In these cases, you can use Weaviate's scalar filtering capabilities - either alone, or in combination with the vector search. - -Try the following: - -import CodeQueryWhere1 from '/\_includes/code/quickstart.query.where.1.mdx' - - - -This query asks Weaviate for **Question** objects whose category contains the string `ANIMALS`. You should see a result like this: - -```json -{ - "data": { - "Get": { - "Question": [ - { - "answer": "the diamondback rattler", - "category": "ANIMALS", - "question": "Heaviest of all poisonous snakes is this North American rattlesnake" - }, - { - "answer": "Elephant", - "category": "ANIMALS", - "question": "It's the only living mammal in the order Proboseidea" - }, - { - "answer": "the nose or snout", - "category": "ANIMALS", - "question": "The gavial looks very much like a crocodile except for this bodily feature" - }, - { - "answer": "Antelope", - "category": "ANIMALS", - "question": "Weighing around a ton, the eland is the largest species of this animal in Africa" - } - ] - } - } -} -``` - -Now that you've seen a scalar filter, let's see how it can be combined with vector search functions. - -### Vector search with scalar filters - -Combining a filter with a vector search is an additive process. Let us show you what we mean by that. - -import CodeQueryWhere2 from '/\_includes/code/quickstart.query.where.2.mdx' - - - -This query asks Weaviate for **Question** objects that are closest to "biology", but within the category of `ANIMALS`. You should see a result like this: - -```json -{ - "data": { - "Get": { - "Question": [ - { - "_additional": { - "certainty": 0.8918434679508209 - }, - "answer": "the nose or snout", - "category": "ANIMALS", - "question": "The gavial looks very much like a crocodile except for this bodily feature" - }, - { - "_additional": { - "certainty": 0.8867587149143219 - }, - "answer": "Elephant", - "category": "ANIMALS", - "question": "It's the only living mammal in the order Proboseidea" - } - ] - } - } -} -``` - -Note that the results are confined to the choices from the 'animals' category. Note that these results, while not being cutting-edge science, are biological factoids. - -## Metadata with `Aggregate` - -As the name suggests, the `Aggregate` function can be used to show aggregated data such as on entire classes or groups of objects. - -For example, the following query will return the number of data objects in the `Question` class: - -import CodeQueryAggregate1 from '/\_includes/code/quickstart.query.aggregate.1.mdx' - - - -And you can also use the `Aggregate` function with filters, just as you saw with the `Get` function above. For example, this query will return the number of **Question** objects with the category "ANIMALS". - -import CodeQueryAggregate2 from '/\_includes/code/quickstart.query.aggregate.2.mdx' - - - -And as you saw above, there are four objects that match the query filter. - -```json -{ - "data": { - "Aggregate": { - "Question": [ - { - "meta": { - "count": 4 - } - } - ] - } - } -} -``` - -Here, Weaviate has identified the same objects that you saw earlier in the similar `Get` queries. The difference is that instead of returning the individual objects you are seeing the requested aggregated statistic (count) here. - -As you can see, the `Aggregate` function can return handy aggregated, or metadata, information from the Weaviate Database. - -## Recap - -- `Get` queries are used for retrieving data objects. -- `Aggregate` queries can be used to retrieve metadata, or aggregated data. -- Operators such as `nearText` or `nearVector` can be used for vector queries. -- Scalar filters can be used for exact filtering, taking advantage of inverted indexes. -- Vector and scalar filters can be combined, and are available on both `Get` and `Aggregate` queries - -## Suggested reading - -- [Tutorial: Schemas in detail](../starter-guides/managing-collections/index.mdx) -- [Tutorial: Import in detail](./import.mdx) -- [Tutorial: Introduction to modules](./modules.md) -- [Tutorial: Introduction to Weaviate Console](/cloud/tools/query-tool.mdx) - -## Notes - -### How is certainty calculated? - -`certainty` in Weaviate is a measure of distance from the vector to the data objects. You can also calculate the cosine similarity based on the certainty as described [here](/weaviate/config-refs/distances#distance-vs-certainty). - -## Questions and feedback - -import DocsFeedback from '/\_includes/docs-feedback.mdx'; - - diff --git a/docs/weaviate/tutorials/quick-tour-of-weaviate.mdx b/docs/weaviate/tutorials/quick-tour-of-weaviate.mdx index 776f338cf..e2d5b057b 100644 --- a/docs/weaviate/tutorials/quick-tour-of-weaviate.mdx +++ b/docs/weaviate/tutorials/quick-tour-of-weaviate.mdx @@ -58,9 +58,9 @@ flowchart LR ### Prerequisites -In order to perform Retrieval Augmented Generation (RAG) in the last step, you will need a [Cohere](https://dashboard.cohere.com/) account. You can use a free Cohere trial API key. +In order to perform Retrieval Augmented Generation (RAG) in the last step, you will need an [OpenAI](https://platform.openai.com/) account and an OpenAI API key. -If you have another preferred [model provider](/weaviate/model-providers), you can use that instead of Cohere. +If you have another preferred [model provider](/weaviate/model-providers), you can use that instead of OpenAI.
@@ -212,11 +212,11 @@ We can now add data to our collection. The following example: - Loads objects, and -- Adds objects to the target collection (`Question`) using a batch process. +- Adds objects to the target collection (`Question`) with a batch import. :::tip Batch imports -([Batch imports](../manage-objects/import.mdx)) are the most efficient way to add large amounts of data, as it sends multiple objects in a single request. See the [How-to: Batch import](../manage-objects/import.mdx) guide for more information. +Batch imports are the most efficient way to add large amounts of data, because they send objects in groups instead of one request per object. See the [How-to: Batch import](../manage-objects/import.mdx) guide for the available methods, including [server-side batching](../manage-objects/import.mdx#server-side-batching), where the server tells the client how much data to send next. ::: @@ -245,20 +245,18 @@ import QueryNearText from "/_includes/code/quickstart/quickstart.query.neartext. Run this code to perform the query. Our query found entries for `DNA` and `species`.
- Example full response in JSON format + Example response ```json { - { - "answer": "DNA", - "question": "In 1953 Watson & Crick built a model of the molecular structure of this, the gene-carrying substance", - "category": "SCIENCE" - }, - { - "answer": "species", - "question": "2000 news: the Gunnison sage grouse isn't just another northern sage grouse, but a new one of this classification", - "category": "SCIENCE" - } + "answer": "DNA", + "question": "In 1953 Watson & Crick built a model of the molecular structure of this, the gene-carrying substance", + "category": "SCIENCE" +} +{ + "answer": "species", + "question": "2000 news: the Gunnison sage grouse isn't just another northern sage grouse, but a new one of this classification", + "category": "SCIENCE" } ``` @@ -377,9 +375,9 @@ import QueryRAG from "/_includes/code/quickstart/quickstart.query.rag.mdx"; -:::info Cohere API key in the header +:::info OpenAI API key in the header -Note that this code includes an additional header for the Cohere API key. Weaviate uses this key to access the Cohere generative AI model and perform retrieval augmented generation (RAG). +Note that this code includes an additional header for the OpenAI API key. Weaviate uses this key to access the OpenAI generative AI model and perform retrieval augmented generation (RAG). ::: diff --git a/docs/weaviate/tutorials/rbac.mdx b/docs/weaviate/tutorials/rbac.mdx index db84ea65f..859bdc5dc 100644 --- a/docs/weaviate/tutorials/rbac.mdx +++ b/docs/weaviate/tutorials/rbac.mdx @@ -15,7 +15,7 @@ import RolePyCode from '!!raw-loader!/_includes/code/python/howto.configure.rbac import UserPyCode from '!!raw-loader!/_includes/code/python/howto.configure.rbac.users.py'; import RoleTSCode from '!!raw-loader!/_includes/code/typescript/howto.configure.rbac.roles.ts'; -**Role-Based Access Control (RBAC)** is a powerful security mechanism that allows you to manage who can access and modify your Weaviate instance. In this tutorial, you'll learn how to set up RBAC in Weaviate by defining roles with tailored permissions and assigning them to users. This enables granular control over operations—from reading and writing data to managing collections and tenants—ensuring that only authorized users can perform specific actions. +**Role-Based Access Control (RBAC)** is a powerful security mechanism that allows you to manage who can access and modify your Weaviate instance. In this tutorial, you'll learn how to set up RBAC in Weaviate by defining roles with tailored permissions and assigning them to users. This enables granular control over operations, from reading and writing data to managing collections and tenants, ensuring that only authorized users can perform specific actions. In the steps that follow, we’ll cover: diff --git a/docs/weaviate/tutorials/spark-connector.md b/docs/weaviate/tutorials/spark-connector.md index 98eb17247..6dc1f732f 100644 --- a/docs/weaviate/tutorials/spark-connector.md +++ b/docs/weaviate/tutorials/spark-connector.md @@ -17,7 +17,7 @@ By the end of this tutorial, you'll be able to see how to you can import your da ## Installation -We recommend reading the [Quickstart tutorial](docs/weaviate/quickstart/index.md) first before tackling this tutorial. +We recommend reading the [Quickstart tutorial](../quickstart/index.md) first before tackling this tutorial. We will install the python `weaviate-client` and also run Spark locally for which we need to install the python `pyspark` package. Use the following command in your terminal to get both: ```bash @@ -32,7 +32,7 @@ We will also need the Weaviate Spark connector. You can download this by running curl https://github.com/weaviate/spark-connector/releases/download/v||site.spark_connector_version||/spark-connector-assembly-||site.spark_connector_version||.jar --output spark-connector-assembly-||site.spark_connector_version||.jar ``` -For this tutorial, you will also need a Weaviate instance running at `http://localhost:8080`. This instance does not need to have any modules and can be setup by following the [Quickstart tutorial](docs/weaviate/quickstart/index.md). +For this tutorial, you will also need a Weaviate instance running at `http://localhost:8080`. This instance does not need to have any modules and can be setup by following the [Quickstart tutorial](../quickstart/index.md). You will also need Java 8+ and Scala 2.12 installed. You can get these separately setup or a more convenient way to get both of these set up is to install [IntelliJ](https://www.jetbrains.com/idea/). @@ -102,7 +102,7 @@ To verify this is done correctly we can have a look at the first few records: ## Writing to Weaviate :::tip -Prior to this step, make sure your Weaviate instance is running at `http://localhost:8080`. You can refer to the [Quickstart tutorial](docs/weaviate/quickstart/index.md) for instructions on how to set that up. +Prior to this step, make sure your Weaviate instance is running at `http://localhost:8080`. You can refer to the [Quickstart tutorial](../quickstart/index.md) for instructions on how to set that up. ::: To quickly get a Weaviate instance running you can save the following `docker-compose.yml` file to your local machine: @@ -199,7 +199,15 @@ Let's examine the code above to understand exactly what's happening and all the By now we've written our data to Weaviate, and we understand the capabilities of the Spark connector and its settings. As a last step, we can query the data via the Python client to confirm that the data has been loaded. ```python -client.query.get("Sphere", "title").do() +spheres = client.collections.use("Sphere") + +response = spheres.query.fetch_objects( + limit=3, + return_properties=["title"], +) + +for obj in response.objects: + print(obj.properties["title"]) ``` ## Additional options diff --git a/docs/weaviate/tutorials/tls-ssl.mdx b/docs/weaviate/tutorials/tls-ssl.mdx index 0ff91d3be..58c40ccc1 100644 --- a/docs/weaviate/tutorials/tls-ssl.mdx +++ b/docs/weaviate/tutorials/tls-ssl.mdx @@ -12,7 +12,7 @@ When your Weaviate database is exposed to the internet, unencrypted connections - **Authentication credentials** if transmitted without encryption - **API keys and tokens** that could grant unauthorized access to your systems -That's exactly why it's important to secure your database using **SSL/TLS (Secure Sockets Layer/Transport Layer Security)** — encryption protocols that create a secure, encrypted connection between your clients and your database. SSL/TLS ensures that even if network traffic is intercepted, the data remains unreadable to unauthorized parties. +That's exactly why it's important to secure your database using **SSL/TLS (Secure Sockets Layer/Transport Layer Security)**, encryption protocols that create a secure, encrypted connection between your clients and your database. SSL/TLS ensures that even if network traffic is intercepted, the data remains unreadable to unauthorized parties. :::info Managed Weaviate deployments @@ -20,7 +20,7 @@ If you're using **Weaviate Cloud** or other managed Weaviate services, SSL/TLS i ::: -This guide will take you through three different options for securing your self-hosted database. Whether you're a small startup or an enterprise giant - there's a solution for you! +This guide will take you through three different options for securing your self-hosted database. Whether you're a small startup or an enterprise giant, there's a solution for you! ### When is SSL/TLS required? diff --git a/docs/weaviate/tutorials/tokenization.md b/docs/weaviate/tutorials/tokenization.md index 0af1c9a81..d4041da13 100644 --- a/docs/weaviate/tutorials/tokenization.md +++ b/docs/weaviate/tutorials/tokenization.md @@ -348,14 +348,14 @@ We create three properties: one without folding (`text_default`), one with full **Key observations:** - Without folding, only exact accented forms match - With `asciiFold: true`, both accented and unaccented queries match -- `asciiFoldIgnore` lets you preserve specific characters — `"cafe"` no longer matches `"Café"` when `é` is ignored +- `asciiFoldIgnore` lets you preserve specific characters: `"cafe"` no longer matches `"Café"` when `é` is ignored - `asciiFoldIgnore` is immutable after property creation ## Example 5: Custom and per-property stopword presets -The default stopword presets are `en` and `none`. For a French property, neither is appropriate — `la`, `le`, and `et` should be filtered, but they are not in the English list. Define a custom preset on the collection and assign it to specific properties. +The default stopword presets are `en` and `none`. For a French property, neither is appropriate: `la`, `le`, and `et` should be filtered, but they are not in the English list. Define a custom preset on the collection and assign it to specific properties. ### Create a collection with custom stopwords @@ -483,7 +483,7 @@ The default stopword presets are `en` and `none`. For a French property, neither **Key observations:** - The `fr` preset filters out `la`, `le`, and `et` from BM25 scoring on the French property - The same words are not filtered on the English property (they are not English stopwords) -- Stopwords are still indexed — only filtered at query time — so changing presets does not require reindexing +- Stopwords are still indexed (they are only filtered at query time), so changing presets does not require reindexing - A preset name that matches a built-in (`en`, `none`) replaces the built-in for this collection. To tweak a built-in with `additions`/`removals`, use the collection-level `invertedIndexConfig.stopwords` field instead ## Example 6: Inspecting tokenization with the tokenize endpoint diff --git a/docs/weaviate/tutorials/vector-indexing-deep-dive.mdx b/docs/weaviate/tutorials/vector-indexing-deep-dive.mdx index f977ece02..80e8db767 100644 --- a/docs/weaviate/tutorials/vector-indexing-deep-dive.mdx +++ b/docs/weaviate/tutorials/vector-indexing-deep-dive.mdx @@ -124,9 +124,11 @@ To create a collection with the `HFresh` index using default settings, simply pa - `distance_metric`: `cosine` - `replicas`: `4` (number of posting lists per vector) -- `search_probe`: `64` (number of posting lists to search) +- `search_probe`: `256` (number of posting lists to search) - `max_posting_size_kb`: `48` +The `search_probe` default is `256` in `v1.36.20`, `v1.37.10`, `v1.38.2` and later. Earlier releases on each of those lines default to `64`. + =1.40.0", "pytest>=8.3.5", "python-dotenv>=1.1.1", + "pyyaml>=6.0", "requests>=2.32.3", "tqdm>=4.67.1", - "weaviate-agents>=1.6.0", - "weaviate-client==4.22.0", + "weaviate-agents>=1.7.0", + "weaviate-client==4.23.0", "weaviate-demo-datasets>=0.8.1", "weaviate-engram>=0.3.0", "anthropic>=0.40.0", diff --git a/sidebars.js b/sidebars.js index 146d3abb0..69a9f679d 100644 --- a/sidebars.js +++ b/sidebars.js @@ -703,7 +703,6 @@ const sidebars = { "weaviate/tutorials/cross-references", "weaviate/tutorials/spark-connector", //"weaviate/tutorials/vector-provision-options", - //"weaviate/tutorials/query", //"weaviate/tutorials/wikipedia", //"weaviate/tutorials/modules", ], @@ -1370,6 +1369,11 @@ const sidebars = { id: "query-agent/reference/advanced_collections", className: "sidebar-item", }, + { + type: "doc", + id: "query-agent/reference/structured_outputs", + className: "sidebar-item", + }, ], }, { diff --git a/src/scripts/scarf.js b/src/scripts/scarf.js index 6e0d0384a..eab4276ab 100644 --- a/src/scripts/scarf.js +++ b/src/scripts/scarf.js @@ -5,6 +5,12 @@ const scarfScript = { src: "https://pixel.weaviate.cloud/a.png?x-pxid=a41b0758-a3a9-4874-a880-8b5d5a363d40", referrerPolicy: "no-referrer-when-downgrade", style: "display: none;", + // Decorative, hidden analytics beacon: aria-hidden is what keeps it out of + // the accessibility tree, so assistive technology never reaches it and never + // has to announce it. Do not add `alt: ""` here: Docusaurus validates + // headTags attributes with a Joi `string()` schema that rejects the empty + // string, so an empty alt fails the site build outright. + "aria-hidden": "true", }, }; diff --git a/tests/README-INDEXABILITY.md b/tests/README-INDEXABILITY.md index d0b77b37a..ce68a3cec 100644 --- a/tests/README-INDEXABILITY.md +++ b/tests/README-INDEXABILITY.md @@ -21,7 +21,7 @@ Documentation content that is hidden behind JavaScript interactions (tabs, colla ### HTML structure tests (Part 1) -These tests fetch ~11 representative pages from all doc sections and check: +These tests fetch ~13 representative pages from all doc sections and check: | Test | What it checks | |------|---------------| @@ -81,7 +81,7 @@ HTML structure tests always run. Agent tests only run if `ANTHROPIC_API_KEY` and ## Test pages -The suite tests 11 representative URLs covering all doc sections: +The suite tests 13 representative URLs covering all doc sections: | Page | Features tested | |------|----------------| @@ -91,9 +91,11 @@ The suite tests 11 representative URLs covering all doc sections: | `/weaviate/search/hybrid` | tabs, code | | `/weaviate/connections/connect-cloud` | tabs, code | | `/weaviate/config-refs/collections` | details, table | -| `/weaviate/concepts/data-import` | images | -| `/cloud/quickstart` | code, images | -| `/cloud/manage-clusters/create` | images | +| `/weaviate/concepts/data-import` | no structural features (200, meta tags, headings, LLM notice only) | +| `/cloud/quickstart` | code | +| `/cloud/manage-clusters/create` | no structural features (200, meta tags, headings, LLM notice only) | +| `/cloud/tools/query-tool` | images | +| `/weaviate/manage-collections/tenant-states` | images | | `/query-agent/recipes/query-agent-ecommerce-assistant` | code | | `/weaviate/search` | landing page | @@ -129,4 +131,21 @@ TEST_PAGES = [ ] ``` -Available feature tags: `tabs`, `code`, `details`, `images`, `table`. Pages are parametrized — each feature tag enables the corresponding structural test for that page. +Available feature tags: `tabs`, `code`, `details`, `images`, `table`. Pages are parametrized — each feature tag enables the corresponding structural test for that page. Tag only what the page actually has: a page tagged `images` with no content image fails rather than passing quietly, which is what keeps the tags from going stale. + +### Landing pages + +If the page you are adding routes readers onward instead of carrying content of its own — a hub page that is essentially a list of links to its children — also add its path to the `LANDING_PAGES` set in the same file: + +```python +LANDING_PAGES = {"/weaviate/search"} +``` + +`LANDING_PAGES` is the **only** thing that exempts a page from the "content pages have h2 headings" assertion in `test_heading_hierarchy`. An empty feature set does not exempt it. The two mean different things: + +- an empty feature set says the page has none of the structural features listed above (no tabs, no code, no details, no table, no images); +- `LANDING_PAGES` says the page owes the reader no h2 headings at all. + +A page can be plain prose with an empty feature set and still be a content page that must have h2s, which is why the exemption is tracked separately. + +So if you add a landing page with `set()` and leave it out of `LANDING_PAGES`, it fails with `content page has no h2 headings` — a confusing failure, because the feature set already looks like it says "this page has nothing". Add the path to `LANDING_PAGES` instead. diff --git a/tests/README-LLMS-TXT.md b/tests/README-LLMS-TXT.md index b465a0c43..0b22bf4a3 100644 --- a/tests/README-LLMS-TXT.md +++ b/tests/README-LLMS-TXT.md @@ -116,6 +116,8 @@ no Weaviate cluster. 2. Run that language's `test_llms_txt` and confirm it passes — **never** hand-write a snippet without running it. 3. Copy the verified marked region **verbatim** into `weaviate-io/static/llms.txt`. + The PR-time sync warning (see *CI* below) flags this for you and prints the exact + block to paste. 4. New file? Add its path to the `test_llms_txt` parametrize list in the matching `tests/test_*.py`. @@ -182,12 +184,13 @@ after the matching `weaviate-io` change is live; otherwise mark with ## CI -Two separate workflows cover this directory: +Three separate workflows cover this directory: | Workflow | What it runs | |---|---| | `.github/workflows/docs_tests.yml` | Per-language **execution** tests — `test_llms_txt*` in `test_python.py`, `test_typescript.py`, `test_java.py`, `test_csharp.py`. Rides the existing `pyv4` / `ts` / `java` / `csharp` / `agents` markers, so no separate job for these. | | `.github/workflows/llms_txt_tests.yml` | The three **guard tests** in `test_llms_txt_code.py` — snippet coverage, version freshness, link validity. Single job `test-llms-txt`, runs `uv run pytest tests/test_llms_txt_code.py -m "llms_txt"`. | +| `.github/workflows/llms_txt_snippet_sync.yml` | The **PR-time warning** (`check_llms_txt_drift.py`). Advisory only, see below. | The guard workflow triggers on: @@ -204,6 +207,41 @@ Results post to Slack via the shared `./.github/actions/handle-test-results` composite under `test-type: 'llms.txt'`, with `continue-on-error: true` on the pytest step so the notification still fires when a guard fails. +### PR-time snippet sync warning + +The guard workflow runs weekly, so a snippet change here can break the published +`llms.txt` days before anyone notices. `tests/check_llms_txt_drift.py` closes that +gap on the docs side. `.github/workflows/llms_txt_snippet_sync.yml` runs it on every +PR touching a file that matches `SNIPPET_GLOBS`, and it answers one question: once +this PR merges, which `llms.txt` code blocks would `test_llms_txt_snippets_are_covered` +no longer find? Those, and only those, are the blocks `weaviate-io/static/llms.txt` +has to update in lockstep. + +It reuses this directory's matching logic (`SNIPPET_GLOBS`, the marker and fence +regexes, `_normalize`, `_load_llms_txt`), so it cannot disagree with the weekly job +about what "matches" means. Findings surface as GitHub warning annotations on the +changed snippet lines, plus a job summary carrying the new block to paste into +`llms.txt`. + +**It is advisory and never fails the job once it runs.** When a snippet PR is opened, weaviate-io has +not merged or deployed yet, so the live `llms.txt` legitimately cannot match yet; a +blocking check would fire on every honest PR and would just get overridden. It also +stays quiet whenever there is nothing to do: weaviate-io shipped first and `llms.txt` +already carries the new block, only scaffolding outside the markers changed, or a +region was moved or renamed without its code changing. + +Run it locally against uncommitted edits: + +```bash +uv run python tests/check_llms_txt_drift.py --base HEAD +``` + +The reverse direction (an `llms.txt` edit in weaviate-io that no longer matches these +snippets) is not automated. There is no check in weaviate-io; this repo's PR-time warning +and the weekly `llms_txt_tests.yml` job are the only automation. An `llms.txt` edit made +directly in weaviate-io that breaks the verbatim match is not caught until the weekly job +runs. + ## Per-language gotchas - **Python** — `text2vec_ollama` / `generative_ollama` take `api_endpoint` and diff --git a/tests/check_llms_txt_drift.py b/tests/check_llms_txt_drift.py new file mode 100644 index 000000000..0f32277df --- /dev/null +++ b/tests/check_llms_txt_drift.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""Advisory PR check: warn when a snippet change strands a block in llms.txt. + +`llms.txt` is hand-maintained in the **weaviate-io** repo (`static/llms.txt`), but its +code blocks must appear verbatim between `START`/`END` markers in this repo's snippet +files. The only thing enforcing that today is the weekly +`test_llms_txt_code.py::test_llms_txt_snippets_are_covered` job, so a snippet change +here can break the published `llms.txt` days before anyone notices. + +This script answers one question at PR time: once this PR merges, which `llms.txt` code +blocks would that weekly coverage test no longer find? Those blocks, and only those, are +the ones `weaviate-io/static/llms.txt` has to update in lockstep. + +It is deliberately advisory: **once it runs it never fails the job**, because every +reporting path exits 0. (Only a malformed invocation, such as omitting `--base`, exits +non-zero, and that comes from argparse before any checking happens.) A legitimate snippet +PR cannot have a matching live `llms.txt` yet, because weaviate-io has not merged or +deployed, so a blocking check here would fail every honest PR and get routinely +overridden. Findings are reported as GitHub warning annotations plus a job summary +instead. + +Usage: + + python tests/check_llms_txt_drift.py --base + +The pre-change state is read with `git show :`; the post-change state is read +from the working tree, so the changed-file set is `git diff --name-only `. That also +makes the script easy to exercise locally: edit a snippet file and run it with +`--base HEAD`. + +Honors `LLMS_TXT_PATH` exactly like the guard tests do, so it can be pointed at a local +weaviate-io checkout instead of the live https://weaviate.io/llms.txt. +""" +import argparse +import os +import subprocess +import sys +from dataclasses import dataclass +from fnmatch import fnmatch +from typing import Optional + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +# Reuse the guard test's matching logic (SNIPPET_GLOBS, marker/fence regexes, +# normalization, llms.txt loading) so this check and the weekly job can never +# disagree about what "matches" means. +import test_llms_txt_code as guard # noqa: E402 + +LLMS_TXT_SOURCE = "weaviate-io/static/llms.txt" + +# guard._load_llms_txt() calls pytest.skip() when the fetch fails, and Skipped derives +# from BaseException, so a bare `except Exception` would let it through. +_LOAD_FAILURES = (Exception, guard.pytest.skip.Exception) + + +@dataclass +class Finding: + """A marked region that exists at the base ref but not after the change.""" + + path: str + language: str + marker: str + old_code: str + new_code: Optional[str] # same marker's code after the change, if it still exists + new_line: Optional[int] # 1-based line of the surviving `START` marker + + +def _git(*args): + return subprocess.run(["git", *args], capture_output=True, text=True, check=False) + + +def _repo_root(): + result = _git("rev-parse", "--show-toplevel") + return result.stdout.strip() if result.returncode == 0 else None + + +def _snippet_patterns(): + return [(lang, pattern) for lang, patterns in guard.SNIPPET_GLOBS.items() for pattern in patterns] + + +def _language_for(path): + for language, pattern in _snippet_patterns(): + if fnmatch(path, pattern): + return language + return None + + +def _changed_snippet_files(base): + """Paths matching SNIPPET_GLOBS that differ between `base` and the working tree.""" + result = _git("diff", "--name-only", base) + if result.returncode != 0: + return None, result.stderr.strip() or f"git diff against {base} failed" + changed = [path for path in result.stdout.splitlines() if _language_for(path)] + return sorted(changed), None + + +def _regions(text): + """[(marker, normalized code, 1-based START line)] for each region in `text`.""" + found = [] + for match in guard.MARKER_RE.finditer(text): + line = text.count("\n", 0, match.start()) + 1 + found.append((match.group(1), guard._normalize(match.group(2)), line)) + return found + + +def _blob_at(ref, path): + """File contents at `ref`, or None if the file did not exist there.""" + result = _git("show", f"{ref}:{path}") + return result.stdout if result.returncode == 0 else None + + +def _working_tree_text(path): + if not os.path.exists(path): + return "" # file deleted by the change + with open(path, encoding="utf-8") as handle: + return handle.read() + + +def _lost_regions(base, changed_paths): + """Regions present at `base` whose exact code no longer exists anywhere in the repo. + + Comparing against every snippet file (not just the changed one) means a region that + was merely moved or renamed is correctly treated as still covered. + """ + surviving = guard._collect_marked_regions() + lost = [] + for path in changed_paths: + language = _language_for(path) + base_text = _blob_at(base, path) + if base_text is None: + continue # new file: nothing can be stranded + head_regions = {marker: (code, line) for marker, code, line in _regions(_working_tree_text(path))} + for marker, code, _ in _regions(base_text): + if code in surviving[language]: + continue + new_code, new_line = head_regions.get(marker, (None, None)) + lost.append(Finding(path, language, marker, code, new_code, new_line)) + return lost + + +def _llms_txt_blocks(): + """({language: {normalized code}}, None), or (None, reason) if llms.txt is unreadable.""" + try: + content = guard._load_llms_txt() + except _LOAD_FAILURES as exc: + return None, str(exc) + blocks = {language: set() for language in guard.SNIPPET_GLOBS} + for match in guard.FENCE_RE.finditer(content): + language = guard.LANG_ALIASES.get(match.group(1).lower()) + if language is not None: + blocks[language].add(guard._normalize(match.group(2))) + return blocks, None + + +def _escape(value, is_property=False): + escaped = value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + if is_property: + escaped = escaped.replace(":", "%3A").replace(",", "%2C") + return escaped + + +def _annotate(level, message, path=None, line=None, title=None): + properties = [] + if path: + properties.append(f"file={_escape(path, True)}") + if line: + properties.append(f"line={line}") + if title: + properties.append(f"title={_escape(title, True)}") + joined = "," + ",".join(properties) if properties else "" + print(f"::{level}{joined}::{_escape(message)}") + + +def _write_summary(markdown): + print(markdown) + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + with open(summary_path, "a", encoding="utf-8") as handle: + handle.write(markdown + "\n") + + +def _report_in_sync(headline): + _write_summary(f"## llms.txt snippet sync\n\n{headline}\n") + + +def _report_drift(findings): + for finding in findings: + if finding.new_code is None: + detail = f"The `{finding.marker}` region was removed or renamed" + else: + detail = f"The `{finding.marker}` region changed" + _annotate( + "warning", + f"{detail}, but {LLMS_TXT_SOURCE} still publishes the previous version. " + f"Update {LLMS_TXT_SOURCE} in lockstep with this PR, or the weekly llms.txt " + f"coverage job will fail once this merges.", + path=finding.path, + line=finding.new_line, + title="llms.txt needs a matching update", + ) + + rows = "\n".join( + f"| `{finding.path}` | `{finding.marker}` | " + f"{'removed or renamed' if finding.new_code is None else 'changed'} |" + for finding in findings + ) + blocks = "\n\n".join( + f"
{finding.marker} " + f"({'no replacement in this repo' if finding.new_code is None else 'new block to copy into llms.txt'})" + f"\n\n```{finding.language}\n" + f"{finding.new_code if finding.new_code is not None else finding.old_code}\n```\n\n
" + for finding in findings + ) + plural = "block" if len(findings) == 1 else "blocks" + _write_summary( + "## llms.txt snippet sync\n\n" + f"This PR strands {len(findings)} code {plural} that `{LLMS_TXT_SOURCE}` publishes " + "verbatim. **`weaviate-io/static/llms.txt` must be updated in the same window**, " + "otherwise the weekly llms.txt coverage job starts failing once this merges.\n\n" + "| Snippet file | Marked region | What happened |\n" + "|---|---|---|\n" + f"{rows}\n\n" + f"{blocks}\n\n" + "This check never blocks the merge: weaviate-io cannot have shipped yet when a " + "legitimate snippet PR is opened.\n" + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Warn when a PR strands an llms.txt code block.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--base", + required=True, + help="git ref holding the pre-change state (on a PR, the base commit)", + ) + args = parser.parse_args() + + root = _repo_root() + if root is None: + _annotate("warning", "Not inside a git repository; skipping the llms.txt sync check.") + return 0 + os.chdir(root) + + changed, error = _changed_snippet_files(args.base) + if error is not None: + _annotate( + "warning", + f"Could not diff against {args.base} ({error}), so the llms.txt sync check was " + f"skipped. If this PR changes an llms.txt snippet, update {LLMS_TXT_SOURCE} too.", + title="llms.txt sync check could not run", + ) + return 0 + + if not changed: + _report_in_sync("No llms.txt-backed snippet files changed. Nothing to keep in sync.") + return 0 + + findings = _lost_regions(args.base, changed) + if not findings: + _report_in_sync( + f"{len(changed)} llms.txt snippet file(s) changed, but no `START`/`END` region " + f"lost its previous content, so `{LLMS_TXT_SOURCE}` stays valid." + ) + return 0 + + blocks, load_error = _llms_txt_blocks() + if blocks is None: + listed = ", ".join(f"`{path}`" for path in changed) + _annotate( + "warning", + f"Changed llms.txt snippet regions, but the published llms.txt could not be read " + f"({load_error}), so the comparison was skipped. Check {LLMS_TXT_SOURCE} by hand.", + title="llms.txt sync check could not run", + ) + _write_summary( + "## llms.txt snippet sync\n\n" + f"Could not read the published llms.txt ({load_error}). These files changed a " + f"marked region and may need a matching `{LLMS_TXT_SOURCE}` update: {listed}\n" + ) + return 0 + + stranded = [finding for finding in findings if finding.old_code in blocks[finding.language]] + if not stranded: + shipped = [ + finding + for finding in findings + if finding.new_code is not None and finding.new_code in blocks[finding.language] + ] + if shipped: + markers = ", ".join(f"`{finding.marker}`" for finding in shipped) + _report_in_sync( + f"`{LLMS_TXT_SOURCE}` already publishes the updated {markers} block(s). " + "weaviate-io shipped first, so there is nothing to do." + ) + else: + _report_in_sync( + f"Marked regions changed, but `{LLMS_TXT_SOURCE}` does not publish any of " + "their previous content, so nothing is stranded." + ) + return 0 + + _report_drift(stranded) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/docker-compose-anon-2.yml b/tests/docker-compose-anon-2.yml index 658c43a83..8567892c7 100644 --- a/tests/docker-compose-anon-2.yml +++ b/tests/docker-compose-anon-2.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 ports: - 8090:8080 - 50061:50051 diff --git a/tests/docker-compose-anon-bind.yml b/tests/docker-compose-anon-bind.yml index 094e83b74..0bc137c0b 100644 --- a/tests/docker-compose-anon-bind.yml +++ b/tests/docker-compose-anon-bind.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 ports: - 8380:8080 - 50351:50051 diff --git a/tests/docker-compose-anon-clip.yml b/tests/docker-compose-anon-clip.yml index 5349933c2..3968a8867 100644 --- a/tests/docker-compose-anon-clip.yml +++ b/tests/docker-compose-anon-clip.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 ports: - 8280:8080 - 50251:50051 diff --git a/tests/docker-compose-anon-offload.yml b/tests/docker-compose-anon-offload.yml index a4ef8c56b..fbc06efc9 100644 --- a/tests/docker-compose-anon-offload.yml +++ b/tests/docker-compose-anon-offload.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 ports: - 8080:8080 - 50051:50051 diff --git a/tests/docker-compose-anon.yml b/tests/docker-compose-anon.yml index 1ff802b26..06ca2b391 100644 --- a/tests/docker-compose-anon.yml +++ b/tests/docker-compose-anon.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 ports: - 8080:8080 - 50051:50051 diff --git a/tests/docker-compose-rbac.yml b/tests/docker-compose-rbac.yml index 5ea8add68..8e542e8b8 100644 --- a/tests/docker-compose-rbac.yml +++ b/tests/docker-compose-rbac.yml @@ -7,7 +7,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 ports: - 8580:8080 - 50551:50051 diff --git a/tests/docker-compose-three-nodes.yml b/tests/docker-compose-three-nodes.yml index a9bf30068..1634de0d3 100644 --- a/tests/docker-compose-three-nodes.yml +++ b/tests/docker-compose-three-nodes.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 restart: on-failure:0 ports: - "8180:8080" @@ -36,7 +36,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 restart: on-failure:0 ports: - "8181:8080" @@ -65,7 +65,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 restart: on-failure:0 ports: - "8182:8080" diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 5f56fcd80..487b02a5c 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 ports: - 8099:8080 - 50052:50051 diff --git a/tests/test_agents.py b/tests/test_agents.py index 5f97d6de2..93e5d0555 100644 --- a/tests/test_agents.py +++ b/tests/test_agents.py @@ -18,6 +18,7 @@ "./docs/query-agent/_includes/code/query_agent.py", "./docs/query-agent/_includes/code/quickstart.py", "./docs/query-agent/_includes/code/search_mode.py", + "./docs/query-agent/_includes/code/structured_outputs.py", "./docs/query-agent/_includes/code/suggest_queries.py", "./docs/query-agent/_includes/code/system_prompt.py", # Recipe walkthroughs (Weaviate Cloud + Weaviate Embeddings only) @@ -67,6 +68,7 @@ def test_recipes_requiring_openai_pyv4(script_loc): "./docs/query-agent/_includes/code/query_agent.mts", "./docs/query-agent/_includes/code/quickstart.mts", "./docs/query-agent/_includes/code/search_mode.mts", + "./docs/query-agent/_includes/code/structured_outputs.mts", "./docs/query-agent/_includes/code/suggest_queries.mts", "./docs/query-agent/_includes/code/system_prompt.mts", ], diff --git a/tests/test_docs_indexability.py b/tests/test_docs_indexability.py index 56b3af6cd..ff5ad51d6 100644 --- a/tests/test_docs_indexability.py +++ b/tests/test_docs_indexability.py @@ -31,15 +31,26 @@ ("/weaviate/search/hybrid", {"tabs", "code"}), ("/weaviate/connections/connect-cloud", {"tabs", "code"}), ("/weaviate/config-refs/collections", {"details", "table"}), - ("/weaviate/concepts/data-import", {"images"}), - ("/cloud/quickstart", {"code", "images"}), - ("/cloud/manage-clusters/create", {"images"}), + ("/weaviate/concepts/data-import", set()), + ("/cloud/quickstart", {"code"}), + ("/cloud/manage-clusters/create", set()), + ("/cloud/tools/query-tool", {"images"}), + ("/weaviate/manage-collections/tenant-states", {"images"}), ("/query-agent/recipes/query-agent-ecommerce-assistant", {"code"}), - ("/weaviate/search", set()), # landing page + ("/weaviate/search", set()), ] ALL_PATHS = [path for path, _ in TEST_PAGES] +# Pages that route readers onward instead of carrying content of their own, and +# so are exempt from the "content pages have h2 headings" rule. +# +# This is tracked separately from the feature sets because an empty feature set +# says only that a page has none of the structural features the checks below +# assert on. A page can be plain prose, with no tabs, code, details, table or +# image, and still be a content page that owes the reader h2 headings. +LANDING_PAGES = {"/weaviate/search"} + # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @@ -71,13 +82,6 @@ def _get_soup(path: str) -> BeautifulSoup: return BeautifulSoup(resp.text, "html.parser") -def _features_for(path: str) -> set[str]: - for p, features in TEST_PAGES: - if p == path: - return features - return set() - - # --------------------------------------------------------------------------- # Part 1: HTML Structure Tests (no API keys needed) # --------------------------------------------------------------------------- @@ -120,8 +124,7 @@ def test_heading_hierarchy(path): assert len(h1s) == 1, f"{path}: expected 1 h1, found {len(h1s)}" # Content pages (not landing pages) should have h2s - features = _features_for(path) - if features: # non-empty features means it's a content page + if path not in LANDING_PAGES: h2s = soup.find_all("h2") assert len(h2s) > 0, f"{path}: content page has no h2 headings" @@ -214,17 +217,16 @@ def test_details_content_present(path): ids=[p for p, f in TEST_PAGES if "images" in f], ) def test_images_have_alt_text(path): - """Content images have alt text (excludes SVG icons and badges).""" + """The page's own images have alt text (excludes SVG icons and badges).""" soup = _get_soup(path) + content_images = _content_images(soup) - # Find content images, excluding decorative ones - images = soup.find_all("img") - content_images = [ - img for img in images - if not _is_decorative_image(img) - ] - - assert len(content_images) > 0, f"{path}: no content images found" + if not content_images: + pytest.fail( + f"{path}: labelled 'images' but has no content image in the main " + "content region. Either the page lost its images, or the 'images' " + "label in TEST_PAGES is wrong." + ) missing_alt = [ img.get("src", "unknown") @@ -236,6 +238,26 @@ def test_images_have_alt_text(path): ) +def _main_content(soup): + """The page's main content region. + + Global navbar and footer chrome lives outside it. Scoping to this region + keeps site furniture (the site logo, the Ask AI button logo) from standing + in for the page's own images: that chrome is identical on every page, so + counting it made the alt-text check assert on the layout instead of the + page, and a single navbar change could flip it. + """ + return soup.find("main") or soup.find("article") or soup + + +def _content_images(soup): + """Images that carry the page's own content.""" + return [ + img for img in _main_content(soup).find_all("img") + if not _is_decorative_image(img) + ] + + def _is_decorative_image(img) -> bool: """Check if an image is decorative (SVG icon, badge, etc.).""" src = img.get("src", "") @@ -251,7 +273,17 @@ def _is_decorative_image(img) -> bool: # Skip language/site logo SVGs (e.g., /img/site/logo-py.svg) if src.endswith(".svg") and "/img/site/" in src: return True - # Skip analytics tracking pixels (e.g., Scarf) + # Skip images that carry no content by construction: analytics beacons, + # spacers, and anything hidden from both rendering and the accessibility + # tree. Such an image conveys nothing to a reader, a screen reader, or a + # crawler, so requiring alt text on it is meaningless. This is matched + # structurally rather than by hostname on purpose: the previous + # "static.scarf.sh" check went stale the moment the beacon moved to our own + # CNAME, and any hostname list will go stale again on the next move. + if _is_hidden_non_content(img): + return True + # Retained for the Scarf-hosted pixel, in case it is ever embedded without + # the hidden styling that the structural check above relies on. if "static.scarf.sh" in src: return True # Skip very small images (likely icons) @@ -262,6 +294,29 @@ def _is_decorative_image(img) -> bool: return False +def _is_hidden_non_content(img) -> bool: + """Check if an image is hidden from users and from the accessibility tree. + + Covers the ways a non-content image announces itself: + - inline `display: none` / `visibility: hidden` (removed from the render + tree, and therefore from the accessibility tree) + - `aria-hidden="true"` (explicitly withheld from assistive technology) + - 1x1 dimensions (a tracking pixel or a spacer) + """ + style = (img.get("style") or "").lower().replace(" ", "") + if "display:none" in style or "visibility:hidden" in style: + return True + + if str(img.get("aria-hidden", "")).lower() == "true": + return True + + dims = (img.get("width"), img.get("height")) + if all(d is not None and str(d).strip().isdigit() and int(d) <= 1 for d in dims): + return True + + return False + + @pytest.mark.indexability @pytest.mark.parametrize("path", ALL_PATHS) def test_llm_notice_present(path): diff --git a/tests/test_llms_txt_code.py b/tests/test_llms_txt_code.py index e89ca3c39..d0df9d7fd 100644 --- a/tests/test_llms_txt_code.py +++ b/tests/test_llms_txt_code.py @@ -120,6 +120,62 @@ def test_llms_txt_snippets_are_covered(): ) +# The PR-time drift check only runs when the PR touches a path in this workflow's +# `paths:` filter, so a SNIPPET_GLOBS entry outside that filter disables the check +# for that language without any visible failure. Java and C# snippets already live +# outside `_includes/code/llms-txt/`, so a fifth language landing outside it is a +# realistic way to lose coverage silently. +SNIPPET_SYNC_WORKFLOW = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + ".github", "workflows", "llms_txt_snippet_sync.yml", +) + + +def _glob_to_regex(pattern): + """Compile a GitHub `paths:` filter, where `**` spans directories and `*` does not.""" + out, i = [], 0 + while i < len(pattern): + if pattern.startswith("**", i): + out.append(".*") + i += 2 + elif pattern[i] == "*": + out.append("[^/]*") + i += 1 + else: + out.append(re.escape(pattern[i])) + i += 1 + return re.compile("".join(out) + r"\Z") + + +@pytest.mark.llms_txt +def test_snippet_globs_are_covered_by_workflow_paths(): + """Every SNIPPET_GLOBS pattern must trigger the snippet-sync workflow.""" + # Imported here, not at module scope: check_llms_txt_drift.py imports this + # module in a workflow that installs pytest and nothing else. + import yaml + + with open(SNIPPET_SYNC_WORKFLOW, encoding="utf-8") as handle: + workflow = yaml.safe_load(handle) + + # PyYAML reads the bare `on:` key as the YAML 1.1 boolean True. + triggers = workflow.get("on", workflow.get(True)) + filters = [_glob_to_regex(path) for path in triggers["pull_request"]["paths"]] + + uncovered = [ + pattern + for patterns in SNIPPET_GLOBS.values() + for pattern in patterns + if not any(filt.match(pattern) for filt in filters) + ] + + assert not uncovered, ( + "SNIPPET_GLOBS pattern(s) not covered by the `paths:` filter in " + f"{os.path.basename(SNIPPET_SYNC_WORKFLOW)}: {uncovered}. A snippet change under " + "these paths would not trigger the sync check, so llms.txt could drift unnoticed. " + "Add a matching entry to the workflow's `paths:` list." + ) + + # Each library: (weaviate/* GitHub repo, regex matching the # "**Library**: vX.Y.Z+" bullet in llms.txt). The regex anchors to the bullet's # prefix so adding new libraries to llms.txt doesn't break it. diff --git a/tests/test_python.py b/tests/test_python.py index 478f15171..af815e9d0 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -261,10 +261,6 @@ def test_search(empty_weaviates, script_loc): @pytest.mark.pyv4 -@pytest.mark.skip( - reason="Diversity/MMR not yet in a released weaviate-client; unskip once " - "https://github.com/weaviate/weaviate-python-client/pull/1997 ships" -) @pytest.mark.parametrize( "script_loc", [ diff --git a/uv.lock b/uv.lock index 2659b9c27..b0f0e87f5 100644 --- a/uv.lock +++ b/uv.lock @@ -1826,16 +1826,16 @@ wheels = [ [[package]] name = "weaviate-agents" -version = "1.6.0" +version = "1.7.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx-sse" }, { name = "rich" }, { name = "weaviate-client" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6e/a5/bd394d83a86153bcb54c297052558b312a247368c04ed08149850f0dcf71/weaviate_agents-1.6.0.tar.gz", hash = "sha256:a30c9c5120df3bc55ba1fa2384a4ebf5a3654d427a758294d9fa712cb5791576", size = 107790, upload-time = "2026-06-16T19:38:56.386Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b8/99/750d7d2e33f610bad0d4b1d77e2a7f0c73dbab5459876fa51c7a4a440a3a/weaviate_agents-1.7.0.tar.gz", hash = "sha256:f033085cbde7123424f5579dfab553df5e0128d395dfea3b58c5aea6602340cb", size = 111884, upload-time = "2026-07-27T08:31:58.622Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/96/69/9c5876da7925332d63151bcc61990e272cb8140a6dc7c6e887c905a3aede/weaviate_agents-1.6.0-py3-none-any.whl", hash = "sha256:e68723e3c14bf9639feadf307a55b7af4ddf2010e4d382e87b70766aeec8dae4", size = 50262, upload-time = "2026-06-16T19:38:55.445Z" }, + { url = "https://files.pythonhosted.org/packages/5e/d3/30952e1a4ca1556beb73e756d1f514c0bd62db53c21bdc65cdd6c5c6b78e/weaviate_agents-1.7.0-py3-none-any.whl", hash = "sha256:052e37ad9114424149b29deb8ba96159fce69f54e45874a4d6e42b1831643931", size = 54505, upload-time = "2026-07-27T08:31:57.755Z" }, ] [[package]] @@ -1884,6 +1884,7 @@ dependencies = [ { name = "playwright" }, { name = "pytest" }, { name = "python-dotenv" }, + { name = "pyyaml" }, { name = "requests" }, { name = "tqdm" }, { name = "weaviate-agents" }, @@ -1903,9 +1904,10 @@ requires-dist = [ { name = "playwright", specifier = ">=1.40.0" }, { name = "pytest", specifier = ">=8.3.5" }, { name = "python-dotenv", specifier = ">=1.1.1" }, + { name = "pyyaml", specifier = ">=6.0" }, { name = "requests", specifier = ">=2.32.3" }, { name = "tqdm", specifier = ">=4.67.1" }, - { name = "weaviate-agents", specifier = ">=1.6.0" }, + { name = "weaviate-agents", specifier = ">=1.7.0" }, { name = "weaviate-client", specifier = "==4.22.0" }, { name = "weaviate-demo-datasets", specifier = ">=0.8.1" }, { name = "weaviate-engram", specifier = ">=0.3.0" }, diff --git a/yarn.lock b/yarn.lock index 09ebd00d7..3bb957b4f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15500,10 +15500,10 @@ wcwidth@^1.0.1: dependencies: defaults "^1.0.3" -weaviate-agents@^1.5.0: - version "1.5.0" - resolved "https://registry.yarnpkg.com/weaviate-agents/-/weaviate-agents-1.5.0.tgz#a842c85bccfbdba172f8e3b00e61b8ff9953bfd7" - integrity sha512-0In5bvKaI8kaef8hX+NXp/VwJmkBYIsWz0zEH8Jw+ZEF/kORDzH3FNMcID29+TuoCrGQTRz/kbAx1LI68a+JRQ== +weaviate-agents@^1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/weaviate-agents/-/weaviate-agents-1.6.0.tgz#29139b1afa51c65b64b20b6c8ad180f6c46f7915" + integrity sha512-rOFYiZz9c+RdFj/G28xWp+6jx5I2Nl88dMB14QwZWPNBbieRTFzo8S+DeZbh/7zsXa9/oRjRiTMkxMb/VoNyVA== weaviate-client@^3.12.1: version "3.13.0" @@ -15976,6 +15976,11 @@ zod@^3.23.8: resolved "https://registry.npmjs.org/zod/-/zod-3.24.1.tgz" integrity sha512-muH7gBL9sI1nciMZV67X5fTKKBLtwpZ5VBp1vsOQzj1MhrBZ4wlVCm3gedKZWLp0Oyel8sIGfeiz54Su+OVT+A== +zod@^4.0.0: + version "4.4.3" + resolved "https://registry.yarnpkg.com/zod/-/zod-4.4.3.tgz#b680f172885d18bbebf21a834ea25e55a1bbf356" + integrity sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ== + zod@^4.1.8, zod@^4.3.5: version "4.3.6" resolved "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz"