Add incremental backup support

ci/docker-compose-rbac.yml

@@ -16,7 +16,11 @@ services:
       - 50062:50051
     restart: on-failure:0
     environment:
-      ENABLE_MODULES: "generative-dummy,reranker-dummy"
+      # backup-filesystem is enabled here so that backup behaviour which Weaviate only exposes to
+      # root users (e.g. incremental_base_backup_id) can be tested; ci/docker-compose-backup.yml
+      # is anonymous and therefore never has a root caller.
+      ENABLE_MODULES: "generative-dummy,reranker-dummy,backup-filesystem"
+      BACKUP_FILESYSTEM_PATH: "/tmp/backups"
       PERSISTENCE_DATA_PATH: "./data-weaviate-0"
       CLUSTER_IN_LOCALHOST: "true"
       CLUSTER_GOSSIP_BIND_PORT: "7100"

src/backup/backupCreator.ts

@@ -1,11 +1,12 @@
 import Connection from '../connection/index.js';
-import { WeaviateInvalidInputError } from '../errors.js';
+import { WeaviateInvalidInputError, WeaviateUnsupportedFeatureError } from '../errors.js';
 import {
   BackupConfig,
   BackupCreateRequest,
   BackupCreateResponse,
   BackupCreateStatusResponse,
 } from '../openapi/types.js';
+import { DbVersionSupport } from '../utils/dbVersion.js';
 import { CommandBase } from '../validation/commandBase.js';
 import BackupCreateStatusGetter from './backupCreateStatusGetter.js';
 import { Backend } from './index.js';
@@ -14,6 +15,7 @@ import {
   validateBackupId,
   validateExcludeClassNames,
   validateIncludeClassNames,
+  validateIncrementalBaseBackupId,
 } from './validation.js';
 
 const WAIT_INTERVAL = 1000;
@@ -26,10 +28,17 @@ export default class BackupCreator extends CommandBase {
   private statusGetter: BackupCreateStatusGetter;
   private waitForCompletion!: boolean;
   private config?: BackupConfig;
-
-  constructor(client: Connection, statusGetter: BackupCreateStatusGetter) {
+  private incrementalBaseBackupId?: string;
+  private dbVersionSupport?: DbVersionSupport;
+
+  constructor(
+    client: Connection,
+    statusGetter: BackupCreateStatusGetter,
+    dbVersionSupport?: DbVersionSupport
+  ) {
     super(client);
     this.statusGetter = statusGetter;
+    this.dbVersionSupport = dbVersionSupport;
   }
 
   withIncludeClassNames(...classNames: string[]) {
@@ -70,12 +79,27 @@ export default class BackupCreator extends CommandBase {
     return this;
   }
 
+  /**
+   * The ID of an existing backup to use as the base for a file-based incremental backup. Files
+   * that are identical to the base backup are not copied and are restored from the base instead,
+   * so deleting a base backup breaks every incremental backup built on it.
+   *
+   * Requires Weaviate v1.37.0 or higher.
+   */
+  withIncrementalBaseBackupId(backupId: string) {
+    // Weaviate lowercases backup IDs, so normalize here: the payload matches what the server
+    // stores, and validate() compares like for like.
+    this.incrementalBaseBackupId = typeof backupId === 'string' ? backupId.toLowerCase() : backupId;
+    return this;
+  }
+
   validate = (): void => {
     this.addErrors([
       ...validateIncludeClassNames(this.includeClassNames),
       ...validateExcludeClassNames(this.excludeClassNames),
       ...validateBackend(this.backend),
       ...validateBackupId(this.backupId),
+      ...validateIncrementalBaseBackupId(this.incrementalBaseBackupId, this.backupId),
     ]);
   };
 
@@ -90,12 +114,29 @@ export default class BackupCreator extends CommandBase {
       config: this.config,
       include: this.includeClassNames,
       exclude: this.excludeClassNames,
+      incremental_base_backup_id: this.incrementalBaseBackupId,
     } as BackupCreateRequest;
 
-    if (this.waitForCompletion) {
-      return this._createAndWaitForCompletion(payload);
+    return this.checkIncrementalSupport().then(() =>
+      this.waitForCompletion ? this._createAndWaitForCompletion(payload) : this._create(payload)
+    );
+  };
+
+  /**
+   * Weaviate below v1.37.0 ignores `incremental_base_backup_id` and silently writes a full backup,
+   * so fail loudly rather than hand back something other than what was asked for.
+   *
+   * No-op when the creator was built without a version provider, or for non-incremental backups.
+   */
+  private checkIncrementalSupport = (): Promise<void> => {
+    if (this.incrementalBaseBackupId === undefined || this.dbVersionSupport === undefined) {
+      return Promise.resolve();
     }
-    return this._create(payload);
+    return this.dbVersionSupport.supportsIncrementalBackups().then((check) => {
+      if (!check.supports) {
+        throw new WeaviateUnsupportedFeatureError(check.message);
+      }
+    });
   };
 
   _create = (payload: BackupCreateRequest): Promise<BackupCreateResponse> => {

src/backup/index.ts

@@ -1,4 +1,5 @@
 import Connection from '../connection/index.js';
+import { DbVersionSupport } from '../utils/dbVersion.js';
 import BackupCreateStatusGetter from './backupCreateStatusGetter.js';
 import BackupCreator from './backupCreator.js';
 import BackupRestoreStatusGetter from './backupRestoreStatusGetter.js';
@@ -22,9 +23,9 @@ export interface Backup {
   restoreStatusGetter: () => BackupRestoreStatusGetter;
 }
 
-const backup = (client: Connection): Backup => {
+const backup = (client: Connection, dbVersionSupport: DbVersionSupport): Backup => {
   return {
-    creator: () => new BackupCreator(client, new BackupCreateStatusGetter(client)),
+    creator: () => new BackupCreator(client, new BackupCreateStatusGetter(client), dbVersionSupport),
     createStatusGetter: () => new BackupCreateStatusGetter(client),
     restorer: () => new BackupRestorer(client, new BackupRestoreStatusGetter(client)),
     restoreStatusGetter: () => new BackupRestoreStatusGetter(client),

src/backup/validation.ts

@@ -45,3 +45,19 @@ export function validateBackupId(backupId?: string) {
   }
   return [];
 }
+
+export function validateIncrementalBaseBackupId(incrementalBaseBackupId?: string, backupId?: string) {
+  if (incrementalBaseBackupId === undefined || incrementalBaseBackupId === null) {
+    return [];
+  }
+  if (!isValidStringProperty(incrementalBaseBackupId)) {
+    return [
+      'string incrementalBaseBackupId must be a non-empty string - set with .withIncrementalBaseBackupId(backupId)',
+    ];
+  }
+  // Weaviate treats backup IDs as case-insensitive, so 'B1' and 'b1' name the same backup.
+  if (incrementalBaseBackupId.toLowerCase() === backupId?.toLowerCase()) {
+    return ['incrementalBaseBackupId must be different from the ID of the backup being created'];
+  }
+  return [];
+}

src/collections/backup/client.ts

@@ -5,7 +5,11 @@ import {
   BackupRestoreStatusGetter,
   BackupRestorer,
 } from '../../backup/index.js';
-import { validateBackend, validateBackupId } from '../../backup/validation.js';
+import {
+  validateBackend,
+  validateBackupId,
+  validateIncrementalBaseBackupId,
+} from '../../backup/validation.js';
 import Connection from '../../connection/index.js';
 import {
   WeaviateBackupCanceled,
@@ -14,24 +18,27 @@ import {
   WeaviateInvalidInputError,
   WeaviateUnexpectedResponseError,
   WeaviateUnexpectedStatusCodeError,
+  WeaviateUnsupportedFeatureError,
 } from '../../errors.js';
 import {
   BackupCreateResponse,
   BackupCreateStatusResponse,
+  BackupListResponse,
   BackupRestoreResponse,
 } from '../../openapi/types.js';
+import { DbVersionSupport } from '../../utils/dbVersion.js';
 import {
   BackupArgs,
   BackupCancelArgs,
-  BackupConfigCreate,
   BackupConfigRestore,
+  BackupCreateArgs,
   BackupReturn,
   BackupStatusArgs,
   BackupStatusReturn,
   ListBackupOptions,
 } from './types.js';
 
-export const backup = (connection: Connection): Backup => {
+export const backup = (connection: Connection, dbVersionSupport: DbVersionSupport): Backup => {
   const parseStatus = (res: BackupCreateStatusResponse | BackupRestoreResponse): BackupStatusReturn => {
     if (res.id === undefined) {
       throw new WeaviateUnexpectedResponseError('Backup ID is undefined in response');
@@ -47,6 +54,11 @@ export const backup = (connection: Connection): Backup => {
       error: res.error,
       path: res.path,
       status: res.status,
+      // Restore responses carry neither of these fields; see BackupStatusReturn for when Weaviate
+      // omits `incremental_base_backup_id` from a create status.
+      size: 'size' in res ? res.size : undefined,
+      incrementalBaseBackupId:
+        'incremental_base_backup_id' in res ? res.incremental_base_backup_id || undefined : undefined,
     };
   };
   const parseResponse = (res: BackupCreateResponse | BackupRestoreResponse): BackupReturn => {
@@ -109,10 +121,22 @@ export const backup = (connection: Connection): Backup => {
 
       return true;
     },
-    create: async (args: BackupArgs<BackupConfigCreate>): Promise<BackupReturn> => {
+    create: async (args: BackupCreateArgs): Promise<BackupReturn> => {
       let builder = new BackupCreator(connection, new BackupCreateStatusGetter(connection))
         .withBackupId(args.backupId)
         .withBackend(args.backend);
+      if (args.incrementalBaseBackupId !== undefined) {
+        const errors = validateIncrementalBaseBackupId(args.incrementalBaseBackupId, args.backupId);
+        if (errors.length > 0) {
+          throw new WeaviateInvalidInputError(errors.join(', '));
+        }
+        const check = await dbVersionSupport.supportsIncrementalBackups();
+        if (!check.supports) {
+          throw new WeaviateUnsupportedFeatureError(check.message);
+        }
+        // The builder lowercases the ID to match how Weaviate stores it.
+        builder = builder.withIncrementalBaseBackupId(args.incrementalBaseBackupId);
+      }
       if (args.includeCollections) {
         builder = builder.withIncludeClassNames(...args.includeCollections);
       }
@@ -213,7 +237,12 @@ export const backup = (connection: Connection): Backup => {
       if (opts?.startedAtAsc) {
         url += '?order=asc';
       }
-      return connection.get<BackupReturn[]>(url);
+      return connection.get<BackupListResponse>(url).then((res) =>
+        res.map(({ incremental_base_backup_id: baseBackupId, ...rest }) => ({
+          ...rest,
+          incrementalBaseBackupId: baseBackupId || undefined,
+        }))
+      ) as Promise<BackupReturn[]>;
     },
   };
 };
@@ -231,13 +260,16 @@ export interface Backup {
   /**
    * Create a backup of the database.
    *
-   * @param {BackupArgs} args The arguments for the request.
+   * Set `incrementalBaseBackupId` for a file-based incremental backup (Weaviate `v1.37.0` or higher).
+   *
+   * @param {BackupCreateArgs} args The arguments for the request.
    * @returns {Promise<BackupReturn>} The response from Weaviate.
    * @throws {WeaviateInvalidInputError} If the input is invalid.
+   * @throws {WeaviateUnsupportedFeatureError} If `incrementalBaseBackupId` is used with Weaviate <1.37.0.
    * @throws {WeaviateBackupFailed} If the backup creation fails.
    * @throws {WeaviateBackupCanceled} If the backup creation is canceled.
    */
-  create(args: BackupArgs<BackupConfigCreate>): Promise<BackupReturn>;
+  create(args: BackupCreateArgs): Promise<BackupReturn>;
   /**
    * Get the status of a backup creation.
    *

src/collections/backup/collection.ts

@@ -1,7 +1,8 @@
 import { Backend } from '../../backup/index.js';
 import Connection from '../../connection/index.js';
+import { DbVersionSupport } from '../../utils/dbVersion.js';
 import { backup } from './client.js';
-import { BackupReturn, BackupStatusArgs, BackupStatusReturn } from './types.js';
+import { BackupCreateArgs, BackupReturn, BackupStatusArgs, BackupStatusReturn } from './types.js';
 
 /** The arguments required to create and restore backups. */
 export type BackupCollectionArgs = {
@@ -13,10 +14,18 @@ export type BackupCollectionArgs = {
   waitForCompletion?: boolean;
 };
 
-export const backupCollection = (connection: Connection, name: string) => {
-  const handler = backup(connection);
+/** The arguments required to create a backup of a collection. */
+export type BackupCollectionCreateArgs = BackupCollectionArgs &
+  Pick<BackupCreateArgs, 'incrementalBaseBackupId'>;
+
+export const backupCollection = (
+  connection: Connection,
+  name: string,
+  dbVersionSupport: DbVersionSupport
+) => {
+  const handler = backup(connection, dbVersionSupport);
   return {
-    create: (args: BackupCollectionArgs) =>
+    create: (args: BackupCollectionCreateArgs) =>
       handler.create({
         ...args,
         includeCollections: [name],
@@ -35,13 +44,16 @@ export interface BackupCollection {
   /**
    * Create a backup of this collection.
    *
-   * @param {BackupArgs} args The arguments for the request.
+   * Set `incrementalBaseBackupId` for a file-based incremental backup (Weaviate `v1.37.0` or higher).
+   *
+   * @param {BackupCollectionCreateArgs} args The arguments for the request.
    * @returns {Promise<BackupReturn>} The response from Weaviate.
    * @throws {WeaviateInvalidInputError} If the input is invalid.
+   * @throws {WeaviateUnsupportedFeatureError} If `incrementalBaseBackupId` is used with Weaviate <1.37.0.
    * @throws {WeaviateBackupFailed} If the backup creation fails.
    * @throws {WeaviateBackupCanceled} If the backup creation is canceled.
    */
-  create(args: BackupCollectionArgs): Promise<BackupReturn>;
+  create(args: BackupCollectionCreateArgs): Promise<BackupReturn>;
   /**
    * Get the status of a backup.
    *

src/collections/backup/index.ts

@@ -1,3 +1,9 @@
 export type { Backup } from './client.js';
-export type { BackupCollection, BackupCollectionArgs } from './collection.js';
-export type { BackupArgs, BackupConfigCreate, BackupConfigRestore, BackupStatusArgs } from './types.js';
+export type { BackupCollection, BackupCollectionArgs, BackupCollectionCreateArgs } from './collection.js';
+export type {
+  BackupArgs,
+  BackupConfigCreate,
+  BackupConfigRestore,
+  BackupCreateArgs,
+  BackupStatusArgs,
+} from './types.js';

src/collections/backup/types.ts

@@ -16,6 +16,14 @@ export type BackupStatusReturn = {
   status: BackupStatus;
   /** Size of the backup in Gibs */
   size?: number;
+  /**
+   * The ID of the base backup an incremental backup was built on.
+   *
+   * Undefined when the backup is not incremental. Also undefined for an incremental backup when the
+   * caller is not a root user, when the server is older than `v1.37.6`, and on the return of
+   * `create()` without `waitForCompletion` — the create response carries no such field.
+   */
+  incrementalBaseBackupId?: string;
 };
 
 /** The return type of a backup creation or restoration operation */
@@ -64,6 +72,12 @@ export type BackupArgs<C extends BackupConfigCreate | BackupConfigRestore> = {
   config?: C;
 };
 
+/** The arguments required to create a backup. */
+export type BackupCreateArgs = BackupArgs<BackupConfigCreate> & {
+  /** The ID of an existing backup to build a file-based incremental backup on. Files identical to the base are not copied and are restored from the base instead, so deleting a base backup breaks every incremental built on it. Requires Weaviate `v1.37.0` or higher. */
+  incrementalBaseBackupId?: string;
+};
+
 /** The arguments required to get the status of a backup. */
 export type BackupStatusArgs = {
   /** The ID of the backup. */

src/collections/collection/index.ts

@@ -138,7 +138,7 @@ const collection = <T, N, V>(
   );
   return {
     aggregate: aggregateCollection,
-    backup: backupCollection(connection, capitalizedName),
+    backup: backupCollection(connection, capitalizedName, dbVersionSupport),
     config: config<T>(connection, capitalizedName, dbVersionSupport, tenant),
     data: data<T>(connection, capitalizedName, dbVersionSupport, consistencyLevel, tenant),
     filter: filter<T extends undefined ? any : T>(),

src/index.ts

@@ -233,7 +233,7 @@ async function client(params: ClientParams): Promise<WeaviateClient> {
 
   const ifc: WeaviateClient = {
     alias: alias(connection),
-    backup: backup(connection),
+    backup: backup(connection, dbVersionSupport),
     batch: batch(connection, dbVersionSupport),
     cluster: cluster(connection),
     collections: collections(connection, dbVersionSupport),

src/openapi/types.ts

@@ -15,6 +15,7 @@ export type DataObject = definitions['Object'];
 export type BackupCreateRequest = definitions['BackupCreateRequest'];
 export type BackupCreateResponse = definitions['BackupCreateResponse'];
 export type BackupCreateStatusResponse = definitions['BackupCreateStatusResponse'];
+export type BackupListResponse = definitions['BackupListResponse'];
 export type BackupRestoreRequest = definitions['BackupRestoreRequest'];
 export type BackupRestoreResponse = definitions['BackupRestoreResponse'];
 export type BackupRestoreStatusResponse = definitions['BackupRestoreStatusResponse'];

src/utils/dbVersion.ts

@@ -163,6 +163,13 @@ export class DbVersionSupport {
       message: this.errorMessage('Tokenize endpoint stopwords / stopwordPresets', version.show(), '1.37.2'),
     }));
 
+  supportsIncrementalBackups = () =>
+    this.dbVersionProvider.getVersion().then((version) => ({
+      version,
+      supports: version.isAtLeast(1, 37, 0),
+      message: this.errorMessage('Incremental backups', version.show(), '1.37.0'),
+    }));
+
   supportsServerSideDefaultVectorIndexType = () =>
     this.dbVersionProvider.getVersion().then((version) => ({
       version,