Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions libs/vectoriadb/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@
"default": "./dist/src/worker.js"
}
},
"dependencies": {
"tslib": "^2.8.1"
},
"peerDependencies": {
"@huggingface/transformers": "^3.2.2"
},
Expand Down
32 changes: 32 additions & 0 deletions libs/vectoriadb/src/__tests__/worker-entry.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
/**
* The worker/browser entry must load without Node built-ins: a browser bundle cannot resolve them,
* and a V8 isolate without Node compatibility has none.
*/

const NODE_BUILT_INS = ['crypto', 'fs', 'fs/promises', 'path', 'os', 'child_process', 'net', 'stream'];

describe('the worker entry', () => {
it('loads without requiring a Node built-in', () => {
jest.isolateModules(() => {
for (const builtIn of NODE_BUILT_INS) {
jest.doMock(builtIn, () => {
throw new Error(`the worker entry required the Node built-in "${builtIn}"`);
});
}

expect(() => require('../worker')).not.toThrow();
});
});

it('builds and searches a TF-IDF index', () => {
jest.isolateModules(() => {
const { TFIDFVectoria } = require('../worker');
const db = new TFIDFVectoria({ defaultTopK: 2 });
db.addDocument('deploy', 'deploy the server to cloudflare workers', { id: 'deploy' });
db.addDocument('test', 'write unit tests with jest', { id: 'test' });
db.reindex();

expect(db.search('cloudflare deploy')[0]?.id).toBe('deploy');
});
});
});
30 changes: 30 additions & 0 deletions libs/vectoriadb/src/sanitize.utils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/**
* Sanitize an object to prevent prototype pollution
* Creates a clean object without dangerous properties
*/
export function sanitizeObject(obj: any): any {
if (obj === null || typeof obj !== 'object') {
return obj;
}

// Handle arrays
if (Array.isArray(obj)) {
return obj.map((item) => sanitizeObject(item));
}

// Create clean object without prototype chain
const clean: any = {};

// Copy only safe properties
for (const key of Object.keys(obj)) {
// Block dangerous keys
if (key === '__proto__' || key === 'constructor' || key === 'prototype') {
continue;
}

// Recursively sanitize nested objects
clean[key] = sanitizeObject(obj[key]);
}

return clean;
}
34 changes: 3 additions & 31 deletions libs/vectoriadb/src/storage/serialization.utils.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import * as crypto from 'crypto';
import type { DocumentEmbedding, DocumentMetadata } from '../interfaces';
import type { SerializedEmbedding } from './adapter.interface';
import { sanitizeObject } from '../sanitize.utils';

export { sanitizeObject };

/**
* Serialize a DocumentEmbedding to a SerializedEmbedding
Expand Down Expand Up @@ -55,34 +58,3 @@ export function createToolsHash(documents: Array<{ id: string; text: string }>):
.join('|');
return hash(content);
}

/**
* Sanitize an object to prevent prototype pollution
* Creates a clean object without dangerous properties
*/
export function sanitizeObject(obj: any): any {
if (obj === null || typeof obj !== 'object') {
return obj;
}

// Handle arrays
if (Array.isArray(obj)) {
return obj.map((item) => sanitizeObject(item));
}

// Create clean object without prototype chain
const clean: any = {};

// Copy only safe properties
for (const key of Object.keys(obj)) {
// Block dangerous keys
if (key === '__proto__' || key === 'constructor' || key === 'prototype') {
continue;
}

// Recursively sanitize nested objects
clean[key] = sanitizeObject(obj[key]);
}

return clean;
}
2 changes: 1 addition & 1 deletion libs/vectoriadb/src/vectoria-tfidf.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { TFIDFEmbeddingService } from './tfidf.embedding.service';
import type { DocumentMetadata, SearchOptions, SearchResult } from './interfaces';
import { sanitizeObject } from './storage/serialization.utils';
import { sanitizeObject } from './sanitize.utils';

/**
* Document with TF-IDF sparse vector representation
Expand Down
1 change: 1 addition & 0 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -6803,6 +6803,7 @@ __metadata:
version: 0.0.0-use.local
resolution: "vectoriadb@workspace:libs/vectoriadb"
dependencies:
tslib: "npm:^2.8.1"
typescript: "npm:^5.9.3"
peerDependencies:
"@huggingface/transformers": ^3.2.2
Expand Down
Loading