Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -218,3 +218,4 @@ __pycache__/
*.metallib
.vite-agent
/Model-Optimizer
.vitest/
15 changes: 9 additions & 6 deletions .yarnrc.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
approvedGitRepositories:
- "**"
- '**'

catalog:
"@types/node": "npm:@types/node@26.4.0"
vite: "npm:@voidzero-dev/vite-plus-core@0.3.0"
vite-plus: 0.3.0
vitest: "npm:vitest@4.1.11"
'@types/node': 'npm:@types/node@26.4.0'
vite: npm:@voidzero-dev/vite-plus-core@0.0.0-commit.e0f77ee2353b38c77d7165d3305f9f6d705b4d62
vite-plus: 0.0.0-commit.e0f77ee2353b38c77d7165d3305f9f6d705b4d62

enableScripts: true

Expand All @@ -15,6 +14,10 @@ npmMinimalAgeGate: 0

npmScopes:
jsr:
npmRegistryServer: "https://npm.jsr.io"
npmRegistryServer: 'https://npm.jsr.io'

yarnPath: .yarn/releases/yarn-4.18.0.cjs
npmPreapprovedPackages:
- vitest
- '@vitest/*'
npmRegistryServer: https://registry-bridge.viteplus.dev/
6 changes: 1 addition & 5 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,7 @@ serde = { version = "1.0.229", features = ["derive"] }
thiserror = "2.0.20"
tokio = { version = "1.53", features = ["rt-multi-thread", "macros", "sync"] }
tracing = "0.1.44"
tracing-subscriber = { version = "0.3.23", default-features = false, features = [
"std",
"fmt",
"json",
] }
tracing-subscriber = { version = "0.3.23", default-features = false, features = ["std", "fmt", "json"] }
tokenizers = "0.23.1"
minijinja = { version = "2.24", features = ["json", "preserve_order"] }
uuid = { version = "1.26", features = ["v4"] }
Expand Down
11 changes: 10 additions & 1 deletion __test__/cli/serve.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
import { connect } from 'node:net';
import { tmpdir } from 'node:os';
import { join } from 'node:path';

Expand Down Expand Up @@ -164,7 +165,15 @@ describe('mlx serve — end to end', () => {

expect(observedUrl).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/);
expect(exitCodes).toEqual([0]);
await expect(fetch(healthUrl)).rejects.toThrow();
const connectionError = await new Promise<NodeJS.ErrnoException | null>((resolve) => {
const socket = connect(Number(new URL(healthUrl).port), '127.0.0.1');
socket.once('error', (error) => resolve(error as NodeJS.ErrnoException));
socket.once('connect', () => {
socket.destroy();
resolve(null);
});
});
expect(connectionError?.code).toBe('ECONNREFUSED');

// Handlers detached on shutdown: a leaked one would keep a closed host
// alive in the listener list and swallow the next Ctrl+C.
Expand Down
4 changes: 2 additions & 2 deletions __test__/models/chat-session-mtp-default.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@

/** The session surface every case needs, minus the MTP getters. */
function sessionStubs() {
const chatSessionStart = vi.fn(
async (_messages: ChatMessage[], _config?: ChatConfig | null): Promise<ChatResult> => makeChatResult('reply'),
const chatSessionStart = vi.fn(async (_messages: ChatMessage[], _config?: ChatConfig | null): Promise<ChatResult> =>
makeChatResult('reply'),
);
const chatSessionContinue = vi.fn(
async (_messages: ChatMessage[], _config?: ChatConfig | null): Promise<ChatResult> => makeChatResult('reply'),
Expand Down Expand Up @@ -231,6 +231,6 @@
// Collapsing the two into one would make the head unreachable rather than
// merely off-by-default.
expect(typeof NemotronHModel.prototype.hasMtpWeights).toBe('function');
expect(NemotronHModel.prototype.mtpAutoEnabled).not.toBe(NemotronHModel.prototype.hasMtpWeights);

Check warning on line 234 in __test__/models/chat-session-mtp-default.test.ts

View workflow job for this annotation

GitHub Actions / Rust Lint

typescript(unbound-method)

__test__/models/chat-session-mtp-default.test.ts:234:87: Avoid referencing unbound methods which may cause unintentional scoping of `this`.

Check warning on line 234 in __test__/models/chat-session-mtp-default.test.ts

View workflow job for this annotation

GitHub Actions / Rust Lint

typescript(unbound-method)

__test__/models/chat-session-mtp-default.test.ts:234:37: Avoid referencing unbound methods which may cause unintentional scoping of `this`.
});
});
4 changes: 2 additions & 2 deletions __test__/models/gemma4-concurrency-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ function isScheduledGemma(model: LoadableModel): model is LoadableModel & Schedu
);
}

const gemmaDescribe = MODEL_ENV_PRESENT ? describe.sequential : describe.skip;
const gemmaDescribe = MODEL_ENV_PRESENT ? describe : describe.skip;

gemmaDescribe('Gemma4 grouped hybrid KV continuous batching', () => {
gemmaDescribe('Gemma4 grouped hybrid KV continuous batching', { concurrent: false }, () => {
let model: ScheduledGemma;
const sessions: ChatSession[] = [];
const activeSends = new Set<{
Expand Down
2 changes: 1 addition & 1 deletion __test__/models/gemma4-unified-detection.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vite-plus/test';
* resolve to the one shared `gemma4` Rust decoder + `Gemma4Model` registry
* row — there is no separate unified registry entry.
*/
describe.sequential('Gemma4 unified model detection', () => {
describe('Gemma4 unified model detection', { concurrent: false }, () => {
let tempDir: string;

beforeEach(async () => {
Expand Down
2 changes: 1 addition & 1 deletion __test__/models/harrier-detection.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { join } from 'node:path';
import { detectModelType } from '@mlx-node/lm';
import { describe, it, expect, beforeEach, afterEach } from 'vite-plus/test';

describe.sequential('Harrier Model Detection', () => {
describe('Harrier Model Detection', { concurrent: false }, () => {
let tempDir: string;

beforeEach(async () => {
Expand Down
2 changes: 1 addition & 1 deletion __test__/models/harrier.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ const HARRIER_0_6B_CONFIG = {
useQkNorm: true,
};

describe.sequential('HarrierModel', () => {
describe('HarrierModel', { concurrent: false }, () => {
describe('Model Instantiation', () => {
it('should create model from tiny config', () => {
const model = new HarrierModel(TINY_CONFIG);
Expand Down
2 changes: 1 addition & 1 deletion __test__/models/lfm2-moe-detection.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { join } from 'node:path';
import { detectModelType } from '@mlx-node/lm';
import { describe, it, expect, beforeEach, afterEach } from 'vite-plus/test';

describe.sequential('LFM2 MoE Model Detection', () => {
describe('LFM2 MoE Model Detection', { concurrent: false }, () => {
let tempDir: string;

beforeEach(async () => {
Expand Down
2 changes: 1 addition & 1 deletion __test__/models/model-loader-registry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ const UNIFIED_GEMMA_COMPATIBILITY_CASES = [
['competing Harrier probe', { model_type: 'qwen3', architectures: ['Qwen3Model', UNIFIED_GEMMA_ARCHITECTURE] }],
] as const satisfies readonly (readonly [string, Readonly<Record<string, unknown>>])[];

describe.sequential('declarative model loader registry', () => {
describe('declarative model loader registry', { concurrent: false }, () => {
let tempDir: string;

beforeEach(async () => {
Expand Down
2 changes: 1 addition & 1 deletion __test__/models/nemotron-h-detection.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vite-plus/test';
* (mirroring the native loader's config validation in
* crates/mlx-core/src/models/nemotron_h/config.rs).
*/
describe.sequential('Nemotron H model detection', () => {
describe('Nemotron H model detection', { concurrent: false }, () => {
let tempDir: string;

beforeEach(async () => {
Expand Down
2 changes: 1 addition & 1 deletion __test__/models/qwen3-chat.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import { describe, it, expect, beforeAll, afterAll } from 'vite-plus/test';

import { createTempModel, findTokenizerPath, TINY_TEST_CONFIG } from '../test-model-utils';

describe.sequential('Qwen3 Chat Session API', () => {
describe('Qwen3 Chat Session API', { concurrent: false }, () => {
let model: Qwen3Model;
let cleanup: () => void;

Expand Down
2 changes: 1 addition & 1 deletion __test__/models/qwen3-generate-batch-parallel.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import { describe, it, expect, beforeAll } from 'vite-plus/test';
* Each test creates its own GrpoTrainingEngine and resets it when done
* to release the model thread's training state for the next test.
*/
describe.sequential('GRPOTrainer - Parallel Batch Generation', () => {
describe('GRPOTrainer - Parallel Batch Generation', { concurrent: false }, () => {
let model: Qwen3Model | null = null;

beforeAll(async () => {
Expand Down
2 changes: 1 addition & 1 deletion __test__/models/qwen3.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { Qwen3Model } from '@mlx-node/core';
import { QWEN3_CONFIGS, getQwen3Config } from '@mlx-node/lm';
import { describe, it, expect } from 'vite-plus/test';

describe.sequential('Qwen3 Model', () => {
describe('Qwen3 Model', { concurrent: false }, () => {
describe('Model Configuration', () => {
it('should have correct default configurations', () => {
expect(QWEN3_CONFIGS['qwen3-0.6b']).toBeDefined();
Expand Down
2 changes: 1 addition & 1 deletion __test__/models/qwen35-stream.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ function fakeNativeMethod(numTokens: number) {
};
}

describe.sequential('_runChatStream bridge', () => {
describe('_runChatStream bridge', { concurrent: false }, () => {
it('should yield delta chunks followed by a final chunk', async () => {
const events: ChatStreamEvent[] = [];
const messages: ChatMessage[] = [{ role: 'user', content: 'Hi' }];
Expand Down
2 changes: 1 addition & 1 deletion __test__/models/qwen3_5-generation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { afterAll, beforeAll, describe, expect, it } from 'vite-plus/test';
import { createTempQwen35Model, type TempModel } from '../test-model-utils';
import { shape } from '../test-utils';

describe.sequential('Qwen3.5 Generation', () => {
describe('Qwen3.5 Generation', { concurrent: false }, () => {
let tempModel: TempModel;

beforeAll(async () => {
Expand Down
4 changes: 2 additions & 2 deletions __test__/server/concurrent-stage0-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -330,9 +330,9 @@ function startTickProbe(intervalMs = 50): { stop: () => number } {
};
}

const stage0Describe = MODEL_ENV_PRESENT ? describe.sequential : describe.skip;
const stage0Describe = MODEL_ENV_PRESENT ? describe : describe.skip;

stage0Describe('Stage-0 real-model concurrency hazards', () => {
stage0Describe('Stage-0 real-model concurrency hazards', { concurrent: false }, () => {
let instance: ServerInstance;
let nativeModel: SessionCapableModel;
let instrumented: InstrumentedModel;
Expand Down
4 changes: 2 additions & 2 deletions __test__/server/concurrent-stage1-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -162,9 +162,9 @@ async function postResponse(baseUrl: string, input: string): Promise<{ id?: stri
});
}

const stage1Describe = MODEL_ENV_PRESENT ? describe.sequential : describe.skip;
const stage1Describe = MODEL_ENV_PRESENT ? describe : describe.skip;

stage1Describe('Stage-1 real-model server admission', () => {
stage1Describe('Stage-1 real-model server admission', { concurrent: false }, () => {
let instance: ServerInstance;
let nativeModel: SchedulerStatsModel;
let peakActive: () => number;
Expand Down
28 changes: 23 additions & 5 deletions __test__/server/host/inference-host.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { spawnSync } from 'node:child_process';
import { existsSync } from 'node:fs';
import { chmod, mkdir, mkdtemp, readdir, rm, writeFile } from 'node:fs/promises';
import { createServer as createNetServer } from 'node:net';
import { connect as connectNet, createServer as createNetServer } from 'node:net';
import { tmpdir } from 'node:os';
import { join } from 'node:path';

Expand Down Expand Up @@ -324,16 +324,28 @@ describe('createInferenceHost — binding and health', () => {
const address = host.server.server.address();
expect(address !== null && typeof address === 'object' ? address.address : null).toBe('::1');
expect(host.url).toBe(`http://[::1]:${host.port}`);
expect((await fetch(`${host.url}/health`)).status).toBe(200);
const response = await fetch(`${host.url}/health`);
expect(response.status).toBe(200);
await response.arrayBuffer();
});

it('stops answering once closed', async () => {
const modelsDir = await makeModelsDir(['alpha']);
const host = await start({ modelsDir });
const url = `${host.url}/health`;
expect((await fetch(url)).status).toBe(200);
const response = await fetch(url);
expect(response.status).toBe(200);
await response.arrayBuffer();
await host.close({ timeoutMs: 0 });
await expect(fetch(url)).rejects.toThrow();
const connectionError = await new Promise<NodeJS.ErrnoException | null>((resolve) => {
const socket = connectNet(host.port, '127.0.0.1');
socket.once('error', (error) => resolve(error as NodeJS.ErrnoException));
socket.once('connect', () => {
socket.destroy();
resolve(null);
});
});
expect(connectionError?.code).toBe('ECONNREFUSED');
});

/**
Expand Down Expand Up @@ -559,7 +571,13 @@ describe('createInferenceHost — a network-reachable bind must be gated', () =>
const modelsDir = await makeModelsDir(['alpha']);
for (const bind of ['127.0.0.1', 'localhost', undefined]) {
const host = await start({ modelsDir, host: bind, port: 0 });
expect((await fetch(`${host.url}/v1/models`)).status, String(bind)).toBe(200);
expect(host.host).toBe(bind ?? '127.0.0.1');
const address = host.server.server.address();
if (address === null || typeof address === 'string') throw new Error('Expected an IP socket');
const ip = address.family === 'IPv6' ? `[${address.address}]` : address.address;
const response = await fetch(`http://${ip}:${address.port}/v1/models`);
expect(response.status, String(bind)).toBe(200);
await response.arrayBuffer();
}
});
});
Expand Down
2 changes: 1 addition & 1 deletion __test__/server/model-work-coordinator.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { describe, expect, it } from 'vitest';
import { describe, expect, it } from 'vite-plus/test';

import { ModelWorkCoordinator } from '../../packages/server/src/model-work-coordinator.js';

Expand Down
12 changes: 10 additions & 2 deletions __test__/server/server-close.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { once } from 'node:events';
import { request as httpRequest } from 'node:http';
import type { AddressInfo } from 'node:net';
import { connect, type AddressInfo } from 'node:net';

import type { SessionCapableModel } from '@mlx-node/lm';
import { activeSSEStreamCount, createServer, type ServerInstance } from '@mlx-node/server';
Expand Down Expand Up @@ -278,7 +278,15 @@ describe('close() on a quiet server', () => {
it('stops the server from accepting new requests', async () => {
const { instance, base } = await start();
await instance.close({ timeoutMs: 1_000 });
await expect(fetch(`${base}/health`)).rejects.toThrow();
const connectionError = await new Promise<NodeJS.ErrnoException | null>((resolve) => {
const socket = connect(Number(new URL(base).port), '127.0.0.1');
socket.once('error', (error) => resolve(error as NodeJS.ErrnoException));
socket.once('connect', () => {
socket.destroy();
resolve(null);
});
});
expect(connectionError?.code).toBe('ECONNREFUSED');
});
});

Expand Down
15 changes: 3 additions & 12 deletions __test__/tokenization/qwen3-tokenizer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,16 @@
* Comprehensive tests for the Rust-based Qwen3 tokenizer implementation
*/

import { join } from 'node:path';
import { fileURLToPath } from 'node:url';

import { Qwen3Tokenizer, type ChatMessage } from '@mlx-node/core';
import { describe, it, expect, beforeAll } from 'vite-plus/test';

import { findTokenizerPath } from '../test-model-utils';

describe('Qwen3Tokenizer', () => {
let tokenizer: Qwen3Tokenizer;

const TOKENIZER_PATH = join(
fileURLToPath(import.meta.url),
'..',
'..',
'..',
'.cache/models/qwen3-0.6b-mlx-bf16/tokenizer.json',
);

beforeAll(async () => {
tokenizer = await Qwen3Tokenizer.fromPretrained(TOKENIZER_PATH);
tokenizer = await Qwen3Tokenizer.fromPretrained(findTokenizerPath());
});

describe('Initialization', () => {
Expand Down
Loading
Loading