diff --git a/crowd-pilot-serializer b/crowd-pilot-serializer index 9c35c9d..7c457f7 160000 --- a/crowd-pilot-serializer +++ b/crowd-pilot-serializer @@ -1 +1 @@ -Subproject commit 9c35c9d72d366f28d97066022ad37bb2f708ed72 +Subproject commit 7c457f79a729a892b705deb6d1265e9570931baa diff --git a/package-lock.json b/package-lock.json index 6c52f03..55c55b3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,16 +10,17 @@ "devDependencies": { "@types/mocha": "^10.0.10", "@types/node": "22.x", - "@types/vscode": "^1.105.0", + "@types/vscode": "^1.99.3", "@typescript-eslint/eslint-plugin": "^8.45.0", "@typescript-eslint/parser": "^8.45.0", "@vscode/test-cli": "^0.0.11", "@vscode/test-electron": "^2.5.2", "eslint": "^9.36.0", + "mocha": "^10.8.2", "typescript": "^5.9.3" }, "engines": { - "vscode": "^1.105.0" + "vscode": "^1.99.3" } }, "node_modules/@bcoe/v8-coverage": { @@ -679,6 +680,100 @@ "node": ">=18" } }, + "node_modules/@vscode/test-cli/node_modules/diff": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz", + "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/@vscode/test-cli/node_modules/mocha": { + "version": "11.7.5", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-11.7.5.tgz", + "integrity": "sha512-mTT6RgopEYABzXWFx+GcJ+ZQ32kp4fMf0xvpZIIfSq9Z8lC/++MtcCnQ9t5FP2veYEP95FIYSvW+U9fV4xrlig==", + "dev": true, + "dependencies": { + "browser-stdout": "^1.3.1", + "chokidar": "^4.0.1", + "debug": "^4.3.5", + "diff": "^7.0.0", + "escape-string-regexp": "^4.0.0", + "find-up": "^5.0.0", + "glob": "^10.4.5", + "he": "^1.2.0", + "is-path-inside": "^3.0.3", + "js-yaml": "^4.1.0", + "log-symbols": "^4.1.0", + "minimatch": "^9.0.5", + "ms": "^2.1.3", + "picocolors": "^1.1.1", + "serialize-javascript": "^6.0.2", + "strip-json-comments": "^3.1.1", + "supports-color": "^8.1.1", + "workerpool": "^9.2.0", + "yargs": "^17.7.2", + "yargs-parser": "^21.1.1", + "yargs-unparser": "^2.0.0" + }, + "bin": { + "_mocha": "bin/_mocha", + "mocha": "bin/mocha.js" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@vscode/test-cli/node_modules/mocha/node_modules/chokidar": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "dev": true, + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@vscode/test-cli/node_modules/mocha/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/@vscode/test-cli/node_modules/readdirp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "dev": true, + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@vscode/test-cli/node_modules/workerpool": { + "version": "9.3.4", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-9.3.4.tgz", + "integrity": "sha512-TmPRQYYSAnnDiEB0P/Ytip7bFGvqnSU6I2BcuSw7Hx+JSg/DsUi5ebYfc8GYaSdpuvOcEs6dXxPurOYpe9QFwg==", + "dev": true + }, "node_modules/@vscode/test-electron": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/@vscode/test-electron/-/test-electron-2.5.2.tgz", @@ -746,6 +841,15 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ansi-colors": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", + "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/ansi-regex": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", @@ -1152,11 +1256,10 @@ "license": "MIT" }, "node_modules/diff": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz", - "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==", + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.2.tgz", + "integrity": "sha512-vtcDfH3TOjP8UekytvnHH1o1P4FcUdt4eQ1Y+Abap1tk/OB2MWQvcwS2ClCd1zuIhc3JKOx6p3kod8Vfys3E+A==", "dev": true, - "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" } @@ -1876,7 +1979,6 @@ "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", "dev": true, - "license": "MIT", "engines": { "node": ">=8" } @@ -2194,32 +2296,30 @@ } }, "node_modules/mocha": { - "version": "11.7.5", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-11.7.5.tgz", - "integrity": "sha512-mTT6RgopEYABzXWFx+GcJ+ZQ32kp4fMf0xvpZIIfSq9Z8lC/++MtcCnQ9t5FP2veYEP95FIYSvW+U9fV4xrlig==", + "version": "10.8.2", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.8.2.tgz", + "integrity": "sha512-VZlYo/WE8t1tstuRmqgeyBgCbJc/lEdopaa+axcKzTBJ+UIdlAB9XnmvTCAH4pwR4ElNInaedhEBmZD8iCSVEg==", "dev": true, - "license": "MIT", "dependencies": { + "ansi-colors": "^4.1.3", "browser-stdout": "^1.3.1", - "chokidar": "^4.0.1", + "chokidar": "^3.5.3", "debug": "^4.3.5", - "diff": "^7.0.0", + "diff": "^5.2.0", "escape-string-regexp": "^4.0.0", "find-up": "^5.0.0", - "glob": "^10.4.5", + "glob": "^8.1.0", "he": "^1.2.0", - "is-path-inside": "^3.0.3", "js-yaml": "^4.1.0", "log-symbols": "^4.1.0", - "minimatch": "^9.0.5", + "minimatch": "^5.1.6", "ms": "^2.1.3", - "picocolors": "^1.1.1", "serialize-javascript": "^6.0.2", "strip-json-comments": "^3.1.1", "supports-color": "^8.1.1", - "workerpool": "^9.2.0", - "yargs": "^17.7.2", - "yargs-parser": "^21.1.1", + "workerpool": "^6.5.1", + "yargs": "^16.2.0", + "yargs-parser": "^20.2.9", "yargs-unparser": "^2.0.0" }, "bin": { @@ -2227,37 +2327,91 @@ "mocha": "bin/mocha.js" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": ">= 14.0.0" } }, - "node_modules/mocha/node_modules/chokidar": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", - "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "node_modules/mocha/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/mocha/node_modules/cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", "dev": true, - "license": "MIT", "dependencies": { - "readdirp": "^4.0.1" + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "node_modules/mocha/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/mocha/node_modules/glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" }, "engines": { - "node": ">= 14.16.0" + "node": ">=12" }, "funding": { - "url": "https://paulmillr.com/funding/" + "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/mocha/node_modules/readdirp": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", - "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "node_modules/mocha/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", "dev": true, - "license": "MIT", + "dependencies": { + "brace-expansion": "^2.0.1" + }, "engines": { - "node": ">= 14.18.0" + "node": ">=10" + } + }, + "node_modules/mocha/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" }, - "funding": { - "type": "individual", - "url": "https://paulmillr.com/funding/" + "engines": { + "node": ">=8" + } + }, + "node_modules/mocha/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" } }, "node_modules/mocha/node_modules/supports-color": { @@ -2276,6 +2430,50 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/mocha/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/mocha/node_modules/yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "dev": true, + "dependencies": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/mocha/node_modules/yargs-parser": { + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", + "dev": true, + "engines": { + "node": ">=10" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -2559,8 +2757,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, - "license": "ISC" + "dev": true }, "node_modules/picomatch": { "version": "2.3.1", @@ -3154,11 +3351,10 @@ } }, "node_modules/workerpool": { - "version": "9.3.4", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-9.3.4.tgz", - "integrity": "sha512-TmPRQYYSAnnDiEB0P/Ytip7bFGvqnSU6I2BcuSw7Hx+JSg/DsUi5ebYfc8GYaSdpuvOcEs6dXxPurOYpe9QFwg==", - "dev": true, - "license": "Apache-2.0" + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.5.1.tgz", + "integrity": "sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA==", + "dev": true }, "node_modules/wrap-ansi": { "version": "8.1.0", diff --git a/package.json b/package.json index 0911e6c..d8526f4 100644 --- a/package.json +++ b/package.json @@ -72,16 +72,6 @@ "default": "qwen/qwen3-8b", "description": "Model name to use for completions" }, - "crowd-pilot.minAvgLogprob": { - "type": "number", - "default": -1.0, - "description": "Minimum average log-probability per token for displaying suggestions. Higher values (closer to 0) require more confidence. -1.0 ≈ perplexity 2.7" - }, - "crowd-pilot.maxContextTokens": { - "type": "number", - "default": 120000, - "description": "Context length (in tokens). Older messages are truncated to fit. Set below your model's limit to leave room for the response." - }, "crowd-pilot.enablePreferenceLogging": { "type": "boolean", "default": true, @@ -133,22 +123,21 @@ "pretest": "npm run compile && npm run lint", "lint": "eslint src", "test": "vscode-test", + "test:unit": "npm run compile && mocha --ui tdd out/test/parsing.test.js", "clean": "rm -rf out *.tgz", "clean:all": "rm -rf out *.tgz node_modules package-lock.json", "rebuild-serializer": "cd crowd-pilot-serializer/crates/napi && npm install && rm -f index.d.ts index.js && npm run build && npm pack && mv *.tgz ../../../ && cd ../../.. && rm -rf node_modules/@crowd-pilot && npm install" }, - "dependencies": { - "@crowd-pilot/serializer": "file:./crowd-pilot-serializer-0.1.0.tgz" - }, "devDependencies": { - "@types/vscode": "^1.99.3", "@types/mocha": "^10.0.10", "@types/node": "22.x", + "@types/vscode": "^1.99.3", "@typescript-eslint/eslint-plugin": "^8.45.0", "@typescript-eslint/parser": "^8.45.0", - "eslint": "^9.36.0", - "typescript": "^5.9.3", "@vscode/test-cli": "^0.0.11", - "@vscode/test-electron": "^2.5.2" + "@vscode/test-electron": "^2.5.2", + "eslint": "^9.36.0", + "mocha": "^10.8.2", + "typescript": "^5.9.3" } } diff --git a/src/extension.ts b/src/extension.ts index 4a83c23..0831faa 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -3,8 +3,15 @@ import * as http from 'http'; import * as fs from 'fs'; import * as path from 'path'; import { Buffer } from 'buffer'; -import { ConversationStateManager, estimateTokens, getDefaultSystemPrompt } from '@crowd-pilot/serializer'; import { PreviewManager, Action } from './preview'; +import { logToOutput } from './utils/utilities'; +import { + LineRange, + MinimalChangeRange, + extractLastCodeBlock, + computeChangedLineRange, + computeMinimalChangeRange, +} from './utils/parsing'; // -------------------- Preference Data Collection -------------------- @@ -40,6 +47,37 @@ function getPreferenceLogPath(): string { throw new Error("No preference log path found."); } +function getPreferenceLogDir(): string { + return path.dirname(getPreferenceLogPath()); +} + +function createModelLogId(): string { + return `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; +} + +async function writeModelLog(id: string, contents: string, append: boolean): Promise { + const baseDir = getPreferenceLogDir(); + const dir = path.join(baseDir, 'model-logs'); + await fs.promises.mkdir(dir, { recursive: true }); + const filePath = path.join(dir, `${id}.txt`); + if (append) { + await fs.promises.appendFile(filePath, contents, 'utf8'); + } else { + await fs.promises.writeFile(filePath, contents, 'utf8'); + } +} + +async function logModelPrompt(id: string, prompt: string): Promise { + logToOutput(`[crowd-pilot] Model prompt (${id}):\n${prompt}`); + const contents = `=== Prompt ===\n${prompt}\n\n`; + await writeModelLog(id, contents, false); +} + +async function logModelResponse(id: string, raw: string): Promise { + const contents = `=== Response ===\n${raw}\n`; + await writeModelLog(id, contents, true); +} + /** * Log a preference sample to the JSONL file. * Each line is a complete JSON object for easy streaming/parsing. @@ -118,8 +156,6 @@ function markPendingAsIgnored(): void { recordPreferenceOutcome('ignored'); } } - - // Configuration helper function getConfig() { const config = vscode.workspace.getConfiguration('crowd-pilot'); @@ -128,67 +164,41 @@ function getConfig() { port: config.get('port', 30000), basePath: config.get('basePath', '/v1/chat/completions'), modelName: config.get('modelName', 'qwen/qwen3-8b'), - minAvgLogprob: config.get('minAvgLogprob', -1.0), - maxContextTokens: config.get('maxContextTokens', 120000), preferenceLogPath: config.get('preferenceLogPath', ''), enablePreferenceLogging: config.get('enablePreferenceLogging', true), viewportRadius: config.get('viewportRadius', 10), }; } -// -------------------- Context Window Management -------------------- - -/** - * Truncate conversation messages to fit within the context window. - * Assumes system prompt is the first message. - * Uses drop-half strategy: when over budget, drops the first half of conversation - * messages to maximize KV cache hits. - */ -function truncateToContextLimit( - messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>, - maxTokens: number -): Array<{ role: 'system' | 'user' | 'assistant'; content: string }> { - if (messages.length === 0) { return messages; } - - const systemTokens = estimateTokens(messages[0].content); - const availableTokens = maxTokens - systemTokens; - - const conversationMessages = messages.slice(1); - const totalConversationTokens = conversationMessages.reduce( - (sum, m) => sum + estimateTokens(m.content), 0 - ); - - if (totalConversationTokens <= availableTokens) { - return messages; - } - - // Drop first half of conversation messages to maximize KV cache hits - const halfIndex = Math.ceil(conversationMessages.length / 2); - const keptMessages = conversationMessages.slice(halfIndex); - const keptTokens = keptMessages.reduce((sum, m) => sum + estimateTokens(m.content), 0); - - console.log(`[crowd-pilot] Dropped first ${halfIndex} messages (${systemTokens + totalConversationTokens} -> ${systemTokens + keptTokens} tokens)`); - return [messages[0], ...keptMessages]; -} - +type EditHistoryEvent = { oldPath: string; path: string; diff: string }; +type LastEditEvent = { + oldText: string; + newText: string; + editRange: LineRange; + lastEditTimeMs: number; +}; + +const EDIT_HISTORY_LIMIT = 6; +const CHANGE_GROUPING_LINE_SPAN = 8; +const LAST_CHANGE_GROUPING_TIME_MS = 1000; +const DIFF_CONTEXT_LINES = 3; +const BYTES_PER_TOKEN_GUESS = 3; +const MAX_EDITABLE_TOKENS = 180; +const MAX_CONTEXT_TOKENS = 350; + +const fileTextByUri = new Map(); +const editHistoryByUri = new Map(); +const lastEditByUri = new Map(); -// Global conversation state manager instance -let conversationManager: ConversationStateManager; - -// Track activated files (files whose content we've captured) -// TODO (f.srambical): This logic remains on the extension-side -// for backwards-compatibility (with the crowd-code dataset). -// Eventually, we should move the file tracking logic to -// p-doom/crowd-pilot-serializer. -const activatedFiles = new Set(); - -/** - * Clear all conversation context - resets the conversation manager and activated files. - * Call this to start fresh without accumulated history. - */ function clearContext(): void { - conversationManager.reset(); - activatedFiles.clear(); + fileTextByUri.clear(); + editHistoryByUri.clear(); + lastEditByUri.clear(); + currentAction = undefined; + lastPredictionContext = null; + hidePreviewUI(true); + currentAbortController?.abort(); + currentAbortController = undefined; console.log('[crowd-pilot] Context cleared'); } @@ -207,16 +217,99 @@ function updateStatusBarItem(): void { statusBarItem.backgroundColor = new vscode.ThemeColor('statusBarItem.warningBackground'); } } +const TEACHER_PROMPT_TEMPLATE = `# Instructions -export function activate(context: vscode.ExtensionContext) { +You are an edit prediction assistant in a code editor. Your task is to predict the next edit to a given region of code surrounding the user's cursor. - console.log('[crowd-pilot] Extension activated'); +1. Analyze the edit history to understand what the programmer is trying to achieve +2. Identify any incomplete refactoring or changes that need to be finished +3. Make the remaining edits that a human programmer would logically make next (by rewriting the code around their cursor) - const cfg = getConfig(); - conversationManager = new ConversationStateManager({ - viewportRadius: cfg.viewportRadius, - }); +## Focus on + +- Completing any partially-applied changes made +- Ensuring consistency with the programming style and patterns already established +- Making edits that maintain or improve code quality + +## Rules + +- Do not just mechanically apply patterns - reason about what changes make sense given the context and the programmer's apparent goals. +- Do not just fix syntax errors - look for the broader refactoring pattern and apply it systematically throughout the code. +- Keep existing formatting unless it's absolutely necessary +- Don't write a lot of code if you're not sure what to do + +# Input Format + +You will be provided with: +1. The user's *edit history*, in chronological order. Use this to infer the user's trajectory and predict the next most logical edit. +2. A set of *related excerpts* from the user's codebase. Some of these may be needed for correctly predicting the next edit. + - \`…\` may appear within a related file to indicate that some code has been skipped. +3. An excerpt from the user's *current file*. + - Within the user's current file, there is an *editable region* delimited by the \`<|editable_region_start|>\` and \`<|editable_region_end|>\` tags. You can only predict edits in this region. + - The \`<|user_cursor|>\` tag marks the user's current cursor position, as it stands after the last edit in the history. + +# Output Format +- Briefly explain the user's current intent based on the edit history and their current cursor location. +- Output the entire editable region, applying the edits that you predict the user will make next. +- If you're unsure some portion of the next edit, you may still predict the surrounding code (such as a function definition, \`for\` loop, etc) and place the \`<|user_cursor|>\` within it for the user to fill in. +- Wrap the edited code in a codeblock with exactly five backticks. + +## Example + +### Input + +\`\`\`\`\` +struct Product { + name: String, + price: u32, +} + +fn calculate_total(products: &[Product]) -> u32 { +<|editable_region_start|> + let mut total = 0; + for product in products { + total += <|user_cursor|>; + } + total +<|editable_region_end|> +} +\`\`\`\`\` + +### Output + +The user is computing a sum based on a list of products. The only numeric field on \`Product\` is \`price\`, so they must intend to sum the prices. + +\`\`\`\`\` + let mut total = 0; + for product in products { + total += product.price; + } + total +\`\`\`\`\` + +# 1. User Edits History + +\`\`\`\`\` +{{edit_history}} +\`\`\`\`\` + +# 2. Related excerpts + +{{context}} + +# 3. Current File + +{{cursor_excerpt}} +`; + +const EDITABLE_REGION_START_LINE = "<|editable_region_start|>"; +const EDITABLE_REGION_END_LINE = "<|editable_region_end|>"; +const USER_CURSOR_MARKER = "<|user_cursor|>"; + +export function activate(context: vscode.ExtensionContext) { + + console.log('[crowd-pilot] Extension activated'); previewManager = new PreviewManager(); previewManager.register(context); @@ -254,8 +347,8 @@ export function activate(context: vscode.ExtensionContext) { hidePreviewUI(true); } vscode.window.showInformationMessage( - suggestionsEnabled - ? '[crowd-pilot]: Tab suggestions enabled' + suggestionsEnabled + ? '[crowd-pilot]: Tab suggestions enabled' : '[crowd-pilot]: Tab suggestions disabled' ); }); @@ -342,66 +435,19 @@ export function activate(context: vscode.ExtensionContext) { if (e.textEditor === vscode.window.activeTextEditor) { suppressAutoPreview = false; schedulePredictionRefresh(true, false); - - const editor = e.textEditor; - const selection = e.selections[0]; - if (selection) { - const filePath = editor.document.uri.fsPath; - const offset = editor.document.offsetAt(selection.start); - conversationManager.handleSelectionEvent(filePath, offset); - } } }); - const onActiveChange = vscode.window.onDidChangeActiveTextEditor((editor) => { + const onActiveChange = vscode.window.onDidChangeActiveTextEditor(() => { suppressAutoPreview = false; + seedDocumentState(); schedulePredictionRefresh(true, false); - - if (editor) { - const filePath = editor.document.uri.fsPath; - const currentFileUri = editor.document.uri.toString(); - let tabEventText: string | null = null; - - if (!activatedFiles.has(currentFileUri)) { - tabEventText = editor.document.getText(); - activatedFiles.add(currentFileUri); - } - - conversationManager.handleTabEvent(filePath, tabEventText); - } }); - const onDocChange = vscode.workspace.onDidChangeTextDocument((e) => { if (vscode.window.activeTextEditor?.document === e.document) { suppressAutoPreview = false; + recordDocumentChange(e.document); schedulePredictionRefresh(true, false); - - const filePath = e.document.uri.fsPath; - for (const change of e.contentChanges) { - const offset = change.rangeOffset; - const length = change.rangeLength; - const newText = change.text; - conversationManager.handleContentEvent(filePath, offset, length, newText); - } - } - }); - - // Terminal focus event - const onTerminalChange = vscode.window.onDidChangeActiveTerminal((terminal) => { - if (terminal) { - conversationManager.handleTerminalFocusEvent(); - } - }); - - // Terminal command execution event - const onTerminalCommand = vscode.window.onDidStartTerminalShellExecution(async (event) => { - const commandLine = event.execution.commandLine.value; - conversationManager.handleTerminalCommandEvent(commandLine); - - // Capture terminal output - const stream = event.execution.read(); - for await (const data of stream) { - conversationManager.handleTerminalOutputEvent(data); } }); @@ -415,20 +461,11 @@ export function activate(context: vscode.ExtensionContext) { showPendingAction, onSelChange, onActiveChange, - onDocChange, - onTerminalChange, - onTerminalCommand + onDocChange ); - // Initialize: capture current active editor if any - const initialEditor = vscode.window.activeTextEditor; - if (initialEditor) { - const filePath = initialEditor.document.uri.fsPath; - const currentFileUri = initialEditor.document.uri.toString(); - const tabEventText = initialEditor.document.getText(); - activatedFiles.add(currentFileUri); - conversationManager.handleTabEvent(filePath, tabEventText); - } + seedDocumentState(); + schedulePredictionRefresh(true, false); } export function deactivate() { @@ -437,22 +474,17 @@ export function deactivate() { // -------------------- Execution -------------------- let currentAction: Action | undefined; - -function getActiveOrCreateTerminal(): vscode.Terminal { - if (vscode.window.activeTerminal) { - return vscode.window.activeTerminal; - } - return vscode.window.createTerminal('crowd-pilot'); -} +type PredictionContext = { + docUri: string; + docVersion: number; + editableRange: LineRange; + cursorLine: number; +}; +let lastPredictionContext: PredictionContext | null = null; async function executeAction(action: Action): Promise { const editor = vscode.window.activeTextEditor; if (!editor) { return; } - const doc = editor.document; - if (action.kind === 'showTextDocument') { - await vscode.window.showTextDocument(doc); - return; - } if (action.kind === 'setSelections') { editor.selections = action.selections.map(s => new vscode.Selection( new vscode.Position(s.start[0], s.start[1]), @@ -461,18 +493,6 @@ async function executeAction(action: Action): Promise { editor.revealRange(editor.selections[0], vscode.TextEditorRevealType.InCenterIfOutsideViewport); return; } - if (action.kind === 'editInsert') { - await editor.edit((e: vscode.TextEditorEdit) => e.insert(new vscode.Position(action.position[0], action.position[1]), action.text)); - return; - } - if (action.kind === 'editDelete') { - const range = new vscode.Range( - new vscode.Position(action.range.start[0], action.range.start[1]), - new vscode.Position(action.range.end[0], action.range.end[1]) - ); - await editor.edit((e: vscode.TextEditorEdit) => e.delete(range)); - return; - } if (action.kind === 'editReplace') { const range = new vscode.Range( new vscode.Position(action.range.start[0], action.range.start[1]), @@ -481,29 +501,6 @@ async function executeAction(action: Action): Promise { await editor.edit((e: vscode.TextEditorEdit) => e.replace(range, action.text)); return; } - if (action.kind === 'terminalShow') { - const term = getActiveOrCreateTerminal(); - term.show(); - return; - } - if (action.kind === 'terminalSendText') { - const term = getActiveOrCreateTerminal(); - term.show(); - term.sendText(action.text, false); - return; - } - if (action.kind === 'openFile') { - const uri = vscode.Uri.file(action.filePath); - const openedEditor = await vscode.window.showTextDocument(uri); - if (action.selections) { - openedEditor.selections = action.selections.map(s => new vscode.Selection( - new vscode.Position(s.start[0], s.start[1]), - new vscode.Position(s.end[0], s.end[1]) - )); - openedEditor.revealRange(openedEditor.selections[0], vscode.TextEditorRevealType.InCenterIfOutsideViewport); - } - return; - } } // -------------------- UI State & Helpers -------------------- @@ -523,7 +520,6 @@ let nextQueuedPredictionId = 0; let pendingPredictions: PendingPrediction[] = []; const cancelledPredictionIds = new Set(); let lastPredictionTimestamp: number | undefined; - /** * Show preview UI for the given action using the PreviewManager. */ @@ -546,6 +542,46 @@ function hidePreviewUI(suppress?: boolean): void { } } +function canRequestPrediction(editor: vscode.TextEditor, userRequested: boolean): boolean { + if (!userRequested && suppressAutoPreview) { + return false; + } + if (!userRequested) { + if (!vscode.window.state.focused) { + return false; + } + if (editor.document.getText().length === 0) { + return false; + } + if (editor.selections.some(selection => !selection.isEmpty)) { + return false; + } + } + return true; +} + +function shouldReuseCurrentPrediction(editor: vscode.TextEditor): boolean { + if (!currentAction || !previewManager.isVisible()) { + return false; + } + if (!lastPredictionContext) { + return false; + } + const doc = editor.document; + if (doc.uri.toString() !== lastPredictionContext.docUri) { + return false; + } + if (doc.version !== lastPredictionContext.docVersion) { + return false; + } + if (editor.selections.some(selection => !selection.isEmpty)) { + return false; + } + const cursorLine = editor.selection.active.line; + return cursorLine >= lastPredictionContext.editableRange.start + && cursorLine <= lastPredictionContext.editableRange.end; +} + /** * Schedule a model preview refresh, coalescing rapid editor events and * throttling how often we actually talk to the model. @@ -554,9 +590,6 @@ function schedulePredictionRefresh(debounce: boolean, userRequested: boolean): v if (!suggestionsEnabled) { return; } - if (!userRequested && suppressAutoPreview) { - return; - } const editor = vscode.window.activeTextEditor; if (!editor) { @@ -564,15 +597,9 @@ function schedulePredictionRefresh(debounce: boolean, userRequested: boolean): v return; } - if (!userRequested) { - if (!vscode.window.state.focused) { - hidePreviewUI(); - return; - } - if (editor.document.getText().length === 0) { - hidePreviewUI(); - return; - } + if (!canRequestPrediction(editor, userRequested)) { + hidePreviewUI(); + return; } const now = Date.now(); @@ -612,10 +639,153 @@ function schedulePredictionRefresh(debounce: boolean, userRequested: boolean): v } } +function seedDocumentState(): void { + for (const editor of vscode.window.visibleTextEditors) { + const doc = editor.document; + const uri = doc.uri.toString(); + if (!fileTextByUri.has(uri)) { + fileTextByUri.set(uri, doc.getText()); + } + } +} + +function recordDocumentChange(doc: vscode.TextDocument): void { + const uri = doc.uri.toString(); + const newText = doc.getText(); + const oldText = fileTextByUri.get(uri); + if (oldText === undefined) { + fileTextByUri.set(uri, newText); + return; + } + if (oldText === newText) { + return; + } + const changedRange = computeChangedLineRange(oldText, newText); + if (!changedRange) { + fileTextByUri.set(uri, newText); + return; + } + + const nowMs = Date.now(); + const lastEvent = lastEditByUri.get(uri); + if (lastEvent && nowMs - lastEvent.lastEditTimeMs >= LAST_CHANGE_GROUPING_TIME_MS) { + finalizeLastEdit(uri, doc.fileName, lastEvent); + } + + const updatedLastEvent = lastEditByUri.get(uri); + if ( + updatedLastEvent && + rangesAreNearby(updatedLastEvent.editRange, changedRange, CHANGE_GROUPING_LINE_SPAN) + ) { + updatedLastEvent.newText = newText; + updatedLastEvent.editRange = { + start: Math.min(updatedLastEvent.editRange.start, changedRange.start), + end: Math.max(updatedLastEvent.editRange.end, changedRange.end), + }; + updatedLastEvent.lastEditTimeMs = nowMs; + lastEditByUri.set(uri, updatedLastEvent); + } else { + if (updatedLastEvent) { + finalizeLastEdit(uri, doc.fileName, updatedLastEvent); + } + lastEditByUri.set(uri, { + oldText, + newText, + editRange: changedRange, + lastEditTimeMs: nowMs, + }); + } + + fileTextByUri.set(uri, newText); +} + +function finalizeLastEdit(uri: string, filePath: string, lastEvent: LastEditEvent): void { + const diff = buildUnifiedDiff(lastEvent.oldText, lastEvent.newText, DIFF_CONTEXT_LINES); + if (!diff.trim()) { + lastEditByUri.delete(uri); + return; + } + const events = editHistoryByUri.get(uri) ?? []; + events.push({ oldPath: filePath, path: filePath, diff }); + while (events.length > EDIT_HISTORY_LIMIT) { + events.shift(); + } + editHistoryByUri.set(uri, events); + lastEditByUri.delete(uri); +} + +function rangesAreNearby(a: LineRange, b: LineRange, span: number): boolean { + if (a.start <= b.end && b.start <= a.end) { + return true; + } + if (a.start > b.end) { + return (a.start - b.end) <= span; + } + return (b.start - a.end) <= span; +} + +function buildUnifiedDiff(oldText: string, newText: string, contextLines: number): string { + const oldLines = oldText.split(/\r?\n/); + const newLines = newText.split(/\r?\n/); + let prefix = 0; + while ( + prefix < oldLines.length && + prefix < newLines.length && + oldLines[prefix] === newLines[prefix] + ) { + prefix += 1; + } + let suffix = 0; + while ( + suffix < oldLines.length - prefix && + suffix < newLines.length - prefix && + oldLines[oldLines.length - 1 - suffix] === newLines[newLines.length - 1 - suffix] + ) { + suffix += 1; + } + if (prefix === oldLines.length && prefix === newLines.length) { + return ""; + } + const oldStart = Math.max(0, prefix - contextLines); + const oldEnd = Math.min(oldLines.length, oldLines.length - suffix + contextLines); + const newStart = Math.max(0, prefix - contextLines); + const newEnd = Math.min(newLines.length, newLines.length - suffix + contextLines); + const oldLen = Math.max(0, oldEnd - oldStart); + const newLen = Math.max(0, newEnd - newStart); + + const diffLines: string[] = []; + diffLines.push(`@@ -${oldStart + 1},${oldLen} +${newStart + 1},${newLen} @@`); + + const prefixContext = oldLines.slice(oldStart, prefix); + for (const line of prefixContext) { + diffLines.push(` ${line}`); + } + const oldChanged = oldLines.slice(prefix, oldLines.length - suffix); + for (const line of oldChanged) { + diffLines.push(`-${line}`); + } + const newChanged = newLines.slice(prefix, newLines.length - suffix); + for (const line of newChanged) { + diffLines.push(`+${line}`); + } + const suffixContext = oldLines.slice(oldLines.length - suffix, oldEnd); + for (const line of suffixContext) { + diffLines.push(` ${line}`); + } + + return diffLines.join("\n"); +} async function autoShowNextAction(): Promise { if (suppressAutoPreview) { return; } const editor = vscode.window.activeTextEditor; if (!editor) { return; } + if (!canRequestPrediction(editor, false)) { + hidePreviewUI(); + return; + } + if (shouldReuseCurrentPrediction(editor)) { + return; + } try { currentAbortController?.abort(); const controller = new AbortController(); @@ -623,7 +793,14 @@ async function autoShowNextAction(): Promise { const requestId = ++latestRequestId; const next = await requestModelActions(editor, controller.signal); if (requestId !== latestRequestId) { return; } - if (next) { showPreviewUI(next); } else { hidePreviewUI(); } + if (next) { + if (currentAction && previewManager.isVisible() && !shouldReplaceAction(currentAction, next)) { + return; + } + showPreviewUI(next); + } else { + hidePreviewUI(); + } } catch (err) { const e = err as any; const isAbort = e?.name === 'AbortError' || /aborted/i.test(String(e?.message ?? '')); @@ -632,14 +809,43 @@ async function autoShowNextAction(): Promise { } } +function shouldReplaceAction(currentAction: Action, nextAction: Action): boolean { + if (currentAction.kind !== nextAction.kind) { + return true; + } + if (currentAction.kind === 'editReplace' && nextAction.kind === 'editReplace') { + const sameRange = + currentAction.range.start[0] === nextAction.range.start[0] && + currentAction.range.start[1] === nextAction.range.start[1] && + currentAction.range.end[0] === nextAction.range.end[0] && + currentAction.range.end[1] === nextAction.range.end[1]; + if (!sameRange) { + return true; + } + return !nextAction.text.startsWith(currentAction.text); + } + if (currentAction.kind === 'setSelections' && nextAction.kind === 'setSelections') { + const current = currentAction.selections[0]; + const next = nextAction.selections[0]; + if (!current || !next) { + return true; + } + return !( + current.start[0] === next.start[0] && + current.start[1] === next.start[1] && + current.end[0] === next.end[0] && + current.end[1] === next.end[1] + ); + } + return true; +} + // -------------------- SGLang Client (simple test) -------------------- async function callSGLangChat(): Promise { const cfg = getConfig(); const headers: any = { 'Content-Type': 'application/json' }; - - const requestBody: any = { model: cfg.modelName, messages: [ @@ -653,6 +859,12 @@ async function callSGLangChat(): Promise { requestBody.chat_template_kwargs = { enable_thinking: false }; + const requestId = createModelLogId(); + try { + await logModelPrompt(requestId, JSON.stringify(requestBody, null, 2)); + } catch (err) { + console.error('[crowd-pilot] Failed to log model prompt:', err); + } const postData = JSON.stringify(requestBody); headers['Content-Length'] = Buffer.byteLength(postData); @@ -672,14 +884,21 @@ async function callSGLangChat(): Promise { res.on('data', (chunk: Buffer) => { data += chunk.toString(); }); - res.on('end', () => { + res.on('end', () => { + void (async () => { + try { + await logModelResponse(requestId, data); + } catch (err) { + console.error('[crowd-pilot] Failed to log model response:', err); + } try { resolve(JSON.parse(data)); } catch (err) { reject(new Error(`Failed to parse response: ${err instanceof Error ? err.message : String(err)}`)); } - }); + })(); }); + }); req.on('error', (err: Error) => { reject(err); @@ -697,40 +916,34 @@ async function callSGLangChat(): Promise { } // -------------------- Model-planned Actions -------------------- -async function requestModelActions(editor: vscode.TextEditor, signal?: AbortSignal): Promise { +async function requestModelActions(editor: vscode.TextEditor, signal?: AbortSignal): Promise { const cfg = getConfig(); const headers: any = { 'Content-Type': 'application/json' }; - const doc = editor.document; - - const systemPrompt = getDefaultSystemPrompt(cfg.viewportRadius); - - const accumulatedMessages = conversationManager.finalizeForModel(); - - let conversationMessages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }> = [ - { role: 'system', content: systemPrompt }, + const promptContext = buildTeacherPrompt(editor); + const conversationMessages = [ + { role: 'system', content: promptContext.prompt } ]; - - for (const msg of accumulatedMessages) { - const role = msg.from === 'User' ? 'user' : 'assistant'; - conversationMessages.push({ role, content: msg.value }); + const requestId = createModelLogId(); + try { + await logModelPrompt(requestId, promptContext.prompt); + } catch (err) { + console.error('[crowd-pilot] Failed to log model prompt:', err); } - conversationMessages = truncateToContextLimit(conversationMessages, cfg.maxContextTokens); - const requestBody: any = { model: cfg.modelName, - messages: conversationMessages - }; - requestBody.temperature = 0.7; - requestBody.top_p = 0.8; - requestBody.top_k = 20; - requestBody.min_p = 0; - requestBody.logprobs = true; - requestBody.chat_template_kwargs = { - enable_thinking: false + messages: conversationMessages, + temperature: 0.7, + top_p: 0.8, + top_k: 20, + min_p: 0, + logprobs: true, + chat_template_kwargs: { + enable_thinking: false + } }; const postData = JSON.stringify(requestBody); @@ -752,11 +965,18 @@ async function requestModelActions(editor: vscode.TextEditor, signal?: AbortSign let data = ''; res.on('data', (chunk: Buffer) => { data += chunk.toString(); }); res.on('end', () => { - try { - resolve(JSON.parse(data)); - } catch (err) { - reject(new Error(`Failed to parse response: ${err instanceof Error ? err.message : String(err)}`)); - } + void (async () => { + try { + await logModelResponse(requestId, data); + } catch (err) { + console.error('[crowd-pilot] Failed to log model response:', err); + } + try { + resolve(JSON.parse(data)); + } catch (err) { + reject(new Error(`Failed to parse response: ${err instanceof Error ? err.message : String(err)}`)); + } + })(); }); }); req.on('error', (err: Error) => reject(err)); @@ -765,19 +985,59 @@ async function requestModelActions(editor: vscode.TextEditor, signal?: AbortSign }); const avgLogprob = calculateAverageLogprob(json); - if (avgLogprob < cfg.minAvgLogprob) { - return undefined as any; // Low confidence, silently skip suggestion - } const content = extractChatContent(json); if (typeof content !== 'string' || content.trim().length === 0) { throw new Error('Empty model content'); } - const action = parseAction(content, doc); - + + // If document changed while waiting, try to rebase the model's prediction. + // Only rebase for minor changes (same line count). If the user added/removed lines, + // the content has shifted and rebasing would produce incorrect results. + let effectiveContext = promptContext; + if (editor.document.version !== promptContext.doc.version) { + const currentDoc = editor.document; + const currentLines = currentDoc.getText().split(/\r?\n/); + const originalLines = promptContext.editableText.split(/\r?\n/); + + // Check if the editable range is still valid + const editableEnd = Math.min(promptContext.editableRange.end, currentLines.length - 1); + if (editableEnd < promptContext.editableRange.start) { + console.log('[crowd-pilot] Discarding response: editable range no longer valid'); + return undefined; + } + + // Get current text from the same line range + const currentEditableText = currentLines.slice(promptContext.editableRange.start, editableEnd + 1).join('\n'); + const currentEditableLines = currentEditableText.split(/\r?\n/); + + // Only rebase if the line count is the same (user typed on existing lines, didn't add/remove lines) + // If lines were added/removed, the content has shifted and rebasing would be incorrect + if (currentEditableLines.length !== originalLines.length) { + console.log(`[crowd-pilot] Discarding response: line count changed (${originalLines.length} → ${currentEditableLines.length})`); + return undefined; + } + + effectiveContext = { + ...promptContext, + editableText: currentEditableText, + editableRange: { start: promptContext.editableRange.start, end: editableEnd }, + doc: currentDoc, + cursor: editor.selection.active, + }; + console.log(`[crowd-pilot] Rebasing response onto current doc (version ${promptContext.doc.version} → ${currentDoc.version})`); + } + + const action = parseTeacherResponse(content, effectiveContext); if (!action) { - throw new Error('No valid action parsed from model output'); + return undefined; } + lastPredictionContext = { + docUri: effectiveContext.doc.uri.toString(), + docVersion: effectiveContext.doc.version, + editableRange: effectiveContext.editableRange, + cursorLine: effectiveContext.cursor.line, + }; markPendingAsIgnored(); @@ -815,248 +1075,227 @@ function extractChatContent(json: any): string | undefined { * Returns -Infinity if logprobs are not available. */ function calculateAverageLogprob(json: any): number { - const logprobs = json.choices[0]?.logprobs; - const sum = logprobs.content.reduce((s: number, t: any) => s + t.logprob, 0); - return sum / logprobs.content.length; -} - -function parseAction(raw: string, doc?: vscode.TextDocument): Action | undefined { - const command = extractBashCommand(raw); - if (!command) { - return undefined; + const logprobs = json?.choices?.[0]?.logprobs; + const tokens = logprobs?.content; + if (!Array.isArray(tokens) || tokens.length === 0) { + return Number.NEGATIVE_INFINITY; } - const normalized = command.replace(/[\s\S]*?<\/think>/gi, '').trim(); - if (!normalized) { - return undefined; - } - if (doc) { - const editAction = parseEditFromSedCommand(normalized, doc); - if (editAction) { - return editAction; - } - const viewportAction = parseViewportFromCatCommand(normalized, doc); - if (viewportAction) { - return viewportAction; + let sum = 0; + let count = 0; + for (const token of tokens) { + if (typeof token.logprob === 'number') { + sum += token.logprob; + count += 1; } } - // Sanitize terminal commands for cleaner display - const sanitizedCommand = sanitizeCommandForDisplay(normalized); - return { kind: 'terminalSendText', text: sanitizedCommand }; + if (count === 0) { + return Number.NEGATIVE_INFINITY; + } + return sum / count; } -/** - * Sanitize a command string for display, removing shell artifacts and escaping. - */ -function sanitizeCommandForDisplay(cmd: string): string { - return cmd - .replace(/^-[A-Z]\s*/gm, '') // Remove stray flag artifacts at line starts - .replace(/'\"'\"'/g, "'") // Fix shell quote escaping - .replace(/\\\\/g, '\\') // Normalize double backslashes - .replace(/\\n/g, '\n') // Convert escaped newlines - .replace(/\\t/g, '\t') // Convert escaped tabs - .trim(); +type PromptContext = { + prompt: string; + editableRange: LineRange; + editableText: string; + doc: vscode.TextDocument; + cursor: vscode.Position; +}; + +function buildTeacherPrompt(editor: vscode.TextEditor): PromptContext { + const doc = editor.document; + const lines = doc.getText().split(/\r?\n/); + const cursor = editor.selection.active; + const { editable, context } = computeEditableAndContextRanges(lines, cursor.line); + + const historyEvents = collectEditHistoryForPrompt(doc, doc.uri.toString()); + const editHistoryText = formatEditHistory(historyEvents); + const relatedContextText = formatRelatedContext(doc); + const cursorExcerpt = formatCursorExcerpt(doc, editable, context, cursor); + + const prompt = TEACHER_PROMPT_TEMPLATE + .replace('{{edit_history}}', editHistoryText) + .replace('{{context}}', relatedContextText) + .replace('{{cursor_excerpt}}', cursorExcerpt); + + const editableText = lines.slice(editable.start, editable.end + 1).join('\n'); + return { prompt, editableRange: editable, editableText, doc, cursor }; } -/** - * Parse a sed-based edit command of the form emitted by the NeMo serializer into a VS Code edit action. - * - * Supported patterns (1-based line numbers, mirroring serialization_utils.py): - * sed -i 'START,ENDc\n' -> editReplace - * sed -i 'START,ENDd' -> editDelete - * sed -i 'STARTi\n' -> editInsert (before START) - * sed -i '$a\n' -> editInsert (append at EOF) - * - * If the command does not match these patterns, returns undefined. - */ -function parseEditFromSedCommand(command: string, doc: vscode.TextDocument): Action | undefined { - // Only consider the first command before && / ||, since cat -n etc. are for viewport only. - const main = command.split(/&&|\|\|/)[0]?.trim() ?? ''; - if (!main) { - return undefined; +function collectEditHistoryForPrompt(doc: vscode.TextDocument, uri: string): EditHistoryEvent[] { + const events = [...(editHistoryByUri.get(uri) ?? [])]; + const lastEvent = lastEditByUri.get(uri); + if (lastEvent) { + const diff = buildUnifiedDiff(lastEvent.oldText, lastEvent.newText, DIFF_CONTEXT_LINES); + if (diff.trim()) { + events.push({ oldPath: doc.fileName, path: doc.fileName, diff }); + } } + return events.slice(-EDIT_HISTORY_LIMIT); +} - // Match: sed with optional flags like -E, -n, -r, followed by -i, then script and file - // Handles: sed -i '...' file, sed -E -i '...' file, sed -i -E '...' file, etc. - const sedMatch = main.match(/sed\s+(?:-[A-Za-z]+\s+)*-i\s+(?:-[A-Za-z]+\s+)*'([\s\S]*?)'\s+([^\s&|]+)\s*$/); - if (!sedMatch) { - return undefined; +function formatEditHistory(events: EditHistoryEvent[]): string { + if (events.length === 0) { + return "(No edit history)"; } - const script = sedMatch[1] ?? ''; - const targetFile = sedMatch[2] ?? ''; - const activePath = doc.uri.fsPath; - if (targetFile !== activePath) { - return undefined; + return events + .map((event) => `--- a/${event.oldPath}\n+++ b/${event.path}\n${event.diff}`) + .join("\n"); +} + +function formatRelatedContext(currentDoc: vscode.TextDocument): string { + const relatedEditors = vscode.window.visibleTextEditors.filter( + (editor) => editor.document.uri.toString() !== currentDoc.uri.toString(), + ); + if (relatedEditors.length === 0) { + return "(No context)"; } - // Delete: "START,ENDd" - const deleteMatch = script.match(/^(\d+),(\d+)d$/); - if (deleteMatch) { - const startLine1 = Number(deleteMatch[1]); - const endLine1 = Number(deleteMatch[2]); - if (!Number.isFinite(startLine1) || !Number.isFinite(endLine1)) { - return undefined; + const blocks: string[] = []; + for (const editor of relatedEditors) { + const doc = editor.document; + const path = doc.fileName; + const totalLines = doc.lineCount; + const cursorLine = editor.selection.active.line; + const radius = 10; + const start = Math.max(0, cursorLine - radius); + const end = Math.min(totalLines - 1, cursorLine + radius); + const excerptLines: string[] = []; + + if (start > 0) { + excerptLines.push("…"); } - const startLine0 = Math.max(0, startLine1 - 1); - const endLine0 = Math.max(0, endLine1 - 1); - - let endPosLine = endLine0 + 1; - let endPosChar = 0; - if (endPosLine >= doc.lineCount) { - endPosLine = doc.lineCount - 1; - endPosChar = doc.lineAt(endPosLine).range.end.character; + for (let line = start; line <= end; line += 1) { + excerptLines.push(doc.lineAt(line).text); } - return { - kind: 'editDelete', - range: { - start: [startLine0, 0], - end: [endPosLine, endPosChar], - }, - }; + if (end < totalLines - 1) { + excerptLines.push("…"); + } + + const block = [ + `\`\`\`\`\`${path}`, + excerptLines.join("\n"), + "", + "`````", + ].join("\n"); + blocks.push(block); } - // Replace: "START,ENDc\newline" - const replaceMatch = script.match(/^(\d+),(\d+)c\\\n([\s\S]*)$/); - if (replaceMatch) { - const startLine1 = Number(replaceMatch[1]); - const endLine1 = Number(replaceMatch[2]); - let payload = replaceMatch[3] ?? ''; - if (!Number.isFinite(startLine1) || !Number.isFinite(endLine1)) { - return undefined; - } - payload = payload.replace(/'\"'\"'/g, "'"); - // Convert escape sequences to actual characters - payload = payload.replace(/\\n/g, '\n').replace(/\\t/g, '\t').replace(/\\'/g, "'").replace(/\\\\/g, '\\'); - const startLine0 = Math.max(0, startLine1 - 1); - const endLine0 = Math.max(0, endLine1 - 1); - const startPos: [number, number] = [startLine0, 0]; - - let endPosLine = endLine0 + 1; - let endPosChar = 0; - if (endPosLine >= doc.lineCount) { - endPosLine = doc.lineCount - 1; - endPosChar = doc.lineAt(endPosLine).range.end.character; - } + return blocks.join("\n"); +} - const text = payload.endsWith('\n') ? payload : payload + '\n'; - return { - kind: 'editReplace', - range: { start: startPos, end: [endPosLine, endPosChar] }, - text, - }; +function formatCursorExcerpt( + doc: vscode.TextDocument, + editable: LineRange, + context: LineRange, + cursor: vscode.Position, +): string { + const lines = doc.getText().split(/\r?\n/); + const contextStart = Math.max(0, context.start); + const contextEnd = Math.min(lines.length - 1, context.end); + const excerptLines = lines.slice(contextStart, contextEnd + 1); + + const cursorIndex = cursor.line - contextStart; + if (cursorIndex >= 0 && cursorIndex < excerptLines.length) { + const lineText = excerptLines[cursorIndex]; + const charIndex = Math.min(cursor.character, lineText.length); + excerptLines[cursorIndex] = + lineText.slice(0, charIndex) + USER_CURSOR_MARKER + lineText.slice(charIndex); } - const insertMatch = script.match(/^(\d+)i\\\n([\s\S]*)$/); - if (insertMatch) { - const line1 = Number(insertMatch[1]); - let payload = insertMatch[2] ?? ''; - if (!Number.isFinite(line1)) { - return undefined; + const editableStartIndex = editable.start - contextStart; + const editableEndIndex = editable.end - contextStart; + const startInsertIndex = Math.max(0, Math.min(excerptLines.length, editableStartIndex)); + excerptLines.splice(startInsertIndex, 0, EDITABLE_REGION_START_LINE); + const endInsertIndex = Math.max(0, Math.min(excerptLines.length, editableEndIndex + 2)); + excerptLines.splice(endInsertIndex, 0, EDITABLE_REGION_END_LINE); + + return `\`\`\`\`\`${doc.fileName}\n${excerptLines.join("\n")}\n\`\`\`\`\``; +} + +function computeEditableAndContextRanges( + lines: string[], + cursorLine: number, +): { editable: LineRange; context: LineRange } { + const clampedLine = Math.max(0, Math.min(cursorLine, Math.max(0, lines.length - 1))); + const cursorRange = { start: clampedLine, end: clampedLine }; + const editable = expandLineRange(lines, cursorRange, MAX_EDITABLE_TOKENS); + const context = expandLineRange(lines, editable, MAX_CONTEXT_TOKENS); + return { editable, context }; +} + +function expandLineRange(lines: string[], base: LineRange, tokenLimit: number): LineRange { + let start = Math.max(0, base.start); + let end = Math.min(lines.length - 1, base.end); + let remaining = tokenLimit; + + for (let line = start; line <= end; line += 1) { + remaining -= lineTokenGuess(lines[line]); + } + remaining = Math.max(0, remaining); + + while (remaining > 0) { + let expanded = false; + if (start > 0 && remaining > 0) { + start -= 1; + remaining -= lineTokenGuess(lines[start]); + expanded = true; + } + if (end < lines.length - 1 && remaining > 0) { + end += 1; + remaining -= lineTokenGuess(lines[end]); + expanded = true; + } + if (!expanded) { + break; } - payload = payload.replace(/'\"'\"'/g, "'"); - // Convert escape sequences to actual characters - payload = payload.replace(/\\n/g, '\n').replace(/\\t/g, '\t').replace(/\\'/g, "'").replace(/\\\\/g, '\\'); - const insertLine0 = Math.max(0, line1 - 1); - const position: [number, number] = [insertLine0, 0]; - const text = payload.endsWith('\n') ? payload : payload + '\n'; - return { - kind: 'editInsert', - position, - text, - }; } - const appendMatch = script.match(/^\$a\\\n([\s\S]*)$/); - if (appendMatch) { - let payload = appendMatch[1] ?? ''; - payload = payload.replace(/'\"'\"'/g, "'"); - // Convert escape sequences to actual characters - payload = payload.replace(/\\n/g, '\n').replace(/\\t/g, '\t').replace(/\\'/g, "'").replace(/\\\\/g, '\\'); - const insertLine0 = doc.lineCount; - const position: [number, number] = [insertLine0, 0]; - const needsLeadingNewline = doc.lineCount > 0; - const base = payload.endsWith('\n') ? payload : payload + '\n'; - const text = needsLeadingNewline ? '\n' + base : base; - return { - kind: 'editInsert', - position, - text, - }; - } + return { start, end }; +} - return undefined; +function lineTokenGuess(line: string): number { + const bytes = Buffer.byteLength(line, "utf8"); + return Math.max(1, Math.floor(bytes / BYTES_PER_TOKEN_GUESS)); } -/** - * Parse viewport / selection commands of the form: - * cat -n | sed -n 'START,ENDp' - * - * into a lightweight VS Code selection move (setSelections). This mirrors how - * selection and viewport events are serialized in serialization_utils.py. - */ -function parseViewportFromCatCommand(command: string, doc: vscode.TextDocument): Action | undefined { - const main = command.split(/&&|\|\|/)[0]?.trim() ?? ''; - if (!main) { +function parseTeacherResponse(raw: string, context: PromptContext): Action | undefined { + const codeBlock = extractLastCodeBlock(raw); + if (!codeBlock) { return undefined; } - - // Simple file-open: cat -n - const simpleCatMatch = main.match(/^cat\s+-n\s+([^\s|]+)\s*$/); - if (simpleCatMatch) { - const targetFile = simpleCatMatch[1] ?? ''; - if (targetFile !== doc.uri.fsPath) { - return { kind: 'openFile', filePath: targetFile }; - } - // Ensure the active document is visible; rely on existing editor to handle this. - return { kind: 'showTextDocument' }; + const cleaned = codeBlock; + let newEditableText = cleaned.replaceAll(USER_CURSOR_MARKER, ""); + if (context.editableText.endsWith("\n") && !newEditableText.endsWith("\n")) { + newEditableText += "\n"; } - - // Viewport slice: cat -n | sed -n 'START,ENDp' - const viewportMatch = main.match(/^cat\s+-n\s+([^\s|]+)\s*\|\s*sed\s+-n\s+'(\d+),(\d+)p'\s*$/); - if (!viewportMatch) { + if (context.editableText === newEditableText) { return undefined; } + const changeRange = computeMinimalChangeRange(context.editableText, newEditableText); + if (!changeRange) { + return undefined; + } + const absoluteStart = context.editableRange.start + changeRange.oldStart; + const absoluteEnd = context.editableRange.start + changeRange.oldEnd; + const cursorLine = context.cursor.line; - const targetFile = viewportMatch[1] ?? ''; - const startStr = viewportMatch[2] ?? ''; - const endStr = viewportMatch[3] ?? ''; - - const startLine1 = Number(startStr); - const endLine1 = Number(endStr); - - // Place the cursor in the middle of the viewport (1-based to 0-based). - const center1 = Math.floor((startLine1 + endLine1) / 2); - const center0 = Math.max(0, center1 - 1); - - if (targetFile !== doc.uri.fsPath) { + if (cursorLine < absoluteStart - 2 || cursorLine > absoluteEnd + 2) { + const targetLine = Math.max(0, Math.min(absoluteStart, context.doc.lineCount - 1)); return { - kind: 'openFile', - filePath: targetFile, - selections: [{ start: [center0, 0], end: [center0, 0] }] + kind: "setSelections", + selections: [{ start: [targetLine, 0], end: [targetLine, 0] }], }; } - const lastLine = Math.max(0, doc.lineCount - 1); - const line = Math.min(center0, lastLine); + // Only replace the lines that actually changed, not the entire editable region + const replaceStartLine = absoluteStart; + const replaceEndLine = Math.min(absoluteEnd, context.doc.lineCount - 1); + const endChar = context.doc.lineAt(replaceEndLine).range.end.character; + return { - kind: 'setSelections', - selections: [ - { - start: [line, 0], - end: [line, 0], - }, - ], + kind: "editReplace", + range: { start: [replaceStartLine, 0], end: [replaceEndLine, endChar] }, + text: changeRange.newText, }; } - -function extractBashCommand(raw: string): string | undefined { - if (!raw) { - return undefined; - } - const trimmed = raw.trim(); - const fenceMatch = trimmed.match(/```(?:bash)?\s*([\s\S]*?)```/i); - if (fenceMatch && fenceMatch[1]) { - return fenceMatch[1]; - } - // Fallback: treat entire response as the command - return trimmed.length > 0 ? trimmed : undefined; -} \ No newline at end of file diff --git a/src/preview/index.ts b/src/preview/index.ts index 37e6362..64a1866 100644 --- a/src/preview/index.ts +++ b/src/preview/index.ts @@ -5,6 +5,7 @@ import { CrowdPilotInlineProvider } from './inlineProvider'; import { MetaActionHoverProvider } from './hoverProvider'; import { showPendingActionQuickPick, QuickPickResult } from './quickPick'; import { computeDeletionRanges, hasInsertions, analyzeCoherentReplacement, analyzePureInsertion } from '../utils/diff'; +import { computeMinimalChangeRange } from '../utils/parsing'; // Re-export types export { Action, toVscodeRange, toVscodePosition, truncate } from './types'; @@ -183,11 +184,10 @@ export class PreviewManager { } /** - * Show preview for text replacement using decorations. - * Case 1: Pure insertion (no deletions) → show only inserted text inline in green - * Case 2: Has deletions → decorations (red deletion + green addition) - * - If coherent (single substring replacement): show green inline after red - * - If not coherent (scattered changes): show green block on next line + * Show preview for text replacement. + * Uses VS Code's inline completion API (ghost text) for proper multi-line display. + * Case 1: Pure insertion (no deletions) → show as ghost text + * Case 2: Has deletions → red strikethrough decoration + ghost text for new content */ private showReplacePreview(action: { kind: 'editReplace'; range: { start: [number, number]; end: [number, number] }; text: string }, editor?: vscode.TextEditor): void { if (!editor) { @@ -200,10 +200,13 @@ export class PreviewManager { // Case 1: Check for pure insertion first (no deletions) const pureInsertion = analyzePureInsertion(editor.document, range, action.text); if (pureInsertion.isPureInsertion && pureInsertion.insertionPosition && pureInsertion.insertionText) { - // Pure insertion: show only the new text inline in green (no red) - this.showInlineInsertion(editor, pureInsertion.insertionPosition, pureInsertion.insertionText); + // Pure insertion: show the new text as ghost text (multi-line capable) + this.inlineProvider.setInlineReplace({ + position: pureInsertion.insertionPosition, + text: pureInsertion.insertionText + }); } else { - // Case 2: Has deletions - show red strikethrough + // Case 2: Has deletions - show red strikethrough decoration const deletionRanges = computeDeletionRanges(editor.document, range, action.text); if (deletionRanges.length > 0) { @@ -216,18 +219,32 @@ export class PreviewManager { this.decorationPool.setDecorations(editor, 'deletion', [{ range }]); } - // Green highlight on text being added - only if there's actual new content - // Don't show if it's purely a deletion (new text is subset of old text) + // Show new text as ghost text - only if there's actual new content if (hasInsertions(oldText, action.text)) { - // Check if this is a coherent single-substring replacement + // Find where to show the ghost text const coherent = analyzeCoherentReplacement(editor.document, range, action.text); if (coherent.isCoherent && coherent.deletionRange && coherent.insertionText) { - // Coherent: show green text inline right after the red deletion - this.showInlineInsertion(editor, coherent.deletionRange.end, coherent.insertionText); + // Coherent: show ghost text right after the deleted portion + this.inlineProvider.setInlineReplace({ + position: coherent.deletionRange.end, + text: coherent.insertionText + }); } else { - // Not coherent: show green block on next line - this.showInsertionBlock(editor, range.end.line, action.text); + // Not coherent: compute minimal change to show only the actual additions + // This avoids showing the entire replacement text (which would look like duplication) + const minimalChange = computeMinimalChangeRange(oldText, action.text); + + if (minimalChange) { + // Position ghost text at where the change starts in the document + const changeStartLine = range.start.line + minimalChange.oldStart; + const changeStartPos = new vscode.Position(changeStartLine, 0); + + this.inlineProvider.setInlineReplace({ + position: changeStartPos, + text: minimalChange.newText + }); + } } } } @@ -238,57 +255,42 @@ export class PreviewManager { /** * Show inserted text inline at a specific position (right after deleted text). - * Used for coherent single-substring replacements. + * Uses VS Code's inline completion API for proper multi-line display. */ private showInlineInsertion(editor: vscode.TextEditor, position: vscode.Position, text: string): void { - // Format text for display - const displayText = text.replace(/\n/g, '↵').replace(/\t/g, '→'); - const truncatedText = truncate(displayText, 60); + // Don't show empty previews + if (!text.trim()) { + return; + } - const decorationOptions: vscode.DecorationOptions[] = [{ - range: new vscode.Range(position, position), - renderOptions: { - after: { - contentText: truncatedText, - color: COLORS.insertion.foreground, - backgroundColor: COLORS.insertion.background, - fontStyle: 'normal', - border: '1px solid', - borderColor: COLORS.insertion.border, - } - } - }]; - - this.decorationPool.setDecorations(editor, 'insertion-inline', decorationOptions); + // Use inline completion (ghost text) for multi-line support + // This shows the actual code formatting instead of ↵ characters + this.inlineProvider.setInlineReplace({ + position, + text + }); } /** * Show the new/inserted text with green highlight as a block after the specified line. + * Uses VS Code's inline completion API for proper multi-line display. */ private showInsertionBlock(editor: vscode.TextEditor, afterLine: number, text: string): void { - const anchorLine = Math.min(afterLine, editor.document.lineCount - 1); - const anchorPos = new vscode.Position(anchorLine, Number.MAX_SAFE_INTEGER); + // Don't show empty previews + if (!text.trim()) { + return; + } - // Format text for display (escape for CSS content) - const displayText = text.replace(/\n/g, '↵').replace(/\t/g, '→'); - const truncatedText = truncate(displayText, 80); + // Find anchor position - end of the specified line + const anchorLine = Math.min(afterLine, editor.document.lineCount - 1); + const lineLength = editor.document.lineAt(anchorLine).text.length; + const position = new vscode.Position(anchorLine, lineLength); - const decorationOptions: vscode.DecorationOptions[] = [{ - range: new vscode.Range(anchorPos, anchorPos), - renderOptions: { - after: { - contentText: ` + ${truncatedText}`, - color: COLORS.insertion.foreground, - backgroundColor: COLORS.insertion.background, - fontStyle: 'normal', - margin: '0 0 0 2ch', - border: '1px solid', - borderColor: COLORS.insertion.border, - } - } - }]; - - this.decorationPool.setDecorations(editor, 'insertion-block', decorationOptions); + // Use inline completion (ghost text) for multi-line support + this.inlineProvider.setInlineReplace({ + position, + text + }); } /** @@ -319,7 +321,7 @@ export class PreviewManager { const anchorLine = this.getVisibleAnchorLine(editor); const cmdPreview = truncate(action.text, 60); - this.showMetaIndicator(editor, anchorLine, '$(terminal)', `Run: ${cmdPreview}`, COLORS.terminal); + this.showMetaIndicator(editor, anchorLine, '▶', `Run: ${cmdPreview}`, COLORS.terminal); this.hoverProvider.setAction(action, anchorLine); } @@ -342,13 +344,12 @@ export class PreviewManager { if (isTargetVisible) { // Target is visible, show indicator at target anchorLine = targetLine; - icon = '$(arrow-right)'; + icon = '→'; label = 'Move cursor here'; } else { // Target is off-screen, show indicator at edge of visible area anchorLine = this.getVisibleAnchorLine(editor); - const direction = targetLine < anchorLine ? '↑' : '↓'; - icon = `$(arrow-${targetLine < anchorLine ? 'up' : 'down'})`; + icon = targetLine < anchorLine ? '↑' : '↓'; label = `Go to line ${targetLine + 1}`; } @@ -372,7 +373,7 @@ export class PreviewManager { ? `Open: ${fileName}:${targetLine + 1}` : `Open: ${fileName}`; - this.showMetaIndicator(editor, anchorLine, '$(file)', label, COLORS.fileSwitch); + this.showMetaIndicator(editor, anchorLine, '📄', label, COLORS.fileSwitch); this.hoverProvider.setAction(action, anchorLine); } diff --git a/src/preview/inlineProvider.ts b/src/preview/inlineProvider.ts index ff145a0..f7dbf91 100644 --- a/src/preview/inlineProvider.ts +++ b/src/preview/inlineProvider.ts @@ -1,19 +1,43 @@ import * as vscode from 'vscode'; -import { Action, toVscodePosition } from './types'; +import { Action, toVscodePosition, toVscodeRange } from './types'; + +/** + * Data needed to show inline ghost text for replacements. + */ +export interface InlineReplaceData { + /** Position where ghost text should appear (end of deletion range) */ + position: vscode.Position; + /** The new text to show as ghost text */ + text: string; +} /** * Provides inline completion items (ghost text) for code edit actions. * This takes priority over Cursor's hints and works on empty lines. + * Supports multi-line ghost text display. */ export class CrowdPilotInlineProvider implements vscode.InlineCompletionItemProvider { private action: Action | null = null; + private inlineReplaceData: InlineReplaceData | null = null; private enabled: boolean = true; /** - * Set the current action to display as inline completion. + * Set the current action to display as inline completion (for editInsert). */ setAction(action: Action): void { this.action = action; + this.inlineReplaceData = null; + // Trigger VS Code to re-query inline completions + vscode.commands.executeCommand('editor.action.inlineSuggest.trigger'); + } + + /** + * Set inline replacement data for editReplace actions. + * This shows the new text as multi-line ghost text. + */ + setInlineReplace(data: InlineReplaceData): void { + this.inlineReplaceData = data; + this.action = null; // Trigger VS Code to re-query inline completions vscode.commands.executeCommand('editor.action.inlineSuggest.trigger'); } @@ -23,6 +47,7 @@ export class CrowdPilotInlineProvider implements vscode.InlineCompletionItemProv */ clearAction(): void { this.action = null; + this.inlineReplaceData = null; } /** @@ -48,20 +73,31 @@ export class CrowdPilotInlineProvider implements vscode.InlineCompletionItemProv context: vscode.InlineCompletionContext, token: vscode.CancellationToken ): vscode.ProviderResult { - if (!this.enabled || !this.action) { + if (!this.enabled) { return []; } - // Only handle pure insertions (not replacements) - // Replacements are handled by decorations to properly show what's being deleted - if (this.action.kind !== 'editInsert') { + // Handle inline replace data (for editReplace with multi-line new text) + if (this.inlineReplaceData) { + const { position: insertPos, text } = this.inlineReplaceData; + + // Show ghost text at the specified position + const item = new vscode.InlineCompletionItem( + text, + new vscode.Range(insertPos, insertPos) + ); + + return [item]; + } + + // Handle editInsert actions + if (!this.action || this.action.kind !== 'editInsert') { return []; } const insertPos = toVscodePosition(this.action.position); // Only provide completion if insert position is at or after the cursor - // VS Code's inline completion API shows ghost text at/after cursor position if (insertPos.isBefore(position)) { return []; } diff --git a/src/test/diff.test.ts b/src/test/diff.test.ts new file mode 100644 index 0000000..b4492a9 --- /dev/null +++ b/src/test/diff.test.ts @@ -0,0 +1,150 @@ +import * as assert from 'assert'; +import { diffChars, hasInsertions, hasSignificantDiff } from '../utils/diff'; + +suite('Diff Utilities', () => { + + suite('diffChars', () => { + + test('detects simple character insertion', () => { + const diffs = diffChars('hello', 'hello!'); + const insertions = diffs.filter(d => d.type === 'insert'); + assert.strictEqual(insertions.length, 1); + assert.strictEqual(insertions[0].value, '!'); + }); + + test('detects simple character deletion', () => { + const diffs = diffChars('hello!', 'hello'); + const deletions = diffs.filter(d => d.type === 'delete'); + assert.strictEqual(deletions.length, 1); + assert.strictEqual(deletions[0].value, '!'); + }); + + test('detects replacement', () => { + const diffs = diffChars('hello world', 'hello universe'); + const deletions = diffs.filter(d => d.type === 'delete'); + const insertions = diffs.filter(d => d.type === 'insert'); + + // "world" deleted, "universe" inserted + assert.ok(deletions.length > 0); + assert.ok(insertions.length > 0); + }); + + test('handles identical strings', () => { + const diffs = diffChars('hello', 'hello'); + const changes = diffs.filter(d => d.type !== 'equal'); + assert.strictEqual(changes.length, 0); + }); + + test('handles empty to non-empty', () => { + const diffs = diffChars('', 'hello'); + const insertions = diffs.filter(d => d.type === 'insert'); + assert.strictEqual(insertions.length, 1); + assert.strictEqual(insertions[0].value, 'hello'); + }); + + test('handles non-empty to empty', () => { + const diffs = diffChars('hello', ''); + const deletions = diffs.filter(d => d.type === 'delete'); + assert.strictEqual(deletions.length, 1); + assert.strictEqual(deletions[0].value, 'hello'); + }); + + }); + + suite('hasInsertions', () => { + + test('returns true when new content is added', () => { + const result = hasInsertions('def foo():', 'def foo():\n pass'); + assert.strictEqual(result, true); + }); + + test('returns false when content is only deleted', () => { + const result = hasInsertions('def foo():\n pass', 'def foo():'); + assert.strictEqual(result, false); + }); + + test('returns true for replacement with new content', () => { + const result = hasInsertions('hello world', 'hello universe'); + assert.strictEqual(result, true); + }); + + test('returns false for identical strings', () => { + const result = hasInsertions('hello', 'hello'); + assert.strictEqual(result, false); + }); + + test('returns false when insertion is only whitespace', () => { + const result = hasInsertions('hello', 'hello '); + assert.strictEqual(result, false); + }); + + test('returns false for empty new text', () => { + const result = hasInsertions('hello', ''); + assert.strictEqual(result, false); + }); + + }); + + suite('hasSignificantDiff', () => { + + test('returns false for identical strings', () => { + const result = hasSignificantDiff('hello', 'hello'); + assert.strictEqual(result, false); + }); + + test('returns false for whitespace-only differences', () => { + const result = hasSignificantDiff('hello world', 'hello world'); + assert.strictEqual(result, false); + }); + + test('returns true for content differences', () => { + const result = hasSignificantDiff('hello world', 'hello universe'); + assert.strictEqual(result, true); + }); + + test('returns true for added content', () => { + const result = hasSignificantDiff('hello', 'hello world'); + assert.strictEqual(result, true); + }); + + }); + + suite('Integration: Edit preview scenarios', () => { + + test('detects pure insertion at end of line', () => { + // User has "def foo(" and model adds "self):" + const oldText = 'def foo('; + const newText = 'def foo(self):'; + + const result = hasInsertions(oldText, newText); + assert.strictEqual(result, true); + + const diffs = diffChars(oldText, newText); + const insertions = diffs.filter(d => d.type === 'insert'); + assert.ok(insertions.length > 0); + assert.ok(insertions.some(i => i.value.includes('self'))); + }); + + test('detects method body insertion', () => { + const oldText = `def calculate(self, n): +`; + const newText = `def calculate(self, n): + return n * 2 +`; + + const result = hasInsertions(oldText, newText); + assert.strictEqual(result, true); + }); + + test('detects line completion', () => { + const oldText = ' def __init__(self'; + const newText = ' def __init__(self):'; + + const diffs = diffChars(oldText, newText); + const insertions = diffs.filter(d => d.type === 'insert'); + assert.ok(insertions.some(i => i.value === '):')); + }); + + }); + +}); diff --git a/src/test/parsing.test.ts b/src/test/parsing.test.ts new file mode 100644 index 0000000..6e06396 --- /dev/null +++ b/src/test/parsing.test.ts @@ -0,0 +1,390 @@ +import * as assert from 'assert'; +import { + extractLastCodeBlock, + computeChangedLineRange, + computeMinimalChangeRange, + rebaseModelResponse, + removeCursorMarker, +} from '../utils/parsing'; + +suite('Parsing Utilities', () => { + + suite('extractLastCodeBlock', () => { + + test('extracts simple code block with triple backticks', () => { + const text = `Some explanation here. + +\`\`\`python +def foo(): + return 42 +\`\`\` + +More text.`; + const result = extractLastCodeBlock(text); + assert.strictEqual(result, 'def foo():\n return 42'); + }); + + test('extracts last code block when multiple exist', () => { + const text = `First block: +\`\`\` +old code +\`\`\` + +Second block: +\`\`\` +new code here +\`\`\``; + const result = extractLastCodeBlock(text); + assert.strictEqual(result, 'new code here'); + }); + + test('handles five-backtick fences (as used in teacher prompt)', () => { + const text = `The user wants to complete the function. + +\`\`\`\`\` +def calculate(self, n): + if n in self.cached: + return self.cached[n] + return n +\`\`\`\`\``; + const result = extractLastCodeBlock(text); + assert.strictEqual(result, 'def calculate(self, n):\n if n in self.cached:\n return self.cached[n]\n return n'); + }); + + test('preserves leading indentation', () => { + const text = `\`\`\` + class Foo: + def bar(self): + pass +\`\`\``; + const result = extractLastCodeBlock(text); + assert.strictEqual(result, ' class Foo:\n def bar(self):\n pass'); + }); + + test('returns undefined for text without code blocks', () => { + const text = 'Just some plain text without any code blocks.'; + const result = extractLastCodeBlock(text); + assert.strictEqual(result, undefined); + }); + + test('returns undefined for unclosed code block', () => { + const text = `\`\`\` +code without closing fence`; + const result = extractLastCodeBlock(text); + assert.strictEqual(result, undefined); + }); + + test('handles empty code block', () => { + // An "empty" code block still needs content - immediate closing is not valid + // This test verifies that a block with just a newline extracts properly + const text = "```\n\n```"; + const result = extractLastCodeBlock(text); + assert.strictEqual(result, ''); + }); + + test('handles code block with only whitespace', () => { + const text = "```\n \n```"; + const result = extractLastCodeBlock(text); + // Whitespace-only gets trimmed to empty + assert.strictEqual(result, ''); + }); + + }); + + suite('computeChangedLineRange', () => { + + test('returns undefined for identical texts', () => { + const text = 'line 1\nline 2\nline 3'; + const result = computeChangedLineRange(text, text); + assert.strictEqual(result, undefined); + }); + + test('detects change in middle line', () => { + const oldText = 'line 1\nline 2\nline 3'; + const newText = 'line 1\nmodified\nline 3'; + const result = computeChangedLineRange(oldText, newText); + assert.deepStrictEqual(result, { start: 1, end: 1 }); + }); + + test('detects change at start', () => { + const oldText = 'line 1\nline 2\nline 3'; + const newText = 'modified\nline 2\nline 3'; + const result = computeChangedLineRange(oldText, newText); + assert.deepStrictEqual(result, { start: 0, end: 0 }); + }); + + test('detects change at end', () => { + const oldText = 'line 1\nline 2\nline 3'; + const newText = 'line 1\nline 2\nmodified'; + const result = computeChangedLineRange(oldText, newText); + assert.deepStrictEqual(result, { start: 2, end: 2 }); + }); + + test('detects insertion of new lines', () => { + const oldText = 'line 1\nline 3'; + const newText = 'line 1\nline 2\nline 3'; + const result = computeChangedLineRange(oldText, newText); + assert.deepStrictEqual(result, { start: 1, end: 1 }); + }); + + }); + + suite('computeMinimalChangeRange', () => { + + test('returns undefined for identical texts', () => { + const text = 'line 1\nline 2\nline 3'; + const result = computeMinimalChangeRange(text, text); + assert.strictEqual(result, undefined); + }); + + test('computes minimal change for single line modification', () => { + const oldText = 'def foo():\n pass\n return None'; + const newText = 'def foo():\n x = 42\n return None'; + const result = computeMinimalChangeRange(oldText, newText); + assert.deepStrictEqual(result, { + oldStart: 1, + oldEnd: 1, + newText: ' x = 42', + }); + }); + + test('computes change for appended lines', () => { + const oldText = 'def foo():\n pass'; + const newText = 'def foo():\n pass\n return 42'; + const result = computeMinimalChangeRange(oldText, newText); + // When appending, the prefix covers all existing lines + // So the change starts at the position after the last old line + assert.deepStrictEqual(result, { + oldStart: 2, + oldEnd: 2, + newText: ' return 42', + }); + }); + + test('SAFETY: rejects partial model response (no prefix match, much shorter)', () => { + // Simulates model returning only the method body instead of full editable region + const oldText = `def fib(n): + if n <= 1: + return n + return fib(n-1) + fib(n-2) + +class Fibonacci: + def __init__(self): + self.cached = {} + + def calculate()`; + + // Model only returns the calculate method body + const newText = ` def calculate(self, n): + if n in self.cached: + return self.cached[n] + return n`; + + const result = computeMinimalChangeRange(oldText, newText); + // Should reject because prefix=0 and would delete many more lines than adding + assert.strictEqual(result, undefined); + }); + + test('SAFETY: rejects when suffix matches but would delete too much', () => { + // Edge case: last line "return result" matches in both, but first lines don't + const oldText = `def fib(n): + if n <= 1: + return n + return fib(n-1) + fib(n-2) + +class Fibonacci: + def calculate(self): + result = 42 + return result`; + + // Model returns just the method body, but "return result" matches at end + const newText = ` if n in self.cached: + return self.cached[n] + return result`; + + const result = computeMinimalChangeRange(oldText, newText); + // Should reject because would delete too many lines from the beginning + assert.strictEqual(result, undefined); + }); + + test('accepts valid edit with common prefix', () => { + const oldText = `def foo(): + pass`; + const newText = `def foo(): + x = 42 + return x`; + + const result = computeMinimalChangeRange(oldText, newText); + assert.ok(result !== undefined); + assert.strictEqual(result.oldStart, 1); + assert.strictEqual(result.newText, ' x = 42\n return x'); + }); + + }); + + suite('rebaseModelResponse (stale completion handling)', () => { + + test('rebases when user typed what model was going to predict', () => { + // User had "def foo" and typed "()" while waiting + // Model returns "def foo():\n pass" + const originalText = 'def foo'; + const modelOutput = 'def foo():\n pass'; + const currentText = 'def foo()'; // User typed "()" + + const result = rebaseModelResponse(originalText, modelOutput, currentText); + + // Should compute diff between current "def foo()" and model's "def foo():\n pass" + // Result: only need to add ":\n pass" + assert.ok(result !== undefined); + assert.strictEqual(result.oldStart, 0); + assert.strictEqual(result.newText, 'def foo():\n pass'); + }); + + test('rebases multi-line completion with user continuation', () => { + // User had partial class, typed more while waiting + const originalText = `class Foo: + def __init__(self`; + + const modelOutput = `class Foo: + def __init__(self): + self.value = 0`; + + const currentText = `class Foo: + def __init__(self):`; // User typed "):" + + const result = rebaseModelResponse(originalText, modelOutput, currentText); + + assert.ok(result !== undefined); + // Rebasing against current text: line 0 matches, line 1 differs + // Current has "def __init__(self):", model has "def __init__(self):" + // Actually they might match! Then only the body needs adding + assert.ok(result.oldStart >= 1); + }); + + test('returns same result when document unchanged', () => { + const text = 'def foo():\n pass'; + const modelOutput = 'def foo():\n return 42'; + + const result = rebaseModelResponse(text, modelOutput, text); + + assert.ok(result !== undefined); + assert.strictEqual(result.oldStart, 1); + assert.strictEqual(result.newText, ' return 42'); + }); + + test('handles user typing ahead significantly', () => { + // User typed a lot while waiting + const originalText = 'def calculate('; + const modelOutput = 'def calculate(n):\n return n * 2'; + const currentText = 'def calculate(n):'; + + const result = rebaseModelResponse(originalText, modelOutput, currentText); + + assert.ok(result !== undefined); + // After rebasing: current="def calculate(n):", model="def calculate(n):\n return n * 2" + // Line 0 doesn't match exactly (current has no newline continuation) + // So the result includes the full replacement + assert.ok(result.newText.includes('return n * 2')); + }); + + }); + + suite('removeCursorMarker', () => { + + test('removes cursor marker from text', () => { + const text = 'def foo(<|user_cursor|>):'; + const result = removeCursorMarker(text); + assert.strictEqual(result, 'def foo():'); + }); + + test('removes multiple cursor markers', () => { + const text = '<|user_cursor|>def foo():<|user_cursor|>'; + const result = removeCursorMarker(text); + assert.strictEqual(result, 'def foo():'); + }); + + test('handles text without cursor marker', () => { + const text = 'def foo():'; + const result = removeCursorMarker(text); + assert.strictEqual(result, 'def foo():'); + }); + + test('uses custom marker', () => { + const text = 'def foo(CURSOR):'; + const result = removeCursorMarker(text, 'CURSOR'); + assert.strictEqual(result, 'def foo():'); + }); + + }); + + suite('Integration: Full model response parsing', () => { + + test('parses complete teacher model response', () => { + // Simulate a full model response + const modelResponse = `The user is implementing a Fibonacci calculator. They need to complete the calculate method. + +\`\`\`\`\` +class Fibonacci: + def __init__(self): + self.cached = {} + + def calculate(self, n): + if n in self.cached: + return self.cached[n] + if n <= 1: + return n + result = self.calculate(n-1) + self.calculate(n-2) + self.cached[n] = result + return result +\`\`\`\`\``; + + const originalEditableText = `class Fibonacci: + def __init__(self): + self.cached = {} + + def calculate()`; + + // Extract code block + const codeBlock = extractLastCodeBlock(modelResponse); + assert.ok(codeBlock !== undefined); + + // Remove cursor marker if present + const cleanedCode = removeCursorMarker(codeBlock); + + // Compute minimal change + const changeRange = computeMinimalChangeRange(originalEditableText, cleanedCode); + assert.ok(changeRange !== undefined); + + // Should detect that we're changing from line 4 (the calculate definition) + assert.strictEqual(changeRange.oldStart, 4); + }); + + test('rejects model response that would delete class definition', () => { + // Model returns only the method body, would delete 8 lines but only add 2 + const modelResponse = `\`\`\`\`\` + def calculate(self, n): + return n +\`\`\`\`\``; + + // Larger original to trigger safety check (needs linesDeleted > linesAdded + 3) + const originalEditableText = `class Fibonacci: + """A Fibonacci calculator with caching.""" + + def __init__(self): + self.cached = {} + self.calls = 0 + + def calculate()`; + + const codeBlock = extractLastCodeBlock(modelResponse); + assert.ok(codeBlock !== undefined); + + const changeRange = computeMinimalChangeRange(originalEditableText, codeBlock); + // Safety check: prefix=0, deleting 8 lines, adding 2 + // 8 > 2 + 3 = 8 > 5 = true, so should reject + assert.strictEqual(changeRange, undefined); + }); + + }); + +}); diff --git a/src/utils/parsing.ts b/src/utils/parsing.ts new file mode 100644 index 0000000..efddda4 --- /dev/null +++ b/src/utils/parsing.ts @@ -0,0 +1,167 @@ +/** + * Parsing utilities for model responses and edit computation. + * Extracted for testability without VS Code dependencies. + */ + +export type LineRange = { start: number; end: number }; + +export type MinimalChangeRange = { + oldStart: number; // First line that differs in old text + oldEnd: number; // Last line that differs in old text (inclusive) + newText: string; // The replacement text for just those lines +}; + +/** + * Extract the last code block from model response text. + * Handles fenced code blocks with 3+ backticks. + */ +export function extractLastCodeBlock(text: string): string | undefined { + let lastBlock: string | undefined; + let searchStart = 0; + + while (true) { + const start = text.indexOf("```", searchStart); + if (start === -1) { + break; + } + let end = start; + while (end < text.length && text[end] === "`") { + end += 1; + } + const fence = text.slice(start, end); + const lineBreak = text.indexOf("\n", end); + if (lineBreak === -1) { + break; + } + const closing = text.indexOf(`\n${fence}`, lineBreak + 1); + if (closing === -1) { + searchStart = end; + continue; + } + lastBlock = text.slice(lineBreak + 1, closing + 1); + searchStart = closing + fence.length + 1; + } + + // Only trim trailing whitespace to preserve leading indentation + return lastBlock?.trimEnd(); +} + +/** + * Compute the range of lines that changed between old and new text. + * Returns undefined if texts are identical. + */ +export function computeChangedLineRange(oldText: string, newText: string): LineRange | undefined { + const oldLines = oldText.split(/\r?\n/); + const newLines = newText.split(/\r?\n/); + let prefix = 0; + while ( + prefix < oldLines.length && + prefix < newLines.length && + oldLines[prefix] === newLines[prefix] + ) { + prefix += 1; + } + let suffix = 0; + while ( + suffix < oldLines.length - prefix && + suffix < newLines.length - prefix && + oldLines[oldLines.length - 1 - suffix] === newLines[newLines.length - 1 - suffix] + ) { + suffix += 1; + } + if (prefix === oldLines.length && prefix === newLines.length) { + return undefined; + } + const endLine = Math.max(prefix, newLines.length - suffix - 1); + return { start: prefix, end: Math.max(prefix, endLine) }; +} + +/** + * Compute the minimal range of lines that changed between old and new text. + * Returns the precise start/end lines in the old text and the replacement text. + * Returns undefined if no change or if the edit looks unsafe (would delete too much). + */ +export function computeMinimalChangeRange(oldText: string, newText: string): MinimalChangeRange | undefined { + const oldLines = oldText.split(/\r?\n/); + const newLines = newText.split(/\r?\n/); + + // Find common prefix (lines that match at the start) + let prefix = 0; + while ( + prefix < oldLines.length && + prefix < newLines.length && + oldLines[prefix] === newLines[prefix] + ) { + prefix += 1; + } + + // Find common suffix (lines that match at the end) + let suffix = 0; + while ( + suffix < oldLines.length - prefix && + suffix < newLines.length - prefix && + oldLines[oldLines.length - 1 - suffix] === newLines[newLines.length - 1 - suffix] + ) { + suffix += 1; + } + + // If everything matches, no change + if (prefix === oldLines.length && prefix === newLines.length) { + return undefined; + } + + // Compute the changed region + const oldStart = prefix; + const oldEnd = Math.max(prefix, oldLines.length - suffix - 1); + const newStart = prefix; + const newEnd = Math.max(prefix, newLines.length - suffix - 1); + + const linesDeleted = oldEnd - oldStart + 1; + const linesAdded = newEnd - newStart + 1; + + // SAFETY CHECK: If no common prefix (replacing from line 0) AND we would + // delete significantly more lines than we add, the model likely returned + // only a portion of the editable region. Reject to avoid deleting user's code. + if (prefix === 0 && linesDeleted > linesAdded + 3 && linesDeleted > oldLines.length * 0.3) { + return undefined; + } + + // Extract just the changed lines from new text + const changedNewLines = newLines.slice(newStart, newEnd + 1); + + return { + oldStart, + oldEnd, + newText: changedNewLines.join('\n'), + }; +} + +/** + * Simulate rebasing a model response onto a newer document state. + * This is used when the user types while waiting for the model response. + * + * @param originalEditableText - The editable text at the time of the model request + * @param modelNewText - The model's predicted new text for the editable region + * @param currentEditableText - The current editable text (after user's edits) + * @returns The rebased change range, or undefined if rebasing isn't possible + */ +export function rebaseModelResponse( + originalEditableText: string, + modelNewText: string, + currentEditableText: string +): MinimalChangeRange | undefined { + // If current text is the same as original, no rebasing needed + if (originalEditableText === currentEditableText) { + return computeMinimalChangeRange(originalEditableText, modelNewText); + } + + // Rebase: diff model's output against current text instead of original + return computeMinimalChangeRange(currentEditableText, modelNewText); +} + +/** + * Remove cursor marker from text. + */ +export function removeCursorMarker(text: string, marker: string = '<|user_cursor|>'): string { + return text.replaceAll(marker, ''); +} diff --git a/src/utils/utilities.ts b/src/utils/utilities.ts new file mode 100644 index 0000000..b23b340 --- /dev/null +++ b/src/utils/utilities.ts @@ -0,0 +1,13 @@ +import * as vscode from "vscode"; + +export const outputChannel = vscode.window.createOutputChannel("crowd-pilot"); + +export function logToOutput( + message: string, + type: "info" | "success" | "error" = "info", +) { + const time = new Date().toLocaleTimeString(); + + outputChannel.appendLine(`${time} [${type}] ${message}`); + console.log(message); +}