diff --git a/packages/askui-nodejs/.gitignore b/packages/askui-nodejs/.gitignore index 37bc45df906..2f5eed059a9 100644 --- a/packages/askui-nodejs/.gitignore +++ b/packages/askui-nodejs/.gitignore @@ -4,3 +4,5 @@ dist node_modules +agentos-test-screenshot.png +android-test-screenshot.png diff --git a/packages/askui-nodejs/README.md b/packages/askui-nodejs/README.md index 5264b795975..54fa673cd5a 100644 --- a/packages/askui-nodejs/README.md +++ b/packages/askui-nodejs/README.md @@ -1,5 +1,11 @@ # AskUI - What Can Be Said Can Be Solved +> [!IMPORTANT] +> **Maintenance mode.** This library is in maintenance mode and is no longer under +> active development — only critical bug and security fixes are applied. It will be +> **deprecated at the end of 2026**. For new projects and continued development, use +> the [AskUI Python SDK](https://github.com/askui/vision-agent). + **AskUI** allows you to automate the interaction with an application, multiple applications or even the entire operating system. You can use this to write end-to-end tests or automate any kind of application. diff --git a/packages/askui-nodejs/example_projects_templates/templates/askui-helper.nj b/packages/askui-nodejs/example_projects_templates/templates/askui-helper.nj index 7671a4691fd..d2ab2a68c6c 100644 --- a/packages/askui-nodejs/example_projects_templates/templates/askui-helper.nj +++ b/packages/askui-nodejs/example_projects_templates/templates/askui-helper.nj @@ -15,21 +15,8 @@ beforeAll(async () => { await aui.connect(); }); -beforeEach(async () => { - /* Uncomment to enable video recording - await aui.startVideoRecording(); - */ -}); - -afterEach(async () => { - /* Uncomment to enable video recording - await aui.stopVideoRecording(); - {{ allure_stepreporter_attach_video }} - */ -}); - afterAll(async () => { - aui.disconnect(); + await aui.disconnect(); }); export { aui }; diff --git a/packages/askui-nodejs/package-lock.json b/packages/askui-nodejs/package-lock.json index 13fd1954fff..81069da3e80 100644 --- a/packages/askui-nodejs/package-lock.json +++ b/packages/askui-nodejs/package-lock.json @@ -11,9 +11,10 @@ "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "0.53.0", + "@grpc/grpc-js": "^1.14.4", + "@grpc/proto-loader": "^0.8.1", "chalk": "4.1.1", "commander": "12.1.0", - "fkill": "7.2.1", "fs-extra": "11.2.0", "got": "11.8.6", "inquirer": "8.2.5", @@ -27,8 +28,7 @@ "pino-pretty": "11.2.2", "tough-cookie": "5.0.0", "url-join": "4.0.1", - "wait-port": "1.1.0", - "ws": "8.18.0", + "ws": "^8.18.0", "yup": "1.4.0" }, "bin": { @@ -50,7 +50,7 @@ "@types/nunjucks": "3.2.6", "@types/url-join": "4.0.3", "@types/webrtc": "0.0.43", - "@types/ws": "8.5.12", + "@types/ws": "^8.5.12", "@typescript-eslint/eslint-plugin": "5.10.0", "@typescript-eslint/parser": "5.10.0", "cross-env": "^10.1.0", @@ -67,6 +67,7 @@ "release-it": "^20.0.1", "shx": "0.3.4", "ts-jest": "29.2.5", + "ts-node": "^10.9.2", "typescript": "5.6.2" }, "optionalDependencies": { @@ -629,6 +630,30 @@ } } }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, "node_modules/@cyclonedx/cyclonedx-npm": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/@cyclonedx/cyclonedx-npm/-/cyclonedx-npm-4.1.2.tgz", @@ -811,6 +836,37 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.4", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.4.tgz", + "integrity": "sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", + "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.9.5", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.5.tgz", @@ -2088,6 +2144,16 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -2392,6 +2458,63 @@ "node": ">=14" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, "node_modules/@release-it/bumper": { "version": "7.0.5", "resolved": "https://registry.npmjs.org/@release-it/bumper/-/bumper-7.0.5.tgz", @@ -2551,6 +2674,34 @@ "dev": true, "license": "MIT" }, + "node_modules/@tsconfig/node10": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", + "integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", @@ -2828,6 +2979,7 @@ "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.12.tgz", "integrity": "sha512-3tPRkv1EtkDpzlgyKyI8pGsGZAGPEaXeu0DOj5DI25Ja91bdAYddYHbADRYVrZMRbfW+1l5YwXVDKohDJNQxkQ==", "dev": true, + "license": "MIT", "dependencies": { "@types/node": "*" } @@ -3078,6 +3230,19 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/acorn-walk": { + "version": "8.3.5", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.5.tgz", + "integrity": "sha512-HEHNfbars9v4pgpW6SO1KSPkfoS0xVOM/9UzkJltjlsHZmJasxg8aXkuZa7SMf8vKGIBhpUsPluQSqhJFCqebw==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -3087,18 +3252,6 @@ "node": ">= 14" } }, - "node_modules/aggregate-error": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", - "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", - "dependencies": { - "clean-stack": "^2.0.0", - "indent-string": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/ajv": { "version": "6.14.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", @@ -3243,6 +3396,13 @@ "node": ">= 8" } }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true, + "license": "MIT" + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -3445,14 +3605,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/arrify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", - "engines": { - "node": ">=8" - } - }, "node_modules/asap": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", @@ -4250,14 +4402,6 @@ "integrity": "sha512-cuSVIHi9/9E/+821Qjdvngor+xpnlwnuwIyZOaLmHBVdXL+gP+I6QQB9VkO7RI77YIcTV+S1W9AreJ5eN63JBA==", "dev": true }, - "node_modules/clean-stack": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", - "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", - "engines": { - "node": ">=6" - } - }, "node_modules/cli-cursor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", @@ -4347,7 +4491,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", @@ -4672,6 +4815,13 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true, + "license": "MIT" + }, "node_modules/cross-env": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-10.1.0.tgz", @@ -4694,6 +4844,7 @@ "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, "license": "MIT", "dependencies": { "path-key": "^3.1.0", @@ -5077,6 +5228,16 @@ "node": ">=8" } }, + "node_modules/diff": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.4.tgz", + "integrity": "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/diff-sequences": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", @@ -5511,7 +5672,6 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, "engines": { "node": ">=6" } @@ -5960,6 +6120,7 @@ "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "dev": true, "dependencies": { "cross-spawn": "^7.0.3", "get-stream": "^6.0.0", @@ -6300,26 +6461,6 @@ "node": ">=8" } }, - "node_modules/fkill": { - "version": "7.2.1", - "resolved": "https://registry.npmjs.org/fkill/-/fkill-7.2.1.tgz", - "integrity": "sha512-eN9cmsIlRdq06wu3m01OOEgQf5Xh/M7REm0jfZ4eL3V3XisjXzfRq3iyqtKS+FhO6wB36FvWRiRGdeSx5KpLAQ==", - "dependencies": { - "aggregate-error": "^3.1.0", - "arrify": "^2.0.1", - "execa": "^5.0.0", - "pid-port": "^0.1.0", - "process-exists": "^4.0.0", - "ps-list": "^7.2.0", - "taskkill": "^3.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/flat-cache": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", @@ -6502,7 +6643,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, "engines": { "node": "6.* || 8.* || >= 10.*" } @@ -6570,6 +6710,7 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true, "engines": { "node": ">=10" }, @@ -7031,6 +7172,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "dev": true, "engines": { "node": ">=10.17.0" } @@ -7118,14 +7260,6 @@ "node": ">=0.8.19" } }, - "node_modules/indent-string": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", - "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", - "engines": { - "node": ">=8" - } - }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -7548,6 +7682,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, "engines": { "node": ">=8" }, @@ -7648,7 +7783,8 @@ "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true }, "node_modules/issue-parser": { "version": "7.0.1", @@ -9312,6 +9448,12 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, "node_modules/lodash.capitalize": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/lodash.capitalize/-/lodash.capitalize-4.2.1.tgz", @@ -9537,6 +9679,12 @@ "node": ">=4" } }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/lowercase-keys": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", @@ -9642,7 +9790,8 @@ "node_modules/merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", - "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==" + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true }, "node_modules/merge2": { "version": "1.4.1", @@ -10167,6 +10316,7 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, "dependencies": { "path-key": "^3.0.0" }, @@ -10472,14 +10622,6 @@ "node": ">=8" } }, - "node_modules/p-finally": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-2.0.1.tgz", - "integrity": "sha512-vpm09aKwq6H9phqRQzecoDpD8TmVyGw70qmWlyq5onxY7tqyTTFVvxMykxQSQKILBSFlbXpypIw2T1Ml7+DDtw==", - "engines": { - "node": ">=8" - } - }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -10713,6 +10855,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, "engines": { "node": ">=8" } @@ -10783,20 +10926,6 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/pid-port": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/pid-port/-/pid-port-0.1.1.tgz", - "integrity": "sha512-boqPJtSgZC6KOgXKNPC+/XR3xwVtpOtaLa7JLcdf8jfVe0ZM2TwllBXxxLUO8GQbOLJ4/hEtf2+L1QCKbaoHUg==", - "dependencies": { - "execa": "^5.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/pidtree": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/pidtree/-/pidtree-0.6.0.tgz", @@ -11079,28 +11208,6 @@ "node": ">= 0.6.0" } }, - "node_modules/process-exists": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/process-exists/-/process-exists-4.1.0.tgz", - "integrity": "sha512-BBJoiorUKoP2AuM5q/yKwIfT1YWRHsaxjW+Ayu9erLhqKOfnXzzVVML0XTYoQZuI1YvcWKmc1dh06DEy4+KzfA==", - "dependencies": { - "ps-list": "^6.3.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/process-exists/node_modules/ps-list": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/ps-list/-/ps-list-6.3.0.tgz", - "integrity": "sha512-qau0czUSB0fzSlBOQt0bo+I2v6R+xiQdj78e1BR/Qjfl5OHWJ/urXi8+ilw1eHe+5hSeDI1wrwVTgDp2wst4oA==", - "engines": { - "node": ">=8" - } - }, "node_modules/process-warning": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-4.0.0.tgz", @@ -11148,6 +11255,29 @@ "resolved": "https://registry.npmjs.org/property-expr/-/property-expr-2.0.6.tgz", "integrity": "sha512-SVtmxhRE/CGkn3eZY1T6pC8Nln6Fr/lu1mKSgRud0eC73whjGfoAogbn78LkD8aFL0zz3bAFerKSnOl7NlErBA==" }, + "node_modules/protobufjs": { + "version": "7.6.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", + "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/protocols": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/protocols/-/protocols-2.0.2.tgz", @@ -11204,17 +11334,6 @@ "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", "dev": true }, - "node_modules/ps-list": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/ps-list/-/ps-list-7.2.0.tgz", - "integrity": "sha512-v4Bl6I3f2kJfr5o80ShABNHAokIgY+wFDTQfE+X3zWYgSGQOCBeYptLZUpoOALBqO5EawmDN/tjTldJesd0ujQ==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/pump": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", @@ -11926,7 +12045,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -12315,6 +12433,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, "dependencies": { "shebang-regex": "^3.0.0" }, @@ -12326,6 +12445,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, "engines": { "node": ">=8" } @@ -12784,6 +12904,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "dev": true, "engines": { "node": ">=6" } @@ -12900,60 +13021,6 @@ "node": ">=18" } }, - "node_modules/taskkill": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/taskkill/-/taskkill-3.1.0.tgz", - "integrity": "sha512-5KcOFzPvd1nGFVrmB7H4+QAWVjYOf//+QTbOj0GpXbqtqbKGWVczG+rq6VhXAtdtlKLTs16NAmHRyF5vbggQ2w==", - "dependencies": { - "arrify": "^2.0.1", - "execa": "^3.3.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/taskkill/node_modules/execa": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/execa/-/execa-3.4.0.tgz", - "integrity": "sha512-r9vdGQk4bmCuK1yKQu1KTwcT2zwfWdbdaXfCtAh+5nU/4fSX+JAb7vZGvI5naJrQlvONrEB20jeruESI69530g==", - "dependencies": { - "cross-spawn": "^7.0.0", - "get-stream": "^5.0.0", - "human-signals": "^1.1.1", - "is-stream": "^2.0.0", - "merge-stream": "^2.0.0", - "npm-run-path": "^4.0.0", - "onetime": "^5.1.0", - "p-finally": "^2.0.0", - "signal-exit": "^3.0.2", - "strip-final-newline": "^2.0.0" - }, - "engines": { - "node": "^8.12.0 || >=9.7.0" - } - }, - "node_modules/taskkill/node_modules/get-stream": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", - "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", - "dependencies": { - "pump": "^3.0.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/taskkill/node_modules/human-signals": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-1.1.1.tgz", - "integrity": "sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==", - "engines": { - "node": ">=8.12.0" - } - }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -13166,6 +13233,50 @@ } } }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, "node_modules/tsconfig-paths": { "version": "3.15.0", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz", @@ -13503,6 +13614,13 @@ "integrity": "sha512-ocyWc3bAHBB/guyqJQVI5o4BZkPhznPYUG2ea80Gond/BgNWpap8TOmLSeeQG7bnh2KMISxskdADG59j7zruhw==", "dev": true }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true, + "license": "MIT" + }, "node_modules/v8-to-istanbul": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", @@ -13538,45 +13656,6 @@ "node": ">=18" } }, - "node_modules/wait-port": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/wait-port/-/wait-port-1.1.0.tgz", - "integrity": "sha512-3e04qkoN3LxTMLakdqeWth8nih8usyg+sf1Bgdf9wwUkp05iuK1eSY/QpLvscT/+F/gA89+LpUmmgBtesbqI2Q==", - "dependencies": { - "chalk": "^4.1.2", - "commander": "^9.3.0", - "debug": "^4.3.4" - }, - "bin": { - "wait-port": "bin/wait-port.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/wait-port/node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/wait-port/node_modules/commander": { - "version": "9.5.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", - "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==", - "engines": { - "node": "^12.20.0 || >=14" - } - }, "node_modules/walk-up-path": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/walk-up-path/-/walk-up-path-4.0.0.tgz", @@ -13658,6 +13737,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, "dependencies": { "isexe": "^2.0.0" }, @@ -13812,6 +13892,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", + "license": "MIT", "engines": { "node": ">=10.0.0" }, @@ -13877,7 +13958,6 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, "engines": { "node": ">=10" } @@ -13908,7 +13988,6 @@ "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "dev": true, "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -13926,11 +14005,20 @@ "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, "engines": { "node": ">=12" } }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/packages/askui-nodejs/package.json b/packages/askui-nodejs/package.json index 522b7c295e8..993e12711f9 100644 --- a/packages/askui-nodejs/package.json +++ b/packages/askui-nodejs/package.json @@ -34,12 +34,15 @@ "askui-postinstall": "bin/askui-postinstall" }, "scripts": { - "build": "tsc -p tsconfig.json && tsc -p tsconfig-cjs.json && npm run copyTemplate && npm run copyExampleProject", + "build": "shx rm -rf dist && tsc -p tsconfig.json && tsc -p tsconfig-cjs.json && npm run copyTemplate && npm run copyProto && npm run copyExampleProject", "copyTemplate": "shx cp -r src/core/annotation/template.html dist/cjs/core/annotation/ && shx cp -r src/core/annotation/template.html dist/esm/core/annotation/", + "copyProto": "shx mkdir -p dist/cjs/execution/agent-os/proto dist/esm/execution/agent-os/proto && shx cp src/execution/agent-os/proto/Controller_V1.proto dist/cjs/execution/agent-os/proto/ && shx cp src/execution/agent-os/proto/Controller_V1.proto dist/esm/execution/agent-os/proto/", "copyExampleProject": "shx cp -r example_projects_templates/ dist/ && shx cp -r bin/* dist/", "lint": "eslint --cache --fix --max-warnings 0 \"./**/*.{js,ts}\"", "lint:staged": "lint-staged", "test": "cross-env NODE_EXTRA_CA_CERTS='test/proxy/certs/unit_test.pem' jest '/src'", + "test:agentos": "ts-node -T -P tsconfig-cjs.json test.ts", + "test:android": "ts-node -T -P tsconfig-cjs.json test-android.ts", "postinstall": "node -e \"require('./bin/askui-postinstall')\"", "release:prerelease": "cross-env HUSKY=0 release-it --preRelease=next", "release": "cross-env HUSKY=0 release-it", @@ -53,9 +56,10 @@ ], "dependencies": { "@anthropic-ai/sdk": "0.53.0", + "@grpc/grpc-js": "^1.14.4", + "@grpc/proto-loader": "^0.8.1", "chalk": "4.1.1", "commander": "12.1.0", - "fkill": "7.2.1", "fs-extra": "11.2.0", "got": "11.8.6", "inquirer": "8.2.5", @@ -69,8 +73,7 @@ "pino-pretty": "11.2.2", "tough-cookie": "5.0.0", "url-join": "4.0.1", - "wait-port": "1.1.0", - "ws": "8.18.0", + "ws": "^8.18.0", "yup": "1.4.0" }, "devDependencies": { @@ -88,7 +91,7 @@ "@types/nunjucks": "3.2.6", "@types/url-join": "4.0.3", "@types/webrtc": "0.0.43", - "@types/ws": "8.5.12", + "@types/ws": "^8.5.12", "@typescript-eslint/eslint-plugin": "5.10.0", "@typescript-eslint/parser": "5.10.0", "cross-env": "^10.1.0", @@ -105,6 +108,7 @@ "release-it": "^20.0.1", "shx": "0.3.4", "ts-jest": "29.2.5", + "ts-node": "^10.9.2", "typescript": "5.6.2" }, "lint-staged": { diff --git a/packages/askui-nodejs/src/core/runner-protocol/index.ts b/packages/askui-nodejs/src/core/runner-protocol/index.ts deleted file mode 100644 index a05175977f8..00000000000 --- a/packages/askui-nodejs/src/core/runner-protocol/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './request'; -export * from './response'; diff --git a/packages/askui-nodejs/src/core/runner-protocol/request/capture-screenshot-request.ts b/packages/askui-nodejs/src/core/runner-protocol/request/capture-screenshot-request.ts deleted file mode 100644 index 2ceec32e621..00000000000 --- a/packages/askui-nodejs/src/core/runner-protocol/request/capture-screenshot-request.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { RunnerProtocolRequest } from './runner-protocol-request'; - -export class CaptureScreenshotRequest implements RunnerProtocolRequest { - static msgName = 'CAPTURE_SCREENSHOT_REQUEST'; - - msgName: string = CaptureScreenshotRequest.msgName; -} diff --git a/packages/askui-nodejs/src/core/runner-protocol/request/control-request.ts b/packages/askui-nodejs/src/core/runner-protocol/request/control-request.ts deleted file mode 100644 index 66d4895004a..00000000000 --- a/packages/askui-nodejs/src/core/runner-protocol/request/control-request.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { ControlCommand } from '../../ui-control-commands'; -import { RunnerProtocolRequest } from './runner-protocol-request'; - -export class ControlRequest implements RunnerProtocolRequest { - static msgName = 'CONTROL_REQUEST'; - - msgName: string = ControlRequest.msgName; - - constructor(public controlCommand: ControlCommand) { } -} diff --git a/packages/askui-nodejs/src/core/runner-protocol/request/get-server-process-pid.ts b/packages/askui-nodejs/src/core/runner-protocol/request/get-server-process-pid.ts deleted file mode 100644 index 8f76757f1f8..00000000000 --- a/packages/askui-nodejs/src/core/runner-protocol/request/get-server-process-pid.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { RunnerProtocolRequest } from './runner-protocol-request'; - -export class GetProcessPidRequest implements RunnerProtocolRequest { - static msgName = 'GET_PROCESS_PID_REQUEST'; - - msgName: string = GetProcessPidRequest.msgName; -} diff --git a/packages/askui-nodejs/src/core/runner-protocol/request/get-starting-arguments-request.ts b/packages/askui-nodejs/src/core/runner-protocol/request/get-starting-arguments-request.ts deleted file mode 100644 index 090dfd17208..00000000000 --- a/packages/askui-nodejs/src/core/runner-protocol/request/get-starting-arguments-request.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { RunnerProtocolRequest } from './runner-protocol-request'; - -export class GetStartingArgumentsRequest implements RunnerProtocolRequest { - static msgName = 'GET_STARTING_ARGUMENTS_REQUEST'; - - msgName: string = GetStartingArgumentsRequest.msgName; -} diff --git a/packages/askui-nodejs/src/core/runner-protocol/request/index.ts b/packages/askui-nodejs/src/core/runner-protocol/request/index.ts deleted file mode 100644 index 2cbe710fda2..00000000000 --- a/packages/askui-nodejs/src/core/runner-protocol/request/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export { CaptureScreenshotRequest } from './capture-screenshot-request'; -export { ControlRequest } from './control-request'; -export { InteractiveAnnotationRequest } from './interactive-annotation-request'; -export { ReadRecordingRequest } from './read-recording-request'; -export { RunnerProtocolRequest } from './runner-protocol-request'; -export { StartRecordingRequest } from './start-recording-request'; -export { StopRecordingRequest } from './stop-recording-request'; -export { GetProcessPidRequest } from './get-server-process-pid'; -export { GetStartingArgumentsRequest } from './get-starting-arguments-request'; diff --git a/packages/askui-nodejs/src/core/runner-protocol/request/interactive-annotation-request.ts b/packages/askui-nodejs/src/core/runner-protocol/request/interactive-annotation-request.ts deleted file mode 100644 index 9c9caf6c34c..00000000000 --- a/packages/askui-nodejs/src/core/runner-protocol/request/interactive-annotation-request.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { DetectedElement } from '../../model/annotation-result/detected-element'; -import { RunnerProtocolRequest } from './runner-protocol-request'; - -export class InteractiveAnnotationRequest implements RunnerProtocolRequest { - static msgName = 'START_INTERACTIVE_ANNOTATION_REQUEST'; - - boundingBoxes: DetectedElement[]; - - imageString: string; - - msgName: string = InteractiveAnnotationRequest.msgName; - - constructor(boundingBoxes: DetectedElement[], imageString: string) { - this.boundingBoxes = boundingBoxes; - this.imageString = imageString; - } -} diff --git a/packages/askui-nodejs/src/core/runner-protocol/request/read-recording-request.ts b/packages/askui-nodejs/src/core/runner-protocol/request/read-recording-request.ts deleted file mode 100644 index 8247d24d42e..00000000000 --- a/packages/askui-nodejs/src/core/runner-protocol/request/read-recording-request.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { RunnerProtocolRequest } from './runner-protocol-request'; - -export class ReadRecordingRequest implements RunnerProtocolRequest { - static msgName = 'READ_RECORDING_REQUEST'; - - msgName: string = ReadRecordingRequest.msgName; -} diff --git a/packages/askui-nodejs/src/core/runner-protocol/request/runner-protocol-request.ts b/packages/askui-nodejs/src/core/runner-protocol/request/runner-protocol-request.ts deleted file mode 100644 index 20e09fa6fdf..00000000000 --- a/packages/askui-nodejs/src/core/runner-protocol/request/runner-protocol-request.ts +++ /dev/null @@ -1,3 +0,0 @@ -export interface RunnerProtocolRequest { - msgName: string; -} diff --git a/packages/askui-nodejs/src/core/runner-protocol/request/start-recording-request.ts b/packages/askui-nodejs/src/core/runner-protocol/request/start-recording-request.ts deleted file mode 100644 index 4465277e10f..00000000000 --- a/packages/askui-nodejs/src/core/runner-protocol/request/start-recording-request.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { RunnerProtocolRequest } from './runner-protocol-request'; - -export class StartRecordingRequest implements RunnerProtocolRequest { - static msgName = 'START_RECORDING_REQUEST'; - - msgName: string = StartRecordingRequest.msgName; -} diff --git a/packages/askui-nodejs/src/core/runner-protocol/request/stop-recording-request.ts b/packages/askui-nodejs/src/core/runner-protocol/request/stop-recording-request.ts deleted file mode 100644 index 332d1ef0c86..00000000000 --- a/packages/askui-nodejs/src/core/runner-protocol/request/stop-recording-request.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { RunnerProtocolRequest } from './runner-protocol-request'; - -export class StopRecordingRequest implements RunnerProtocolRequest { - static msgName = 'STOP_RECORDING_REQUEST'; - - msgName: string = StopRecordingRequest.msgName; -} diff --git a/packages/askui-nodejs/src/core/runner-protocol/response/index.ts b/packages/askui-nodejs/src/core/runner-protocol/response/index.ts deleted file mode 100644 index 6ce6a06daba..00000000000 --- a/packages/askui-nodejs/src/core/runner-protocol/response/index.ts +++ /dev/null @@ -1,84 +0,0 @@ -export interface RunnerProtocolResponse { - data: { error?: string } - msgName: string -} - -export interface CaptureScreenshotFailureResponse extends RunnerProtocolResponse { - data: { error: string } - msgName: 'CAPTURE_SCREENSHOT_FAILURE_RESPONSE' -} - -export interface CaptureScreenshotResponse extends RunnerProtocolResponse { - data: { error?: string; image: string } - msgName: 'CAPTURE_SCREENSHOT_RESPONSE' -} - -export interface ControlFailureResponse extends RunnerProtocolResponse { - data: { error: string } - msgName: 'CONTROL_FAILURE_RESPONSE' -} - -export interface ControlResponse extends RunnerProtocolResponse { - data: { error?: string; msg: string } - msgName: 'CONTROL_RESPONSE' -} - -export interface ReadRecordingEndResponse extends RunnerProtocolResponse { - data: { error?: string } - msgName: 'READ_RECORDING_END_RESPONSE' -} - -export interface ReadRecordingFailureResponse extends RunnerProtocolResponse { - data: { error: string } - msgName: 'READ_RECORDING_FAILURE_RESPONSE' -} - -export interface ReadRecordingPartResponse extends RunnerProtocolResponse { - data: { error?: string; video: string } - msgName: 'READ_RECORDING_PART_RESPONSE' -} - -export interface StartRecordingFailureResponse extends RunnerProtocolResponse { - data: { error: string } - msgName: 'START_RECORDING_FAILURE_RESPONSE' -} - -export interface StartRecordingResponse extends RunnerProtocolResponse { - data: { error?: string } - msgName: 'START_RECORDING_RESPONSE' -} - -export interface StopRecordingFailureResponse extends RunnerProtocolResponse { - data: { error: string; } - msgName: 'STOP_RECORDING_FAILURE_RESPONSE' -} - -export interface StopRecordingResponse extends RunnerProtocolResponse { - data: { error?: string } - msgName: 'STOP_RECORDING_RESPONSE' -} - -export interface InteractiveAnnotationFailureResponse { - data: { error?: string; data: string } - msgName: 'START_INTERACTIVE_ANNOTATION_FAILURE_RESPONSE'; -} - -export interface InteractiveAnnotationResponse { - data: { error?: string; data: string } - msgName: 'START_INTERACTIVE_ANNOTATION_RESPONSE'; -} - -export interface GetProcessPidFailureResponse { - data: { error: string } - msgName: 'GET_PROCESS_PID_FAILURE_RESPONSE'; -} - -export interface GetProcessPidResponse { - data: { error?: string; pidNumber: number } - msgName: 'GET_PROCESS_PID_RESPONSE'; -} - -export interface GetStartingArgumentsResponse { - data: { error?: string; arguments: Record } - msgName: 'GET_STARTING_ARGUMENTS_RESPONSE'; -} diff --git a/packages/askui-nodejs/src/execution/agent-os/agent-os-action-not-supported-error.ts b/packages/askui-nodejs/src/execution/agent-os/agent-os-action-not-supported-error.ts new file mode 100644 index 00000000000..2c8b9f85c99 --- /dev/null +++ b/packages/askui-nodejs/src/execution/agent-os/agent-os-action-not-supported-error.ts @@ -0,0 +1,12 @@ +import { AgentOsError } from './agent-os-error'; + +export class AgentOsActionNotSupportedError extends AgentOsError { + constructor(action: string) { + super( + `The action "${action}" is not supported by the AskUI AgentOS. ` + + 'Android automation is temporarily unavailable and will be reintroduced ' + + 'in a future release.', + ); + this.name = 'AgentOsActionNotSupportedError'; + } +} diff --git a/packages/askui-nodejs/src/execution/agent-os/agent-os-client.ts b/packages/askui-nodejs/src/execution/agent-os/agent-os-client.ts new file mode 100644 index 00000000000..22e0a47f1d7 --- /dev/null +++ b/packages/askui-nodejs/src/execution/agent-os/agent-os-client.ts @@ -0,0 +1,527 @@ +import * as grpc from '@grpc/grpc-js'; +import * as protoLoader from '@grpc/proto-loader'; +import { randomUUID } from 'crypto'; +import path from 'path'; +import { Action, ControlCommand, InputEvent } from '../../core/ui-control-commands'; +import { logger } from '../../lib/logger'; +import { getSharpFactory } from '../../utils/base_64_image/sharp'; +import { UiControllerClientConnectionState } from '../ui-controller-client-connection-state'; +import { delay } from '../misc'; +import { AgentOsError } from './agent-os-error'; +import { AgentOsNotConnectedError } from './agent-os-not-connected-error'; +import { AgentOsActionNotSupportedError } from './agent-os-action-not-supported-error'; +import { DeviceClient } from '../device-client'; + +interface RunRecordedActionResponse { + actionID: number; + requiredMilliseconds: number; +} + +interface PollResponse { + pollEventID: string; + pollEventParameters?: { + actionFinished?: { + actionID: number; + }; + }; +} + +interface BitmapMessage { + width: number; + height: number; + lineWidth: number; + bitsPerPixel: number; + bytesPerPixel: number; + data: Buffer; +} + +interface CaptureScreenResponse { + bitmap: BitmapMessage; +} + +interface ActionRequest { + actionClassID: string; + actionParameters: Record; +} + +/** + * Client for the AskUI AgentOS (AskUI Remote Device Controller). Talks gRPC to the + * `Askui.API.TDKv1.ControllerAPI` service, mirroring the connection approach of the + * AskUI Python SDK (askui/vision-agent): open an insecure channel to a locally running + * AgentOS, start a session, start execution, and set the active display. + * + * The AgentOS listens on `localhost:23000` when running standalone and on + * `localhost:26000` when managed by the AskUI OS service (`AskuiCoreService`). + */ +export class AgentOsClient implements DeviceClient { + private static readonly INSTALLATION_DOCS_URL = 'https://docs.askui.com/06-agent-os/installation/service'; + + private static readonly DEFAULT_PORT = '26000'; + + private static readonly REQUEST_TIMEOUT_IN_MS = 30000; + + private static readonly CONNECT_TIMEOUT_IN_MS = 10000; + + private static readonly ACTION_POLL_MAX_RETRIES = 10; + + private static readonly ACTION_POLL_INTERVAL_IN_MS = 50; + + private static readonly TYPING_SPEED_IN_CHARACTERS_PER_SECOND = 50; + + private static readonly MOUSE_MOVE_DURATION_IN_MS = 500; + + private static readonly MOUSE_SCROLL_DURATION_IN_MS = 50; + + private static readonly MODIFIER_KEYS = [ + 'command', + 'alt', + 'control', + 'shift', + 'right_shift', + ]; + + connectionState = UiControllerClientConnectionState.NOT_CONNECTED; + + private client: grpc.Client | undefined; + + private sessionInfo: object | undefined; + + private address: string | undefined; + + private displayId = 1; + + constructor(public url: string) {} + + private static buildControllerApiConstructor(): grpc.ServiceClientConstructor { + const packageDefinition = protoLoader.loadSync( + path.join(__dirname, 'proto', 'Controller_V1.proto'), + { + defaults: true, + enums: String, + keepCase: true, + // 64-bit integers (e.g. the session GUID parts) must round-trip without + // precision loss, so they are represented as strings. + longs: String, + oneofs: true, + }, + ); + const grpcObject = grpc.loadPackageDefinition(packageDefinition); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (grpcObject as any).Askui.API.TDKv1.ControllerAPI; + } + + private static normalizeAddress(url: string): string { + const address = url.replace(/^(https?|wss?):\/\//, ''); + if (address.includes(':')) { + return address; + } + return `${address}:${AgentOsClient.DEFAULT_PORT}`; + } + + private static async openChannel(address: string): Promise { + const ControllerApi = AgentOsClient.buildControllerApiConstructor(); + const client = new ControllerApi(address, grpc.credentials.createInsecure(), { + 'grpc.max_receive_message_length': 2 ** 30, + 'grpc.max_send_message_length': 2 ** 30, + }); + return new Promise((resolve, reject) => { + client.waitForReady( + new Date(Date.now() + AgentOsClient.CONNECT_TIMEOUT_IN_MS), + (error?: Error) => { + if (error) { + client.close(); + reject(error); + return; + } + resolve(client); + }, + ); + }); + } + + private requireClient(): grpc.Client { + if ( + this.client === undefined + || this.connectionState !== UiControllerClientConnectionState.CONNECTED + ) { + throw new AgentOsNotConnectedError(); + } + return this.client; + } + + private static invokeOn( + client: grpc.Client, + method: string, + request: object, + timeoutInMs = AgentOsClient.REQUEST_TIMEOUT_IN_MS, + ): Promise { + logger.debug(`AgentOS request: ${method}`); + return new Promise((resolve, reject) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (client as any)[method]( + request, + { deadline: new Date(Date.now() + timeoutInMs) }, + (error: grpc.ServiceError | null, response: TResponse) => { + if (error) { + reject( + new AgentOsError( + `The AgentOS request "${method}" failed. Cause: ${error.message}`, + ), + ); + return; + } + resolve(response); + }, + ); + }); + } + + private invoke( + method: string, + request: object, + timeoutInMs = AgentOsClient.REQUEST_TIMEOUT_IN_MS, + ): Promise { + const { client } = this; + if (client === undefined) { + throw new AgentOsNotConnectedError(); + } + return AgentOsClient.invokeOn(client, method, request, timeoutInMs); + } + + async connect(): Promise { + this.connectionState = UiControllerClientConnectionState.CONNECTING; + const address = AgentOsClient.normalizeAddress(this.url); + try { + this.client = await AgentOsClient.openChannel(address); + this.address = address; + await this.startSession(); + this.connectionState = UiControllerClientConnectionState.CONNECTED; + logger.debug(`Connected to AgentOS at ${address}`); + return this.connectionState; + } catch (error) { + this.connectionState = UiControllerClientConnectionState.ERROR; + this.client?.close(); + this.client = undefined; + this.address = undefined; + throw new AgentOsError( + `Connection to the AskUI AgentOS at "${address}" cannot be established. ` + + 'Make sure the AskUI AgentOS is installed and running. ' + + `Install it from ${AgentOsClient.INSTALLATION_DOCS_URL}. Cause: ${error}`, + ); + } + } + + private async startSession(): Promise { + const sessionGuid = `{${randomUUID().toUpperCase()}}`; + const response = await this.invoke<{ sessionInfo: object }>('StartSession', { + immediateExecution: true, + sessionGUID: sessionGuid, + }); + this.sessionInfo = response.sessionInfo; + await this.invoke('StartExecution', { sessionInfo: this.sessionInfo }); + await this.invoke('SetActiveDisplay', { displayID: this.displayId }); + } + + disconnect(): void { + const { client, sessionInfo } = this; + if (client === undefined) { + return; + } + this.client = undefined; + this.sessionInfo = undefined; + this.connectionState = UiControllerClientConnectionState.NOT_CONNECTED; + // End the session gracefully in the background, then close the channel. + // Keeping this fire-and-forget preserves the synchronous disconnect() API + // while still releasing the single AgentOS session on the server. + (async () => { + try { + await AgentOsClient.invokeOn(client, 'StopExecution', { sessionInfo }); + await AgentOsClient.invokeOn(client, 'EndSession', { sessionInfo }); + } catch (error) { + logger.debug(`Ending the AgentOS session failed: ${error}`); + } finally { + client.close(); + } + })(); + } + + async setActiveDisplay(displayId: number): Promise { + this.requireClient(); + await this.invoke('SetActiveDisplay', { displayID: displayId }); + this.displayId = displayId; + } + + private async runAction(actionRequest: ActionRequest, timeoutInMs?: number): Promise { + const response = await this.invoke( + 'RunRecordedAction', + { + actionClassID: actionRequest.actionClassID, + actionParameters: actionRequest.actionParameters, + sessionInfo: this.sessionInfo, + }, + timeoutInMs, + ); + await delay(response.requiredMilliseconds); + /* eslint-disable no-await-in-loop */ + for (let retry = 0; retry < AgentOsClient.ACTION_POLL_MAX_RETRIES; retry += 1) { + const pollResponse = await this.invoke('Poll', { + pollEventID: 'PollEventID_ActionFinished', + sessionInfo: this.sessionInfo, + }); + if ( + pollResponse.pollEventParameters?.actionFinished?.actionID === response.actionID + ) { + return; + } + await delay(AgentOsClient.ACTION_POLL_INTERVAL_IN_MS); + } + /* eslint-enable no-await-in-loop */ + throw new AgentOsError( + `The AgentOS did not finish the action "${actionRequest.actionClassID}" in time.`, + ); + } + + async requestControl(controlCommand: ControlCommand): Promise { + this.requireClient(); + /* eslint-disable no-await-in-loop, no-restricted-syntax */ + for (const action of controlCommand.actions) { + for (const actionRequest of AgentOsClient.mapAction(action)) { + await this.runAction(actionRequest); + } + } + /* eslint-enable no-await-in-loop, no-restricted-syntax */ + } + + private static mouseButtonPressAndRelease(button: string, count: number): ActionRequest[] { + return [{ + actionClassID: 'ActionClassID_MouseButton_PressAndRelease', + actionParameters: { + mouseButtonPressAndRelease: { count, mouseButton: button }, + }, + }]; + } + + private static parseKeySequence(text: string): { + keyName: string; + modifierKeyNames: string[]; + } { + const modifierKeyNames: string[] = []; + let remaining = text; + let matched = true; + while (matched) { + matched = false; + /* eslint-disable-next-line no-restricted-syntax */ + for (const modifier of AgentOsClient.MODIFIER_KEYS) { + if ( + remaining.length > modifier.length + 1 + && remaining.startsWith(`${modifier}+`) + ) { + modifierKeyNames.push(modifier); + remaining = remaining.substring(modifier.length + 1); + matched = true; + break; + } + } + } + return { keyName: remaining, modifierKeyNames }; + } + + private static typeText(text: string): ActionRequest[] { + return [{ + actionClassID: 'ActionClassID_KeyboardType_UnicodeText', + actionParameters: { + keyboardTypeUnicodeText: { + text: Buffer.from(text, 'utf16le'), + typingSpeed: AgentOsClient.TYPING_SPEED_IN_CHARACTERS_PER_SECOND, + typingSpeedValue: 'TypingSpeedValue_CharactersPerSecond', + }, + }, + }]; + } + + private static mouseScroll(deltaX: number, deltaY: number): ActionRequest[] { + const actionRequests: ActionRequest[] = []; + if (deltaX !== 0) { + actionRequests.push({ + actionClassID: 'ActionClassID_MouseWheelScroll', + actionParameters: { + mouseWheelScroll: { + delta: deltaX, + deltaType: 'MouseWheelDelta_Raw', + direction: 'MouseWheelScrollDirection_Horizontal', + milliseconds: AgentOsClient.MOUSE_SCROLL_DURATION_IN_MS, + }, + }, + }); + } + if (deltaY !== 0) { + actionRequests.push({ + actionClassID: 'ActionClassID_MouseWheelScroll', + actionParameters: { + mouseWheelScroll: { + delta: deltaY, + deltaType: 'MouseWheelDelta_Raw', + direction: 'MouseWheelScrollDirection_Vertical', + milliseconds: AgentOsClient.MOUSE_SCROLL_DURATION_IN_MS, + }, + }, + }); + } + return actionRequests; + } + + private static keyPressOrRelease(action: Action, press: boolean): ActionRequest[] { + const key = typeof action.parameters['key'] === 'string' ? action.parameters['key'] : action.text; + const modifiers = Array.isArray(action.parameters['modifiers']) + ? action.parameters['modifiers'] as string[] + : []; + if (press) { + return [{ + actionClassID: 'ActionClassID_KeyboardKey_Press', + actionParameters: { + keyboardKeyPress: { keyName: key, modifierKeyNames: modifiers }, + }, + }]; + } + return [{ + actionClassID: 'ActionClassID_KeyboardKey_Release', + actionParameters: { + keyboardKeyRelease: { keyName: key, modifierKeyNames: modifiers }, + }, + }]; + } + + private static mapAction(action: Action): ActionRequest[] { + switch (action.inputEvent) { + case InputEvent.NO_COMMAND: + return []; + case InputEvent.MOUSE_MOVE: + return [{ + actionClassID: 'ActionClassID_MouseMove', + actionParameters: { + mouseMove: { + milliseconds: AgentOsClient.MOUSE_MOVE_DURATION_IN_MS, + position: { x: action.position.x, y: action.position.y }, + }, + }, + }]; + case InputEvent.MOUSE_MOVE_RELATIVELY: + return [{ + actionClassID: 'ActionClassID_MouseMove_Delta', + actionParameters: { + mouseMoveDelta: { + delta: { x: action.position.x, y: action.position.y }, + milliseconds: AgentOsClient.MOUSE_MOVE_DURATION_IN_MS, + }, + }, + }]; + case InputEvent.MOUSE_CLICK_LEFT: + return AgentOsClient.mouseButtonPressAndRelease('MouseButton_Left', 1); + case InputEvent.MOUSE_CLICK_RIGHT: + return AgentOsClient.mouseButtonPressAndRelease('MouseButton_Right', 1); + case InputEvent.MOUSE_CLICK_MIDDLE: + return AgentOsClient.mouseButtonPressAndRelease('MouseButton_Middle', 1); + case InputEvent.MOUSE_CLICK_DOUBLE_LEFT: + return AgentOsClient.mouseButtonPressAndRelease('MouseButton_Left', 2); + case InputEvent.MOUSE_CLICK_DOUBLE_RIGHT: + return AgentOsClient.mouseButtonPressAndRelease('MouseButton_Right', 2); + case InputEvent.MOUSE_CLICK_DOUBLE_MIDDLE: + return AgentOsClient.mouseButtonPressAndRelease('MouseButton_Middle', 2); + case InputEvent.MOUSE_DOWN: + return [{ + actionClassID: 'ActionClassID_MouseButton_Press', + actionParameters: { + mouseButtonPress: { mouseButton: 'MouseButton_Left' }, + }, + }]; + case InputEvent.MOUSE_UP: + return [{ + actionClassID: 'ActionClassID_MouseButton_Release', + actionParameters: { + mouseButtonRelease: { mouseButton: 'MouseButton_Left' }, + }, + }]; + case InputEvent.MOUSE_SCROLL: + return AgentOsClient.mouseScroll(action.position.x, action.position.y); + case InputEvent.TYPE: + case InputEvent.TYPE_TEXT: + return AgentOsClient.typeText(action.text); + case InputEvent.PRESS_KEY_SEQUENCE: { + const { keyName, modifierKeyNames } = AgentOsClient.parseKeySequence(action.text); + return [{ + actionClassID: 'ActionClassID_KeyboardKey_PressAndRelease', + actionParameters: { + keyboardKeyPressAndRelease: { keyName, modifierKeyNames }, + }, + }]; + } + case InputEvent.KEY_PRESS: + return AgentOsClient.keyPressOrRelease(action, true); + case InputEvent.KEY_RELEASE: + return AgentOsClient.keyPressOrRelease(action, false); + case InputEvent.EXECUTE_COMMAND: + return [{ + actionClassID: 'ActionClassID_RunCommand', + actionParameters: { + runcommand: { + command: action.text, + timeoutInMilliseconds: AgentOsClient.REQUEST_TIMEOUT_IN_MS, + }, + }, + }]; + case InputEvent.PRESS_ANDROID_KEY_SEQUENCE: + case InputEvent.PRESS_ANDROID_SINGLE_KEY: + throw new AgentOsActionNotSupportedError(action.inputEvent); + default: + throw new AgentOsError(`Unknown input event "${action.inputEvent}".`); + } + } + + private static bitmapToRgba(bitmap: BitmapMessage): Buffer { + const { + width, height, lineWidth, bytesPerPixel, data, + } = bitmap; + const rgba = Buffer.alloc(width * height * 4); + for (let row = 0; row < height; row += 1) { + for (let column = 0; column < width; column += 1) { + const source = row * lineWidth + column * bytesPerPixel; + const target = (row * width + column) * 4; + // The AgentOS returns the bitmap in BGRA channel order. + rgba[target] = data[source + 2] as number; + rgba[target + 1] = data[source + 1] as number; + rgba[target + 2] = data[source] as number; + rgba[target + 3] = 255; + } + } + return rgba; + } + + async requestScreenshot(): Promise { + this.requireClient(); + const response = await this.invoke('CaptureScreen', { + captureParameters: { displayID: this.displayId }, + sessionInfo: this.sessionInfo, + }); + const { width, height } = response.bitmap; + const sharpFactory = await getSharpFactory(); + const pngBuffer = await sharpFactory(AgentOsClient.bitmapToRgba(response.bitmap), { + raw: { channels: 4, height, width }, + }) + .png() + .toBuffer(); + return `data:image/png;base64,${pngBuffer.toString('base64')}`; + } + + async getStartingArguments(): Promise> { + this.requireClient(); + const [host = 'localhost', port = AgentOsClient.DEFAULT_PORT] = ( + this.address ?? '' + ).split(':'); + return { + displayNum: this.displayId, + host, + port, + runtime: 'desktop', + }; + } +} diff --git a/packages/askui-nodejs/src/execution/agent-os/agent-os-error.ts b/packages/askui-nodejs/src/execution/agent-os/agent-os-error.ts new file mode 100644 index 00000000000..11eccbcfe41 --- /dev/null +++ b/packages/askui-nodejs/src/execution/agent-os/agent-os-error.ts @@ -0,0 +1,6 @@ +export class AgentOsError extends Error { + constructor(message: string) { + super(message); + this.name = 'AgentOsError'; + } +} diff --git a/packages/askui-nodejs/src/execution/agent-os/agent-os-not-connected-error.ts b/packages/askui-nodejs/src/execution/agent-os/agent-os-not-connected-error.ts new file mode 100644 index 00000000000..3354f4a37ea --- /dev/null +++ b/packages/askui-nodejs/src/execution/agent-os/agent-os-not-connected-error.ts @@ -0,0 +1,11 @@ +import { AgentOsError } from './agent-os-error'; + +export class AgentOsNotConnectedError extends AgentOsError { + constructor() { + super( + 'There is no connection to the AskUI AgentOS. ' + + 'Make sure to call connect() before sending commands.', + ); + this.name = 'AgentOsNotConnectedError'; + } +} diff --git a/packages/askui-nodejs/src/execution/agent-os/proto/Controller_V1.proto b/packages/askui-nodejs/src/execution/agent-os/proto/Controller_V1.proto new file mode 100644 index 00000000000..e8adc62ac43 --- /dev/null +++ b/packages/askui-nodejs/src/execution/agent-os/proto/Controller_V1.proto @@ -0,0 +1,539 @@ +syntax = "proto3"; + +package Askui.API.TDKv1; + +service ControllerAPI { + // General + rpc StartSession (Request_StartSession) returns (Response_StartSession) {} + rpc EndSession (Request_EndSession) returns (Response_Void) {} + + rpc Send (Request_Send) returns (Response_Send) {} + rpc Poll (Request_Poll) returns (Response_Poll) {} + + rpc StartExecution (Request_StartExecution) returns (Response_Void) {} + rpc StopExecution (Request_StopExecution) returns (Response_Void) {} + + // Run action and record it + rpc RunRecordedAction (Request_RunRecordedAction) returns (Response_RunRecordedAction) {} + + // Schedule an action + rpc ScheduleBatchedAction (Request_ScheduleBatchedAction) returns (Response_ScheduleBatchedAction) {} + + // Start and stop batched execution + rpc StartBatchRun (Request_StartBatchRun) returns (Response_Void) {} + rpc StopBatchRun(Request_StopBatchRun) returns (Response_Void) {} + + // Recorded or batched actions access + rpc GetActionCount (Request_GetActionCount) returns (Response_GetActionCount) {} + rpc GetAction (Request_GetAction) returns (Response_GetAction) {} + + // Modify actions batch + rpc RemoveAction (Request_RemoveAction) returns (Response_Void) {} + + // Clear all batched or recorded actions + rpc RemoveAllActions (Request_RemoveAllActions) returns (Response_Void) {} + + // Capturing + rpc CaptureScreen (Request_CaptureScreen) returns (Response_CaptureScreen) {} + + // Configuration + rpc SetTestConfiguration (Reuqest_SetTestConfiguration) returns (Response_Void) {} + rpc SetMouseDelay (Request_SetMouseDelay) returns (Response_Void) {} // TODO: TDKv2 should use push and pop instead. + rpc SetKeyboardDelay (Request_SetKeyboardDelay) returns (Response_Void) {} // TODO: TDKv2 should use push and pop instead. + + // Device Information + rpc GetDisplayInformation (Request_Void) returns (Response_GetDisplayInformation) {} + rpc GetMousePosition (Request_Void) returns (Response_GetMousePosition) {} + rpc GetProcessList (Request_GetProcessList) returns (Response_GetProcessList) {} + rpc GetWindowList (Request_GetWindowList) returns (Response_GetWindowList) {} + rpc GetAutomationTargetList (Request_Void) returns (Response_GetAutomationTargetList) {} + + // Device Configuration + rpc SetActiveDisplay (Request_SetActiveDisplay) returns (Response_Void) {} + rpc SetActiveWindow (Request_SetActiveWindow) returns (Response_Void) {} + rpc SetActiveAutomationTarget (Request_SetActiveAutomationTarget) returns (Response_Void) {} + + // Deprecated Utilities + rpc GetColor (Request_GetColor) returns (Response_GetColor) {} + rpc GetPixelColor (Request_GetPixelColor) returns (Response_GetPixelColor) {} + + rpc SetDisplayLabel(Request_SetDisplayLabel) returns (Response_Void) {} // TODO: Not yet implemented, supposded to be deleted if not needed. +} + +message Void {} + +message Request_Void {} +message Response_Void {} + +message Size2 { + uint32 width = 1; + uint32 height = 2; +} + +message Delta2 { + int32 x = 1; + int32 y = 2; +} + +message Coordinate2 { + int32 x = 1; + int32 y = 2; +} + +message Rectangle { + int32 left = 1; + int32 top = 2; + int32 right = 3; + int32 bottom = 4; +} + +message Bitmap { + uint32 width = 1; + uint32 height = 2; + uint32 lineWidth = 3; + uint32 bitsPerPixel = 4; + uint32 bytesPerPixel = 5; + bytes data = 6; +} + +message Color { + uint32 r = 1; + uint32 g = 2; + uint32 b = 3; +} + +message GUID { + uint64 highPart = 1; + uint64 lowPart = 2; +} + +message SessionInfo { + GUID sessionGUID = 1; + uint64 sessionID = 2; +} + +message CaptureArea { + Size2 size = 3; + Coordinate2 coordinate = 2; +} + +// Usage +// +// displayID: +// Value from 'DisplayInformation::displayID', use '0' if screenshots shall be taked from virtual sreen. +// +// captureArea: +// Defines either the area to capture on virtual screen or of the display if 'displayID'. +message CaptureParameters { + optional uint32 displayID = 1; + optional CaptureArea captureArea = 2; +} + +// +// Poll +enum PollEventID { + reserved 1; + + PollEventID_Undefined = 0; + //Any = 1; + PollEventID_ActionFinished = 2; +} + +message PollEventParameters_ActionFinished { + uint32 actionID = 1; +} + +message PollEventParameters { + oneof dataOf { + PollEventParameters_ActionFinished actionFinished = 1; + } +} + +// +// +enum MouseButton { + MouseButton_Undefined = 0; + + MouseButton_Left = 1; + MouseButton_Right = 2; + MouseButton_Middle = 3; +} + +// +// Action Parameters +enum ActionClassID { + ActionClassID_Undefined = 0; + + ActionClassID_Wait = 1; + + ActionClassID_MouseButton_Press = 8; + ActionClassID_MouseButton_Release = 9; + ActionClassID_MouseButton_PressAndRelease = 10; + ActionClassID_MouseWheelScroll = 11; + ActionClassID_MouseMove = 12; + ActionClassID_MouseMove_Delta = 13; + + ActionClassID_KeyboardKey_Press = 14; + ActionClassID_KeyboardKey_Release = 15; + ActionClassID_KeyboardKey_PressAndRelease = 16; + ActionClassID_KeyboardKeys_Press = 17; + ActionClassID_KeyboardKeys_Release = 18; + ActionClassID_KeyboardKeys_PressAndRelease = 19; + + ActionClassID_KeyboardType_Text = 20; + ActionClassID_KeyboardType_UnicodeText = 21; + + ActionClassID_RunCommand = 22; +} + +message ActionParameters_Wait { + uint32 milliseconds = 1; +} + +message ActionParameters_MouseButton_Press { + MouseButton mouseButton = 1; +} + +message ActionParameters_MouseButton_Release { + MouseButton mouseButton = 1; +} + +message ActionParameters_MouseButton_PressAndRelease { + MouseButton mouseButton = 1; + uint32 count = 2; +} + +enum MouseWheelDeltaType { + MouseWheelDelta_Undefined = 0; + MouseWheelDelta_Raw = 1; + MouseWheelDelta_Detent = 2; +} + +enum MouseWheelScrollDirection { + MouseWheelScrollDirection_Undefined = 0; + MouseWheelScrollDirection_Vertical = 1; + MouseWheelScrollDirection_Horizontal = 2; +} + +message ActionParameters_MouseWheelScroll { + MouseWheelScrollDirection direction = 1; + MouseWheelDeltaType deltaType = 2; + int32 delta = 3; + int32 milliseconds = 4; +} + +message ActionParameters_MouseMove { + Coordinate2 position = 1; + optional uint32 milliseconds = 2; +} + +message ActionParameters_MouseMove_Delta { + Delta2 delta = 1; + optional uint32 milliseconds = 2; +} + +message ActionParameters_KeyboardKey_Press { + string keyName = 1; + repeated string modifierKeyNames = 2; +} + +message ActionParameters_KeyboardKey_Release { + string keyName = 1; + repeated string modifierKeyNames = 2; +} + +message ActionParameters_KeyboardKey_PressAndRelease { + string keyName = 1; + repeated string modifierKeyNames = 2; +} + +message ActionParameters_KeyboardKeys_Press { + repeated string keyNames = 1; + repeated string modifierKeyNames = 2; +} + +message ActionParameters_KeyboardKeys_Release { + repeated string keyNames = 1; + repeated string modifierKeyNames = 2; +} + +message ActionParameters_KeyboardKeys_PressAndRelease { + repeated string keyNames = 1; + repeated string modifierKeyNames = 2; +} + +enum TypingSpeedValue { + TypingSpeedValue_Undefined = 0; + TypingSpeedValue_CharactersPerSecond = 1; + TypingSpeedValue_Seconds = 2; +} + +message ActionParameters_KeyboardType_Text { + string text = 1; + TypingSpeedValue typingSpeedValue = 2; + optional uint32 typingSpeed = 3; +} + +message ActionParameters_KeyboardType_UnicodeText { + bytes text = 1; + TypingSpeedValue typingSpeedValue = 2; + optional uint32 typingSpeed = 3; +} + +message ActionParameters_RunCommand { + string command = 1; + optional uint32 timeoutInMilliseconds = 2; +} + +message ActionParameters { + oneof dataOf { + Void none = 1; + ActionParameters_Wait wait = 2; + ActionParameters_MouseButton_Press mouseButtonPress = 3; + ActionParameters_MouseButton_Release mouseButtonRelease = 4; + ActionParameters_MouseButton_PressAndRelease mouseButtonPressAndRelease = 5; + ActionParameters_MouseWheelScroll mouseWheelScroll = 6; + ActionParameters_MouseMove mouseMove = 7; + ActionParameters_MouseMove_Delta mouseMoveDelta = 8; + ActionParameters_KeyboardKey_Press keyboardKeyPress = 9; + ActionParameters_KeyboardKey_Release keyboardKeyRelease = 10; + ActionParameters_KeyboardKey_PressAndRelease keyboardKeyPressAndRelease = 11; + ActionParameters_KeyboardKeys_Press keyboardKeysPress = 12; + ActionParameters_KeyboardKeys_Release keyboardKeysRelease = 13; + ActionParameters_KeyboardKeys_PressAndRelease keyboardKeysPressAndRelease = 14; + ActionParameters_KeyboardType_Text keyboardTypeText = 15; + ActionParameters_KeyboardType_UnicodeText keyboardTypeUnicodeText = 16; + ActionParameters_RunCommand runcommand = 17; + } +} + +// +// Request and Response Messages +message Request_StartSession { + string sessionGUID = 1; // "{62F7CE3F-D73B-4F96-826F-43D767DAB5C1}" + bool immediateExecution = 2; // true +} + +message Response_StartSession { + SessionInfo sessionInfo = 1; +} + +message Request_EndSession { + SessionInfo sessionInfo = 1; +} + +message Request_Send { + string message = 1; +} + +message Request_Poll { + SessionInfo sessionInfo = 1; + PollEventID pollEventID = 2; +} + +message Request_StartExecution { + SessionInfo sessionInfo = 1; +} + +message Request_StopExecution { + SessionInfo sessionInfo = 1; +} + +message Response_Send { + string message = 1; +} + +message Response_Poll { + PollEventID pollEventID = 1; + PollEventParameters pollEventParameters = 2; +} + +message Request_RunRecordedAction { + SessionInfo sessionInfo = 1; + ActionClassID actionClassID = 2; + ActionParameters actionParameters = 3; +} + +message Response_RunRecordedAction { + uint32 actionID = 1; + uint32 requiredMilliseconds = 2; +} + +message Request_ScheduleBatchedAction { + SessionInfo sessionInfo = 1; + ActionClassID actionClassID = 2; + ActionParameters actionParameters = 3; +} + +message Response_ScheduleBatchedAction { + uint32 actionID = 1; +} + +message Request_GetActionCount { + SessionInfo sessionInfo = 1; +} + +message Response_GetActionCount { + uint32 actionCount = 1; +} + +message Request_GetAction { + SessionInfo sessionInfo = 1; + uint32 actionIndex = 2; +} + +message Response_GetAction { + uint32 actionID = 1; + ActionClassID actionClassID = 2; + ActionParameters actionParameters = 3; +} + +message Request_RemoveAction { + SessionInfo sessionInfo = 1; + uint32 actionID = 2; +} + +message Request_RemoveAllActions { + SessionInfo sessionInfo = 1; +} + +message Request_StartBatchRun { + SessionInfo sessionInfo = 1; +} + +message Request_StopBatchRun { + SessionInfo sessionInfo = 1; +} + +message Request_CaptureScreen { + SessionInfo sessionInfo = 1; + optional CaptureParameters captureParameters = 2; +} + +message Response_CaptureScreen { + Bitmap bitmap = 1; +} + +message Response_GetContinuousCapturedScreen { + Bitmap bitmap = 1; +} + +message Reuqest_SetTestConfiguration { + SessionInfo sessionInfo = 1; + CaptureParameters defaultCaptureParameters = 2; + uint32 mouseDelayInMilliseconds = 3; + uint32 keyboardDelayInMilliseconds = 4; +} + +message Request_SetMouseDelay { + SessionInfo sessionInfo = 1; + uint32 delayInMilliseconds = 2; +} + +message Request_SetKeyboardDelay { + SessionInfo sessionInfo = 1; + uint32 delayInMilliseconds = 2; +} + +message DisplayInformation { + uint32 displayID = 1; + string name = 2; + Size2 sizeInPixels = 3; + Rectangle virtualScreenRectangle = 4; +} + +message Response_GetDisplayInformation { + repeated DisplayInformation displays = 1; + Rectangle virtualScreenRectangle = 2; +} + +message Response_GetMousePosition { + int32 x = 1; + int32 y = 2; +} + +message ProcessInfoExtended { + bool hasWindow = 1; +} + +message ProcessInfo { + uint64 ID = 1; + string name = 2; + optional ProcessInfoExtended extendedInfo = 3; +} + +message Request_GetProcessList { + bool getExtendedInfo = 1; +} + +message Response_GetProcessList { + repeated ProcessInfo processes = 1; +} + +message Request_GetWindowList { + uint64 processID = 1; +} + +message WindowInfo { + uint64 ID = 1; + string name = 2; +} + +message Response_GetWindowList { + repeated WindowInfo windows = 1; +} + +message Request_SetActiveDisplay { + uint32 displayID = 1; +} + +message Request_SetActiveWindow { + uint64 processID = 1; + uint64 windowID = 2; +} + +enum AutomationTargetType { + AutomationTarget_Local = 0; + AutomationTarget_Background = 1; + AutomationTarget_Companion = 2; +} + +message AutomationTarget { + uint64 ID = 1; + AutomationTargetType type = 2; + string name = 3; + bool active = 4; +} + +message Response_GetAutomationTargetList { + repeated AutomationTarget targets = 1; +} + +message Request_SetActiveAutomationTarget { + uint64 ID = 1; +} + +message Request_GetColor { + int32 x = 1; + int32 y = 2; + Bitmap bitmap = 3; +} + +message Response_GetColor { + Color color = 1; +} + +message Request_GetPixelColor { + int32 x = 1; + int32 y = 2; +} + +message Response_GetPixelColor { + Color color = 1; +} + +message Request_SetDisplayLabel { + SessionInfo sessionInfo = 1; + uint32 displayID = 2; + string label = 3; +} diff --git a/packages/askui-nodejs/src/execution/android/android-adb-client.ts b/packages/askui-nodejs/src/execution/android/android-adb-client.ts new file mode 100644 index 00000000000..0eea8ab6110 --- /dev/null +++ b/packages/askui-nodejs/src/execution/android/android-adb-client.ts @@ -0,0 +1,270 @@ +import { execFile } from 'child_process'; +import { Action, ControlCommand, InputEvent } from '../../core/ui-control-commands'; +import { logger } from '../../lib/logger'; +import { DeviceClient } from '../device-client'; +import { UiControllerClientConnectionState } from '../ui-controller-client-connection-state'; +import { AndroidError } from './android-error'; +import { AndroidNotConnectedError } from './android-not-connected-error'; +import { NoAndroidDeviceError } from './no-android-device-error'; + +export type AndroidKeyboardMode = 'input-text' | 'adb-keyboard'; + +export interface AndroidAdbClientArgs { + /** Select a specific device by its adb serial. Defaults to the first device. */ + readonly id?: string; + /** Path to the `adb` executable. Defaults to `adb` (must be on PATH). */ + readonly adbPath?: string; + /** Delay applied after each action, in milliseconds. Defaults to `0`. */ + readonly actionDelayInMs?: number; + /** + * How text is typed. `input-text` uses `adb shell input text` (ASCII, no + * setup). `adb-keyboard` uses the ADBKeyboard IME broadcast for full Unicode + * support (requires ADBKeyboard installed and set as the default IME). + * Defaults to `input-text`. + */ + readonly keyboard?: AndroidKeyboardMode; +} + +/** + * Drives an Android device directly via `adb`, implementing {@link DeviceClient} + * so it is interchangeable with the desktop AgentOS gRPC client. Replaces the + * legacy UI Controller's WebSocket/adbkit path: every operation is a plain + * `adb` invocation, so no separate controller process or binary is required. + */ +export class AndroidAdbClient implements DeviceClient { + private static readonly EXEC_MAX_BUFFER = 256 * 1024 * 1024; + + connectionState = UiControllerClientConnectionState.NOT_CONNECTED; + + private readonly adbPath: string; + + private readonly keyboard: AndroidKeyboardMode; + + private readonly actionDelayInMs: number; + + private deviceId: string | undefined; + + private screenWidth = 0; + + private screenHeight = 0; + + private mousePosition = { x: 0, y: 0 }; + + constructor(private args: AndroidAdbClientArgs = {}) { + this.adbPath = args.adbPath ?? 'adb'; + this.keyboard = args.keyboard ?? 'input-text'; + this.actionDelayInMs = args.actionDelayInMs ?? 0; + this.deviceId = args.id; + } + + private exec(args: string[]): Promise { + return new Promise((resolve, reject) => { + execFile( + this.adbPath, + args, + { encoding: 'buffer', maxBuffer: AndroidAdbClient.EXEC_MAX_BUFFER }, + (error, stdout, stderr) => { + if (error) { + const details = stderr?.toString().trim() || error.message; + reject(new AndroidError(`adb ${args.join(' ')} failed: ${details}`)); + return; + } + resolve(stdout); + }, + ); + }); + } + + private async shell(command: string): Promise { + if (this.deviceId === undefined) { + throw new AndroidNotConnectedError(); + } + const output = await this.exec(['-s', this.deviceId, 'shell', command]); + return output.toString().trim(); + } + + private async listDevices(): Promise { + const output = (await this.exec(['devices'])).toString(); + return output + .split('\n') + .slice(1) + .map((line) => line.trim()) + .filter((line) => line.endsWith('\tdevice')) + .map((line) => line.split('\t')[0] as string); + } + + private async resolveDeviceId(): Promise { + const devices = await this.listDevices(); + if (this.args.id !== undefined) { + if (!devices.includes(this.args.id)) { + throw new NoAndroidDeviceError(this.args.id); + } + return this.args.id; + } + if (devices.length === 0) { + throw new NoAndroidDeviceError(); + } + return devices[0] as string; + } + + private async updateScreenSize(): Promise { + const output = await this.shell('wm size'); + // Prefer the override size ("Override size: WxH") if present, else physical. + const matches = [...output.matchAll(/(?:Physical|Override) size:\s*(\d+)x(\d+)/g)]; + const last = matches[matches.length - 1]; + if (last) { + this.screenWidth = Number(last[1]); + this.screenHeight = Number(last[2]); + } + } + + async connect(): Promise { + this.connectionState = UiControllerClientConnectionState.CONNECTING; + try { + this.deviceId = await this.resolveDeviceId(); + await this.updateScreenSize(); + this.connectionState = UiControllerClientConnectionState.CONNECTED; + logger.debug(`Connected to Android device ${this.deviceId} (${this.screenWidth}x${this.screenHeight})`); + return this.connectionState; + } catch (error) { + this.connectionState = UiControllerClientConnectionState.ERROR; + this.deviceId = this.args.id; + if (error instanceof AndroidError) { + throw error; + } + throw new AndroidError( + `Connection to an Android device via adb failed. Make sure "${this.adbPath}" ` + + `is installed and a device is connected. Cause: ${error}`, + ); + } + } + + disconnect(): void { + this.deviceId = this.args.id; + this.connectionState = UiControllerClientConnectionState.NOT_CONNECTED; + } + + // eslint-disable-next-line class-methods-use-this + async setActiveDisplay(): Promise { + // Android targets a single device/display selected at connect time. + } + + private requireConnected(): void { + if ( + this.deviceId === undefined + || this.connectionState !== UiControllerClientConnectionState.CONNECTED + ) { + throw new AndroidNotConnectedError(); + } + } + + private delay(): Promise { + if (this.actionDelayInMs <= 0) { + return Promise.resolve(); + } + return new Promise((resolve) => { + setTimeout(resolve, this.actionDelayInMs); + }); + } + + async requestControl(controlCommand: ControlCommand): Promise { + this.requireConnected(); + /* eslint-disable no-await-in-loop, no-restricted-syntax */ + for (const action of controlCommand.actions) { + await this.runAction(action); + await this.delay(); + } + /* eslint-enable no-await-in-loop, no-restricted-syntax */ + } + + private async runAction(action: Action): Promise { + switch (action.inputEvent) { + case InputEvent.NO_COMMAND: + break; + case InputEvent.MOUSE_MOVE: + this.mousePosition = { x: action.position.x, y: action.position.y }; + break; + case InputEvent.MOUSE_MOVE_RELATIVELY: + this.mousePosition = { + x: this.mousePosition.x + action.position.x, + y: this.mousePosition.y + action.position.y, + }; + break; + case InputEvent.MOUSE_DOWN: + await this.shell(`input motionevent DOWN ${this.mousePosition.x} ${this.mousePosition.y}`); + break; + case InputEvent.MOUSE_UP: + await this.shell(`input motionevent UP ${this.mousePosition.x} ${this.mousePosition.y}`); + break; + case InputEvent.MOUSE_CLICK_LEFT: + case InputEvent.MOUSE_CLICK_RIGHT: + case InputEvent.MOUSE_CLICK_MIDDLE: + await this.shell(`input tap ${this.mousePosition.x} ${this.mousePosition.y}`); + break; + case InputEvent.MOUSE_CLICK_DOUBLE_LEFT: + case InputEvent.MOUSE_CLICK_DOUBLE_RIGHT: + case InputEvent.MOUSE_CLICK_DOUBLE_MIDDLE: + await this.shell(`input tap ${this.mousePosition.x} ${this.mousePosition.y}`); + await this.shell(`input tap ${this.mousePosition.x} ${this.mousePosition.y}`); + break; + case InputEvent.MOUSE_SCROLL: + // Negated to match the desktop scroll orientation. + await this.shell(`input roll ${-action.position.x} ${-action.position.y}`); + break; + case InputEvent.TYPE: + await this.typeText(action.text); + break; + case InputEvent.TYPE_TEXT: + await this.shell(`input tap ${action.position.x} ${action.position.y}`); + await this.typeText(action.text); + break; + case InputEvent.PRESS_ANDROID_SINGLE_KEY: + await this.shell(`input keyevent ${action.text.toUpperCase()}`); + break; + case InputEvent.PRESS_ANDROID_KEY_SEQUENCE: { + // Press the keys one after another. `+` and whitespace both separate keys. + const keys = action.text.toUpperCase().split(/[\s+]+/).filter(Boolean); + await this.shell(`input keyevent ${keys.join(' ')}`); + break; + } + case InputEvent.EXECUTE_COMMAND: + await this.shell(action.text); + break; + default: + throw new AndroidError( + `The action "${action.inputEvent}" is not supported on Android.`, + ); + } + } + + private async typeText(text: string): Promise { + if (this.keyboard === 'adb-keyboard') { + // ADBKeyboard IME broadcast (base64-encoded UTF-8) supports full Unicode. + const encoded = Buffer.from(text, 'utf-8').toString('base64'); + await this.shell(`am broadcast -a ADB_INPUT_B64 --es msg ${encoded}`); + return; + } + // `input text` treats spaces specially and does not support Unicode. + const escaped = text + .replace(/(["`$\\])/g, '\\$1') + .replace(/ /g, '%s'); + await this.shell(`input text "${escaped}"`); + } + + async requestScreenshot(): Promise { + this.requireConnected(); + const png = await this.exec(['-s', this.deviceId as string, 'exec-out', 'screencap', '-p']); + return `data:image/png;base64,${png.toString('base64')}`; + } + + async getStartingArguments(): Promise> { + this.requireConnected(); + return { + deviceId: this.deviceId as string, + displayNum: 0, + height: this.screenHeight, + runtime: 'android', + width: this.screenWidth, + }; + } +} diff --git a/packages/askui-nodejs/src/execution/android/android-error.ts b/packages/askui-nodejs/src/execution/android/android-error.ts new file mode 100644 index 00000000000..a2d2ad85f67 --- /dev/null +++ b/packages/askui-nodejs/src/execution/android/android-error.ts @@ -0,0 +1,6 @@ +export class AndroidError extends Error { + constructor(message: string) { + super(message); + this.name = 'AndroidError'; + } +} diff --git a/packages/askui-nodejs/src/execution/android/android-not-connected-error.ts b/packages/askui-nodejs/src/execution/android/android-not-connected-error.ts new file mode 100644 index 00000000000..e99c15f60df --- /dev/null +++ b/packages/askui-nodejs/src/execution/android/android-not-connected-error.ts @@ -0,0 +1,11 @@ +import { AndroidError } from './android-error'; + +export class AndroidNotConnectedError extends AndroidError { + constructor() { + super( + 'There is no connection to an Android device. ' + + 'Make sure to call connect() before sending commands.', + ); + this.name = 'AndroidNotConnectedError'; + } +} diff --git a/packages/askui-nodejs/src/execution/android/no-android-device-error.ts b/packages/askui-nodejs/src/execution/android/no-android-device-error.ts new file mode 100644 index 00000000000..6a929c76381 --- /dev/null +++ b/packages/askui-nodejs/src/execution/android/no-android-device-error.ts @@ -0,0 +1,14 @@ +import { AndroidError } from './android-error'; + +export class NoAndroidDeviceError extends AndroidError { + constructor(deviceId?: string) { + super( + deviceId !== undefined + ? `The Android device "${deviceId}" was not found. Make sure it is connected ` + + '(check "adb devices") and authorized.' + : 'No Android device was found. Make sure a device or emulator is connected ' + + 'and authorized (check "adb devices").', + ); + this.name = 'NoAndroidDeviceError'; + } +} diff --git a/packages/askui-nodejs/src/execution/device-client.ts b/packages/askui-nodejs/src/execution/device-client.ts new file mode 100644 index 00000000000..00c73c384ec --- /dev/null +++ b/packages/askui-nodejs/src/execution/device-client.ts @@ -0,0 +1,24 @@ +import { ControlCommand } from '../core/ui-control-commands'; +import { UiControllerClientConnectionState } from './ui-controller-client-connection-state'; + +/** + * Transport-agnostic client for a device that AskUI automates (a desktop via the + * AgentOS gRPC service, or an Android device via the legacy UI Controller). This + * is the seam {@link ExecutionRuntime} depends on so the transport can be chosen + * per target without the execution/agent layers knowing which one is in use. + */ +export interface DeviceClient { + connectionState: UiControllerClientConnectionState; + + connect(): Promise; + + disconnect(): void; + + requestControl(controlCommand: ControlCommand): Promise; + + requestScreenshot(): Promise; + + getStartingArguments(): Promise>; + + setActiveDisplay(displayId: number): Promise; +} diff --git a/packages/askui-nodejs/src/execution/execution-runtime.ts b/packages/askui-nodejs/src/execution/execution-runtime.ts index 1f8e3c29c55..c2e0e69c679 100644 --- a/packages/askui-nodejs/src/execution/execution-runtime.ts +++ b/packages/askui-nodejs/src/execution/execution-runtime.ts @@ -1,7 +1,7 @@ import { BetaMessage, BetaMessageParam } from '@anthropic-ai/sdk/resources/beta/messages'; import { ControlCommand, ControlCommandCode } from '../core/ui-control-commands'; import { CustomElement } from '../core/model/custom-element'; -import { UiControllerClient } from './ui-controller-client'; +import { DeviceClient } from './device-client'; import { RepeatError } from './repeat-error'; import { delay } from './misc'; import { InferenceClient } from './inference-client'; @@ -18,7 +18,7 @@ import { ModelCompositionBranch } from './model-composition-branch'; export class ExecutionRuntime { constructor( - private uiControllerClient: UiControllerClient, + private deviceClient: DeviceClient, private inferenceClient: InferenceClient, private stepReporter: StepReporter, public retryStrategy: RetryStrategy, @@ -26,31 +26,18 @@ export class ExecutionRuntime { async connect(): Promise { this.inferenceClient.cacheManager.loadFromFile(); - return this.uiControllerClient.connect(); + return this.deviceClient.connect(); } disconnect(): void { this.inferenceClient.cacheManager.saveToFile(); - this.uiControllerClient.disconnect(); - } - - async startVideoRecording(): Promise { - await this.uiControllerClient.startVideoRecording(); - } - - async stopVideoRecording(): Promise { - await this.uiControllerClient.stopVideoRecording(); - } - - async readVideoRecording(): Promise { - const response = await this.uiControllerClient.readVideoRecording(); - return response.data.video; + this.deviceClient.disconnect(); } async requestControl( controlCommand: ControlCommand, ): Promise { - await this.uiControllerClient.requestControl(controlCommand); + await this.deviceClient.requestControl(controlCommand); } async executeInstruction( @@ -154,8 +141,7 @@ export class ExecutionRuntime { } async getScreenshot(): Promise { - const requestScreenshotResponse = await this.uiControllerClient.requestScreenshot(); - return requestScreenshotResponse.data.image; + return this.deviceClient.requestScreenshot(); } private async buildSnapshot(instruction: Instruction): Promise { @@ -195,29 +181,15 @@ export class ExecutionRuntime { return controlCommand; } - async annotateInteractively() { - const annotationResponse = await this.annotateImage(); - await this.uiControllerClient.annotateInteractively( - annotationResponse.detected_elements, - annotationResponse.image, - ); - } - async takeScreenshotIfImageisNotProvided(imagePath?: string): Promise { - let base64Image = ''; if (imagePath !== undefined) { - base64Image = (await Base64Image.fromPath(imagePath)).toString(); - } - if (imagePath === undefined) { - const screenshotResponse = await this.uiControllerClient.requestScreenshot(); - base64Image = screenshotResponse.data.image; + return (await Base64Image.fromPath(imagePath)).toString(); } - return base64Image; + return this.deviceClient.requestScreenshot(); } async getStartingArguments(): Promise> { - const startingArgumentsResponse = await this.uiControllerClient.getStartingArguments(); - return startingArgumentsResponse.data.arguments; + return this.deviceClient.getStartingArguments(); } async getDetectedElements( diff --git a/packages/askui-nodejs/src/execution/legacy-controller/legacy-android-client.ts b/packages/askui-nodejs/src/execution/legacy-controller/legacy-android-client.ts new file mode 100644 index 00000000000..94e91104415 --- /dev/null +++ b/packages/askui-nodejs/src/execution/legacy-controller/legacy-android-client.ts @@ -0,0 +1,161 @@ +import WebSocket from 'ws'; +import { ControlCommand } from '../../core/ui-control-commands'; +import { logger } from '../../lib/logger'; +import { DeviceClient } from '../device-client'; +import { UiControllerClientConnectionState } from '../ui-controller-client-connection-state'; +import { LegacyControllerError } from './legacy-controller-error'; +import { LegacyControllerNotConnectedError } from './legacy-controller-not-connected-error'; +import { RunnerProtocolRequest, RunnerProtocolResponse } from './runner-protocol'; + +export interface LegacyAndroidClientArgs { + /** WebSocket address of the legacy UI Controller. Defaults to `ws://127.0.0.1:6769`. */ + readonly controllerUrl?: string; +} + +/** + * Drives an Android device through the AskUI legacy UI Controller over its + * WebSocket "runner protocol" (default `ws://127.0.0.1:6769`), implementing + * {@link DeviceClient} so it is interchangeable with the desktop AgentOS gRPC + * client and the direct-adb Android client. + * + * The controller must already be running in Android mode + * (`AskUI-StartController ... -r android`). The controller serves one command at + * a time, so requests are serialized. + */ +export class LegacyAndroidClient implements DeviceClient { + private static readonly DEFAULT_URL = 'ws://127.0.0.1:6769'; + + private static readonly REQUEST_TIMEOUT_IN_MS = 30000; + + private static readonly CONNECT_TIMEOUT_IN_MS = 10000; + + connectionState = UiControllerClientConnectionState.NOT_CONNECTED; + + private readonly url: string; + + private ws: WebSocket | undefined; + + private pending: { + resolve: (response: RunnerProtocolResponse) => void; + reject: (reason: unknown) => void; + timeout: NodeJS.Timeout; + } | undefined; + + constructor(args: LegacyAndroidClientArgs = {}) { + this.url = args.controllerUrl ?? LegacyAndroidClient.DEFAULT_URL; + } + + connect(): Promise { + this.connectionState = UiControllerClientConnectionState.CONNECTING; + return new Promise((resolve, reject) => { + const ws = new WebSocket(this.url); + const timeout = setTimeout(() => { + ws.terminate(); + reject( + new LegacyControllerError( + `Connection to the AskUI legacy UI Controller at "${this.url}" timed out. ` + + 'Make sure it is running in Android mode (AskUI-StartController ... -r android).', + ), + ); + }, LegacyAndroidClient.CONNECT_TIMEOUT_IN_MS); + + ws.on('open', () => { + clearTimeout(timeout); + this.ws = ws; + this.connectionState = UiControllerClientConnectionState.CONNECTED; + logger.debug(`Connected to legacy UI Controller at ${this.url}`); + resolve(this.connectionState); + }); + ws.on('message', (data: WebSocket.RawData) => this.onMessage(data)); + ws.on('error', (error: Error) => { + clearTimeout(timeout); + this.connectionState = UiControllerClientConnectionState.ERROR; + reject( + new LegacyControllerError( + `Connection to the AskUI legacy UI Controller at "${this.url}" cannot be established. ` + + 'Make sure it is running in Android mode (AskUI-StartController ... -r android). ' + + `Cause: ${error.message}`, + ), + ); + }); + }); + } + + private onMessage(data: WebSocket.RawData): void { + if (this.pending === undefined) { + return; + } + const { resolve, reject, timeout } = this.pending; + clearTimeout(timeout); + this.pending = undefined; + let response: RunnerProtocolResponse; + try { + response = JSON.parse(data.toString()); + } catch (error) { + reject(new LegacyControllerError(`Malformed response from the legacy UI Controller: ${error}`)); + return; + } + if (response.data?.error) { + reject(new LegacyControllerError(response.data.error)); + return; + } + resolve(response); + } + + disconnect(): void { + this.ws?.close(); + this.ws = undefined; + this.connectionState = UiControllerClientConnectionState.NOT_CONNECTED; + } + + private sendAndReceive(request: RunnerProtocolRequest): Promise { + const { ws } = this; + if (ws === undefined || this.connectionState !== UiControllerClientConnectionState.CONNECTED) { + throw new LegacyControllerNotConnectedError(); + } + if (this.pending !== undefined) { + throw new LegacyControllerError( + 'A request to the legacy UI Controller is already in flight. ' + + 'Requests must be serialized (check for a missing await).', + ); + } + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + this.pending = undefined; + reject( + new LegacyControllerError( + `Request "${request.msgName}" to the legacy UI Controller timed out.`, + ), + ); + }, LegacyAndroidClient.REQUEST_TIMEOUT_IN_MS); + this.pending = { reject, resolve, timeout }; + logger.debug(`Send to legacy UI Controller: ${request.msgName}`); + ws.send(JSON.stringify(request)); + }); + } + + async requestControl(controlCommand: ControlCommand): Promise { + await this.sendAndReceive({ + controlCommand: controlCommand.toJson(), + msgName: 'CONTROL_REQUEST', + }); + } + + async requestScreenshot(): Promise { + const response = await this.sendAndReceive({ msgName: 'CAPTURE_SCREENSHOT_REQUEST' }); + if (response.data.image === undefined) { + throw new LegacyControllerError('The legacy UI Controller returned no screenshot.'); + } + return response.data.image; + } + + async getStartingArguments(): Promise> { + const response = await this.sendAndReceive({ msgName: 'GET_STARTING_ARGUMENTS_REQUEST' }); + return response.data.arguments ?? {}; + } + + // eslint-disable-next-line class-methods-use-this + async setActiveDisplay(): Promise { + // The legacy controller selects its display/device when it is launched. + } +} diff --git a/packages/askui-nodejs/src/execution/legacy-controller/legacy-controller-error.ts b/packages/askui-nodejs/src/execution/legacy-controller/legacy-controller-error.ts new file mode 100644 index 00000000000..ee65ae3c6a9 --- /dev/null +++ b/packages/askui-nodejs/src/execution/legacy-controller/legacy-controller-error.ts @@ -0,0 +1,6 @@ +export class LegacyControllerError extends Error { + constructor(message: string) { + super(message); + this.name = 'LegacyControllerError'; + } +} diff --git a/packages/askui-nodejs/src/execution/legacy-controller/legacy-controller-not-connected-error.ts b/packages/askui-nodejs/src/execution/legacy-controller/legacy-controller-not-connected-error.ts new file mode 100644 index 00000000000..90c4881b068 --- /dev/null +++ b/packages/askui-nodejs/src/execution/legacy-controller/legacy-controller-not-connected-error.ts @@ -0,0 +1,11 @@ +import { LegacyControllerError } from './legacy-controller-error'; + +export class LegacyControllerNotConnectedError extends LegacyControllerError { + constructor() { + super( + 'There is no connection to the AskUI legacy UI Controller. ' + + 'Make sure to call connect() before sending commands.', + ); + this.name = 'LegacyControllerNotConnectedError'; + } +} diff --git a/packages/askui-nodejs/src/execution/legacy-controller/runner-protocol.ts b/packages/askui-nodejs/src/execution/legacy-controller/runner-protocol.ts new file mode 100644 index 00000000000..ba1b9795033 --- /dev/null +++ b/packages/askui-nodejs/src/execution/legacy-controller/runner-protocol.ts @@ -0,0 +1,21 @@ +/** + * Minimal subset of the AskUI legacy UI Controller "runner protocol" (JSON over + * WebSocket, default `ws://127.0.0.1:6769`). Only the messages needed to drive + * an Android device are included; recording and interactive annotation are + * intentionally omitted. + */ + +export type RunnerProtocolRequest = + | { readonly msgName: 'CONTROL_REQUEST'; readonly controlCommand: object } + | { readonly msgName: 'CAPTURE_SCREENSHOT_REQUEST' } + | { readonly msgName: 'GET_STARTING_ARGUMENTS_REQUEST' }; + +export interface RunnerProtocolResponse { + readonly msgName: string; + readonly data: { + readonly error?: string; + readonly image?: string; + readonly msg?: string; + readonly arguments?: Record; + }; +} diff --git a/packages/askui-nodejs/src/execution/read-recording-response-stream-handler.ts b/packages/askui-nodejs/src/execution/read-recording-response-stream-handler.ts deleted file mode 100644 index f5615620c79..00000000000 --- a/packages/askui-nodejs/src/execution/read-recording-response-stream-handler.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { - ReadRecordingEndResponse, - ReadRecordingPartResponse, -} from '../core/runner-protocol'; - -export class ReadRecordingResponseStreamHandler { - videoChunks: string[] = []; - - constructor( - private resolve: ( - value: ReadRecordingPartResponse | PromiseLike - ) => void, - private reject: (reason?: unknown) => void, - ) { } - - onMessage(data: ReadRecordingPartResponse | ReadRecordingEndResponse) { - if (data.msgName === 'READ_RECORDING_PART_RESPONSE') { - this.videoChunks.push(data.data.video); - return; - } - if (data.msgName === 'READ_RECORDING_END_RESPONSE') { - this.resolve({ - msgName: 'READ_RECORDING_PART_RESPONSE', - data: { video: this.videoChunks.join('') }, - }); - return; - } - this.reject(`Wrong message: ${data}`); - } - - onError(err: unknown) { this.reject(err); } -} diff --git a/packages/askui-nodejs/src/execution/ui-control-client-dependency-builder.ts b/packages/askui-nodejs/src/execution/ui-control-client-dependency-builder.ts index 160c7d4265a..5ab219e7c9c 100644 --- a/packages/askui-nodejs/src/execution/ui-control-client-dependency-builder.ts +++ b/packages/askui-nodejs/src/execution/ui-control-client-dependency-builder.ts @@ -1,6 +1,9 @@ import isCI from 'is-ci'; import { HttpClientGot } from '../utils/http/http-client-got'; -import { UiControllerClient } from './ui-controller-client'; +import { AgentOsClient } from './agent-os/agent-os-client'; +import { AndroidAdbClient } from './android/android-adb-client'; +import { LegacyAndroidClient } from './legacy-controller/legacy-android-client'; +import { DeviceClient } from './device-client'; import { InferenceClient } from './inference-client'; import { ClientArgs, @@ -13,6 +16,7 @@ import { StepReporter } from '../core/reporting'; import { readCredentials } from './read-credentials'; import { LinearRetryStrategy } from './retry-strategies/linear-retry-strategy'; import { CacheManager, DummyCacheManager } from '../core/cache'; +import { logger } from '../lib/logger'; export class UiControlClientDependencyBuilder { private static async buildHttpClient( @@ -52,10 +56,17 @@ export class UiControlClientDependencyBuilder { ); } - private static buildUiControllerClient( + private static buildDeviceClient( clientArgs: ClientArgsWithDefaults, - ): UiControllerClient { - return new UiControllerClient(clientArgs.uiControllerUrl); + ): DeviceClient { + if (clientArgs.runtime === 'android') { + const android = clientArgs.android ?? {}; + if ((android.transport ?? 'legacy-controller') === 'adb') { + return new AndroidAdbClient(android); + } + return new LegacyAndroidClient(android); + } + return new AgentOsClient(clientArgs.agentOsUrl); } static async build(clientArgs: ClientArgsWithDefaults): Promise<{ @@ -63,13 +74,13 @@ export class UiControlClientDependencyBuilder { stepReporter: StepReporter; workspaceId: string | undefined; }> { - const uiControllerClient = UiControlClientDependencyBuilder.buildUiControllerClient(clientArgs); + const deviceClient = UiControlClientDependencyBuilder.buildDeviceClient(clientArgs); const inferenceClient = await UiControlClientDependencyBuilder.buildInferenceClient(clientArgs); const stepReporter = new StepReporter(clientArgs.reporter); const workspaceId = clientArgs.credentials?.workspaceId; return { executionRuntime: new ExecutionRuntime( - uiControllerClient, + deviceClient, inferenceClient, stepReporter, clientArgs.retryStrategy ?? new LinearRetryStrategy(), @@ -82,8 +93,15 @@ export class UiControlClientDependencyBuilder { static async getClientArgsWithDefaults( clientArgs: ClientArgs, ): Promise { + if (clientArgs.uiControllerUrl !== undefined && clientArgs.agentOsUrl === undefined) { + logger.warn( + "'uiControllerUrl' is deprecated and will be removed in a future release. " + + "Use 'agentOsUrl' instead.", + ); + } return { ...clientArgs, + agentOsUrl: clientArgs.agentOsUrl ?? clientArgs.uiControllerUrl ?? 'localhost:26000', aiElementArgs: { additionalLocations: clientArgs.aiElementArgs?.additionalLocations ?? [], onLocationNotExist: clientArgs.aiElementArgs?.onLocationNotExist ?? 'error', @@ -96,8 +114,7 @@ export class UiControlClientDependencyBuilder { inferenceServerUrl: clientArgs.inferenceServerUrl ?? 'https://inference.askui.com', proxyAgents: clientArgs.proxyAgents ?? (await envProxyAgents()), - uiControllerUrl: clientArgs.uiControllerUrl ?? 'http://127.0.0.1:6769', - + runtime: clientArgs.runtime ?? 'desktop', }; } } diff --git a/packages/askui-nodejs/src/execution/ui-control-client.ts b/packages/askui-nodejs/src/execution/ui-control-client.ts index 71dcf80eda7..7ab0f430ed1 100644 --- a/packages/askui-nodejs/src/execution/ui-control-client.ts +++ b/packages/askui-nodejs/src/execution/ui-control-client.ts @@ -84,7 +84,7 @@ export class UiControlClient extends ApiCommands { } /** - * Connects to the askui UI Controller. + * Connects to the AskUI AgentOS. */ async connect(): Promise { const connectionState = await this.executionRuntime.connect(); @@ -94,14 +94,14 @@ export class UiControlClient extends ApiCommands { } /** - * Disconnects from the askui UI Controller. + * Disconnects from the AskUI AgentOS. */ disconnect(): void { this.executionRuntime.disconnect(); } /** - * Disconnects from the askui UI Controller. + * Disconnects from the AskUI AgentOS. * * @deprecated Use {@link disconnect} instead. */ @@ -109,18 +109,6 @@ export class UiControlClient extends ApiCommands { this.disconnect(); } - async startVideoRecording(): Promise { - await this.executionRuntime.startVideoRecording(); - } - - async stopVideoRecording(): Promise { - await this.executionRuntime.stopVideoRecording(); - } - - async readVideoRecording(): Promise { - return this.executionRuntime.readVideoRecording(); - } - private shouldAnnotateAfterCommandExecution(error?: Error): boolean { return (this.stepReporter.config.withDetectedElements === 'onFailure' && error !== undefined) || (this.stepReporter.config.withDetectedElements === 'always'); @@ -177,14 +165,6 @@ export class UiControlClient extends ApiCommands { return annotation; } - async annotateInteractively(): Promise { - try { - await this.executionRuntime.annotateInteractively(); - } catch (err) { - logger.error(err); - } - } - // eslint-disable-next-line class-methods-use-this private escapeSeparatorString(instruction: string): string { return instruction.split(Separators.STRING).join('"'); @@ -1143,22 +1123,14 @@ export class UiControlClient extends ApiCommands { } /** - * Retrieves the starting arguments used when the controller server was initialized. + * Retrieves information about the AgentOS connection. * - * Useful for debugging, logging, or verifying the current server configuration. + * Useful for debugging, logging, or verifying the current configuration. * - * @property {string} displayNum - Display number controlled by the controller - * @property {boolean} minimize - Whether controller starts minimized - * @property {string} runtime - Runtime type ("desktop" or "android") - * @property {number} port - Communication port - * @property {number} actionWaitTime - Action wait time + * @property {number} displayNum - Display controlled by the AgentOS + * @property {string} runtime - Runtime type (always "desktop") + * @property {string} port - Communication port * @property {string} host - Host address - * @property {string} logFile - Log file path - * @property {boolean} hideOverlay - Whether overlay is hidden - * @property {boolean} debugDraw - Whether debug drawing is enabled - * @property {string} deviceId - Android device ID - * @property {string} configFile - Configuration file path - * @property {string} logLevel - Logging level * * @example * ```typescript @@ -1166,29 +1138,14 @@ export class UiControlClient extends ApiCommands { * console.log(startingArguments); * // Output example: * // { - * // displayNum: 0, - * // minimize: true, + * // displayNum: 1, + * // host: 'localhost', + * // port: '23000', * // runtime: 'desktop', - * // port: 5000, - * // actionWaitTime: 1000, - * // host: '127.0.0.1', - * // logFile: '/tmp/askui/askui-server.log', - * // hideOverlay: false, - * // debugDraw: false, - * // deviceId: 'emulator-5554', - * // configFile: '/tmp/askui/askui-config.json', - * // logLevel: 'info', * // } * ``` - * - * @example Retrieving Android device ID: - * ```typescript - * const startingArguments = await aui.getControllerStartingArguments(); - * console.log(startingArguments.deviceId); - * // Output example: "emulator-5554" - * ``` */ - async getControllerStartingArguments(): Promise> { + async getControllerStartingArguments(): Promise> { return this.executionRuntime.getStartingArguments(); } } diff --git a/packages/askui-nodejs/src/execution/ui-controller-client-error.ts b/packages/askui-nodejs/src/execution/ui-controller-client-error.ts deleted file mode 100644 index dea9bba2852..00000000000 --- a/packages/askui-nodejs/src/execution/ui-controller-client-error.ts +++ /dev/null @@ -1,6 +0,0 @@ -export class UiControllerClientError extends Error { - constructor(message: string) { - super(message); - this.name = 'UiControllerClientError'; - } -} diff --git a/packages/askui-nodejs/src/execution/ui-controller-client-interface.ts b/packages/askui-nodejs/src/execution/ui-controller-client-interface.ts index d112f7517ee..1b604900e32 100644 --- a/packages/askui-nodejs/src/execution/ui-controller-client-interface.ts +++ b/packages/askui-nodejs/src/execution/ui-controller-client-interface.ts @@ -6,6 +6,32 @@ import { Context } from './context'; import { RetryStrategy } from './retry-strategies/retry-strategy'; import { AIElementArgs } from '../core/ai-element/ai-elements-args'; import { CacheConfig } from '../core/cache'; +import { AndroidAdbClientArgs } from './android/android-adb-client'; +import { LegacyAndroidClientArgs } from './legacy-controller/legacy-android-client'; + +/** + * The runtime AskUI automates: the `desktop` via the AskUI AgentOS (gRPC), or an + * `android` device (see {@link AndroidArgs} for the transport). + */ +export type Runtime = 'desktop' | 'android'; + +/** + * How the Android runtime reaches the device. + * - `legacy-controller` (default): connect to a running AskUI legacy UI Controller + * over WebSocket (`AskUI-StartController ... -r android`). + * - `adb`: drive the device directly via `adb` (no controller process required). + */ +export type AndroidTransport = 'legacy-controller' | 'adb'; + +/** + * Options for the Android runtime. `transport` selects how the device is reached; + * the remaining options apply to the matching transport (`controllerUrl` for + * `legacy-controller`; `id`/`adbPath`/`keyboard` for `adb`; `actionDelayInMs` for + * both). + */ +export interface AndroidArgs extends AndroidAdbClientArgs, LegacyAndroidClientArgs { + readonly transport?: AndroidTransport; +} /** * Context object to provide additional information about the context of (test) automation. @@ -23,9 +49,13 @@ export interface ContextArgs { /** * Configuration options for AskUI's UiControlClient. * - * @property {string} [uiControllerUrl] - Default: `'http://127.0.0.1:6769'`. The address of - * AskUI's UiController that interacts with the operating system, e.g., simulating input - * events and capturing screenshots. + * @property {string} [agentOsUrl] - Default: `'localhost:26000'`. The gRPC address of + * the AskUI AgentOS (AskUI Remote Device Controller) that interacts with the operating + * system, e.g., simulating input events and capturing screenshots. `localhost:26000` is + * the address of the AgentOS managed by the AskUI OS service (`AskuiCoreService`); set it + * to `localhost:23000` to connect to a standalone AgentOS instead. + * @property {string} [uiControllerUrl] - **Deprecated.** Use {@link agentOsUrl} instead. Kept + * for backwards compatibility: used only when `agentOsUrl` is not set. * @property {string} [inferenceServerUrl] - Default: `'https://inference.askui.com'`. * Address of the AskUI's inference server which is responsible for understanding the * screenshots and extracting data from them and returning commands for the UiController. @@ -53,9 +83,22 @@ export interface ContextArgs { * errors or network issues, improving the reliability of interactions with the server. * @property {AIElementArgs} [aiElementArgs] - Options for configuring how AI elements are * collected. + * @property {Runtime} [runtime] - Default: `'desktop'`. The runtime to automate. `'desktop'` + * connects to the AskUI AgentOS via gRPC (see `agentOsUrl`); `'android'` automates an + * Android device (see `android`). + * @property {AndroidArgs} [android] - Optional. Options for the Android runtime. `transport` + * selects how the device is reached (`'legacy-controller'` by default, or `'adb'`); + * `controllerUrl` configures the legacy controller address, `id`/`adbPath`/`keyboard` the + * adb transport. Only used when `runtime` is `'android'`. */ export interface ClientArgs { + readonly agentOsUrl?: string + /** + * @deprecated Use {@link agentOsUrl} instead. Used only when `agentOsUrl` is not set. + */ readonly uiControllerUrl?: string + readonly runtime?: Runtime + readonly android?: AndroidArgs readonly inferenceServerUrl?: string readonly credentials?: CredentialArgs | undefined readonly proxyAgents?: ProxyAgentArgs | undefined @@ -70,7 +113,8 @@ export interface ClientArgs { } export interface ClientArgsWithDefaults extends ClientArgs { - readonly uiControllerUrl: string + readonly agentOsUrl: string + readonly runtime: Runtime readonly inferenceServerUrl: string readonly context: Context readonly inferenceServerApiVersion: string diff --git a/packages/askui-nodejs/src/execution/ui-controller-client.ts b/packages/askui-nodejs/src/execution/ui-controller-client.ts deleted file mode 100644 index 3fc7268fc79..00000000000 --- a/packages/askui-nodejs/src/execution/ui-controller-client.ts +++ /dev/null @@ -1,226 +0,0 @@ -import WebSocket from 'ws'; -import { DetectedElement } from '../core/model/annotation-result/detected-element'; -import { - CaptureScreenshotRequest, - CaptureScreenshotResponse, - ControlRequest, - ControlResponse, - RunnerProtocolRequest, - StartRecordingRequest, - StartRecordingResponse, - StopRecordingResponse, - StopRecordingRequest, - ReadRecordingPartResponse, - ReadRecordingRequest, - RunnerProtocolResponse, - InteractiveAnnotationRequest, - InteractiveAnnotationResponse, - GetProcessPidRequest, - GetProcessPidResponse, - GetStartingArgumentsRequest, - GetStartingArgumentsResponse, -} from '../core/runner-protocol'; -import { ControlCommand } from '../core/ui-control-commands'; -import { logger } from '../lib/logger'; -import { UiControllerClientConnectionState } from './ui-controller-client-connection-state'; -import { ReadRecordingResponseStreamHandler } from './read-recording-response-stream-handler'; -import { UiControllerClientError } from './ui-controller-client-error'; -import { UiControllerNotConnectedError } from './ui-controller-not-connected-error'; - -export class UiControllerClient { - private static readonly EMPTY_REJECT = (_reason?: unknown) => {}; - - private static readonly EMPTY_RESOLVE = (_value: unknown) => {}; - - private static readonly REQUEST_TIMEOUT_IN_MS = 30000; - - ws!: WebSocket; - - connectionState = UiControllerClientConnectionState.NOT_CONNECTED; - - private timeout?: NodeJS.Timeout | undefined; - - private currentReject: (reason?: unknown) => void = UiControllerClient.EMPTY_REJECT; - - private currentResolve: ( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - value: any - ) => void = UiControllerClient.EMPTY_RESOLVE; - - constructor(public url: string) {} - - private clearResponse() { - this.currentReject = UiControllerClient.EMPTY_REJECT; - this.currentResolve = UiControllerClient.EMPTY_RESOLVE; - } - - private onMessage(data: WebSocket.Data) { - logger.debug('onMessage'); - clearTimeout(this.timeout as NodeJS.Timeout); - this.timeout = undefined; - const response: RunnerProtocolResponse = JSON.parse(data.toString()); - if (response.data.error) { - logger.error(response.data.error); - this.currentReject(new UiControllerClientError(response.data.error)); - this.clearResponse(); - return; - } - this.currentResolve(response); - if (response.msgName === 'READ_RECORDING_PART_RESPONSE') { - return; - } - this.clearResponse(); - } - - connect(): Promise { - this.connectionState = UiControllerClientConnectionState.CONNECTING; - return new Promise((resolve, reject) => { - try { - this.ws = new WebSocket(this.url); - this.ws.on('message', (data) => { - this.onMessage(data); - }); - this.ws.on('open', () => { - this.connectionState = UiControllerClientConnectionState.CONNECTED; - resolve(this.connectionState); - }); - this.ws.on('error', (error: WebSocket.ErrorEvent) => { - this.connectionState = UiControllerClientConnectionState.ERROR; - reject( - new UiControllerClientError(`Connection to UI Controller cannot be established, - Probably it was not started. Make sure you started UI Controller with this - Url ${this.url}. Cause: ${error}`), - ); - }); - } catch (error) { - reject( - new UiControllerClientError( - `Connection to UI Controller cannot be established. Cause: ${error}`, - ), - ); - } - }); - } - - disconnect(): void { - this.ws?.close(); - } - - private checkConnection(): void { - if (this.connectionState !== UiControllerClientConnectionState.CONNECTED) { - throw new UiControllerNotConnectedError(); - } - } - - private sendAndReceive( - msg: RunnerProtocolRequest, - requestTimeout = UiControllerClient.REQUEST_TIMEOUT_IN_MS, - ): Promise { - this.checkConnection(); - return new Promise((resolve, reject) => { - logger.debug(`sendAndReceive - ${JSON.stringify(msg)}`); - this.currentResolve = resolve; - this.currentReject = reject; - try { - this.send(msg, requestTimeout); - if (this.timeout) { - throw new UiControllerClientError(`Clear the current request before setting a new one. Check for missing await. Error: ${JSON.stringify(msg)}`); - } - this.timeout = setTimeout( - () => this.currentReject( - new UiControllerClientError( - 'Request to UI Controller timed out. It seems that the UI Controller is not running. Please, make sure that it is running when executing tests.', - ), - ), - UiControllerClient.REQUEST_TIMEOUT_IN_MS, - ); - } catch (error) { - this.currentReject( - new UiControllerClientError( - `The communication to the UI Controller is broken. Cause: ${error}`, - ), - ); - } - }); - } - - private send( - msg: RunnerProtocolRequest, - _requestTimeout = UiControllerClient.REQUEST_TIMEOUT_IN_MS, - ) { - this.checkConnection(); - if (!this.currentReject || !this.currentResolve) { - throw new UiControllerClientError( - 'Request is not finished! It is not possible to have multiple requests at the same time.', - ); - } - logger.debug(`Send: ${JSON.stringify(msg.msgName)}`); - this.ws.send(JSON.stringify(msg)); - } - - requestScreenshot(): Promise { - return this.sendAndReceive( - new CaptureScreenshotRequest(), - ); - } - - getStartingArguments(): Promise { - return this.sendAndReceive( - new GetStartingArgumentsRequest(), - ); - } - - getServerPid(): Promise { - return this.sendAndReceive( - new GetProcessPidRequest(), - ); - } - - startVideoRecording(): Promise { - return this.sendAndReceive( - new StartRecordingRequest(), - ); - } - - stopVideoRecording(): Promise { - return this.sendAndReceive( - new StopRecordingRequest(), - ); - } - - readVideoRecording(): Promise { - return new Promise((resolve, reject) => { - const readRecordingStreamHandler = new ReadRecordingResponseStreamHandler( - resolve, - reject, - ); - this.currentResolve = readRecordingStreamHandler.onMessage.bind( - readRecordingStreamHandler, - ); - this.currentReject = readRecordingStreamHandler.onError.bind( - readRecordingStreamHandler, - ); - this.send(new ReadRecordingRequest(), 60 * 15 * 1000); - }); - } - - annotateInteractively( - boundingBoxes: DetectedElement[], - imageString: string, - ): Promise { - return new Promise((resolve, reject) => { - this.currentResolve = resolve; - this.currentReject = reject; - this.send( - new InteractiveAnnotationRequest(boundingBoxes, imageString), - 60 * 60 * 1000, - ); - }); - } - - requestControl(controlCommand: ControlCommand): Promise { - return this.sendAndReceive( - new ControlRequest(controlCommand), - ); - } -} diff --git a/packages/askui-nodejs/src/execution/ui-controller-not-connected-error.ts b/packages/askui-nodejs/src/execution/ui-controller-not-connected-error.ts deleted file mode 100644 index e0b8afb8e86..00000000000 --- a/packages/askui-nodejs/src/execution/ui-controller-not-connected-error.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { UiControllerClientError } from './ui-controller-client-error'; - -export class UiControllerNotConnectedError extends UiControllerClientError { - constructor() { - super( - 'UI Controller is not connected. Did you call `UiControlClient.connect()`' - + ' before trying to interact with the UI Controller / OS?', - ); - this.name = 'UiControllerNotConnectedError'; - } -} diff --git a/packages/askui-nodejs/src/lib/download-binaries.ts b/packages/askui-nodejs/src/lib/download-binaries.ts deleted file mode 100644 index c30e0721780..00000000000 --- a/packages/askui-nodejs/src/lib/download-binaries.ts +++ /dev/null @@ -1,83 +0,0 @@ -import fs from 'fs'; -import got from 'got'; -import os from 'os'; -import path from 'path'; -import http from 'http'; -import https from 'https'; -import { promisify } from 'util'; -import stream from 'stream'; -import { getPathToNodeModulesRoot } from '../utils/path'; -import { logger } from './logger'; - -enum SupportedPlatform { - DARWIN = 'darwin', - LINUX = 'linux', - WIN32 = 'win32', -} - -const binarySubPathsByPlatform = { - darwin: ['darwin', 'askui-ui-controller.dmg'], - linux: ['linux', 'askui-ui-controller.AppImage'], - win32: ['windows', 'askui-ui-controller.exe'], -} as const; - -function isSupportedPlatform(value: string): value is SupportedPlatform { - return Object.values(SupportedPlatform).includes(value); -} - -export function platform(): SupportedPlatform { - const pf = os.platform(); - if (isSupportedPlatform(pf)) { - return pf; - } - throw new Error(`Platform "${pf}" is not supported.`); -} - -function buildBinaryNotAvailableError(binaryVersion: string): Error { - return new Error(`There was an error during downloading and creating the askui UI Controller. You can try a fresh install. - If you specified a binary version it is possible that the askui UI Controller version "${binaryVersion}" - for your system "${platform()} ${os.arch}" is not available. - If this problem still occurs, please contact us at info@askui.com for more information`); -} - -export function getBinaryFilePath(version: string): string { - return path.join(getPathToNodeModulesRoot(), 'release', version, ...binarySubPathsByPlatform[platform()]); -} - -function getBinaryDownloadUrl(binaryVersion: string): string { - const baseUrl = `https://files.askui.com/releases/askui-ui-controller/${binaryVersion}`; - const arch = os.arch(); - return `${baseUrl}/${platform()}/${arch}/${binarySubPathsByPlatform[platform()][1]}`; -} - -export async function downloadServerBinaries( - binaryVersion: string, - proxyAgent?: { http: http.Agent, https: https.Agent }, -): Promise { - logger.info(`Start downloading UI Controller version "${binaryVersion}"`); - const url = getBinaryDownloadUrl(binaryVersion); - const binaryFilePath = getBinaryFilePath(binaryVersion); - const binaryFolderPath = path.dirname(binaryFilePath); - const tempFilePath = path.join(binaryFolderPath, 'askui-ui-controller.temp'); - const pipeline = promisify(stream.pipeline); - const downloadStream = got.stream(url, proxyAgent ? { agent: proxyAgent } : {}); - const fileWriterStream = fs.createWriteStream(tempFilePath); - - if (!(fs.existsSync(binaryFolderPath))) { - fs.mkdirSync(binaryFolderPath, { recursive: true }); - } - - try { - await pipeline(downloadStream, fileWriterStream); - fs.rename(tempFilePath, binaryFilePath, () => { - logger.info(`UI Controller version ${binaryVersion} for your system "${platform()} ${os.arch}" was downloaded`); - logger.debug(`Binary of UI Controller is located at "${binaryFilePath}".`); - }); - } catch (error) { - logger.error(error); - fs.unlink(tempFilePath, (err) => { logger.error(err); }); - throw buildBinaryNotAvailableError(binaryVersion); - } - - return Promise.resolve(); -} diff --git a/packages/askui-nodejs/src/lib/index.ts b/packages/askui-nodejs/src/lib/index.ts index af103e6d6ae..c3c478d553e 100644 --- a/packages/askui-nodejs/src/lib/index.ts +++ b/packages/askui-nodejs/src/lib/index.ts @@ -1,5 +1,2 @@ export { init } from './interactive_cli'; -export { UnkownError } from './unkown-error'; -export { TimeoutError } from './timeout-error'; -export { UiController } from './ui-controller'; export { logger } from './logger'; diff --git a/packages/askui-nodejs/src/lib/interactive_cli/create-example-project.ts b/packages/askui-nodejs/src/lib/interactive_cli/create-example-project.ts index a1f633fb25c..54209c949d0 100644 --- a/packages/askui-nodejs/src/lib/interactive_cli/create-example-project.ts +++ b/packages/askui-nodejs/src/lib/interactive_cli/create-example-project.ts @@ -118,8 +118,6 @@ export class CreateExampleProject { private async addReporterConfig() { this.helperTemplateConfig['allure_stepreporter_import'] = "import { AskUIAllureStepReporter } from '@askui/askui-reporters';"; this.helperTemplateConfig['reporter_placeholder'] = 'reporter: new AskUIAllureStepReporter(),'; - this.helperTemplateConfig['allure_stepreporter_attach_video'] = `const video = await aui.readVideoRecording(); - await AskUIAllureStepReporter.attachVideo(video);`; } private async addAskuiRunCommand() { diff --git a/packages/askui-nodejs/src/lib/libfuse-error.ts b/packages/askui-nodejs/src/lib/libfuse-error.ts deleted file mode 100644 index 547f5ceb478..00000000000 --- a/packages/askui-nodejs/src/lib/libfuse-error.ts +++ /dev/null @@ -1 +0,0 @@ -export class LibfuseError extends Error { } diff --git a/packages/askui-nodejs/src/lib/timeout-error.ts b/packages/askui-nodejs/src/lib/timeout-error.ts deleted file mode 100644 index 0d798abd93a..00000000000 --- a/packages/askui-nodejs/src/lib/timeout-error.ts +++ /dev/null @@ -1 +0,0 @@ -export class TimeoutError extends Error { } diff --git a/packages/askui-nodejs/src/lib/ui-controller-args.spec.ts b/packages/askui-nodejs/src/lib/ui-controller-args.spec.ts deleted file mode 100644 index ea095878581..00000000000 --- a/packages/askui-nodejs/src/lib/ui-controller-args.spec.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { createArgsWithDefaults, createCliFlagsFromArgs } from './ui-controller-args'; - -describe('createCliFlagsFromArgs()', () => { - test('test createCliFlagsFromArgs should return -d 0 as default output', () => { - const expected = ['-d 0', '-p 6769', '--action_wait_time 1000', '--host 127.0.0.1', '-m ', '--log-level debug']; - const argsWithDefaults = createArgsWithDefaults(); - const actual = createCliFlagsFromArgs(argsWithDefaults); - expect(actual).toStrictEqual(expected); - }); - - test('test createCliFlagsFromArgs output should include -d 0 when no display was selected', () => { - const expected = ['-d 0', '-p 6777', '--action_wait_time 1000', '--host 0.0.0.0', '-m ', '--log-level debug']; - const argsWithDefaults = createArgsWithDefaults({ port: 6777, host: '0.0.0.0' }); - const actual = createCliFlagsFromArgs(argsWithDefaults); - - expect(actual).toStrictEqual(expected); - }); - - test('test createCliFlagsFromArgs output should include the display that was selected ', () => { - const expected = ['-d 99', '-p 6777', '--action_wait_time 1000', '--host 0.0.0.0', '-m ', '--log-level debug']; - const argsWithDefaults = createArgsWithDefaults({ display: 99, host: '0.0.0.0', port: 6777 }); - const actual = createCliFlagsFromArgs(argsWithDefaults); - expect(actual).toStrictEqual(expected); - }); -}); diff --git a/packages/askui-nodejs/src/lib/ui-controller-args.ts b/packages/askui-nodejs/src/lib/ui-controller-args.ts deleted file mode 100644 index 59260d3a6b9..00000000000 --- a/packages/askui-nodejs/src/lib/ui-controller-args.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { ProxyAgentArgs } from '../shared/proxy-agent-args'; -import { LogLevels } from '../shared/log-levels'; - -/** - * Configuration options for the askui UI Controller - * - * @param {number} display - Default: `0` - * You can choose on which display you want to excecute all tests. - * 0 is your main monitor. - * - * If you want to use your second monitor you can change - * the value to `1` (`2` for your third monitor etc.). - * @param {string} binaryVersion - Default: `'latest'` - * Choose the version of the askui UI Controller Server. - * @param {number} actionDelayInMs - Default: `1000` - * Waits x milliseconds after each action. This can be used to slow down or speed up the execution - * @param {number} port - Default: `6769` - * The port the askui UI Controller is running on. - * @param {string} host - Default: `'127.0.0.1'` - * The host the askui UI Controller is running on. - * @param {boolean} minimize - Default: `true` - * askui UI Controller will be started as minimized window. - * @param {boolean} overWriteBinary - Default: `false` - * Download the provided Version of askui UI Controller. - * If a version is already downloaded. This version will be overwritten - * @param {LogLevels} logLevel - Default: `'debug'` - * You can use different types of Log Level. - * Options: "fatal", "error", "warn", "info", "debug", "trace", "silent", "verbose". - * @param {string} logFilePath - Default: `'/askui/askui-server.log'` - * It is possible to specify a path for your log files. - * Per default we create the askui-server.log file and askui folder in your temp folder. - * @param {ProxyAgentArgs} proxyAgents - To configure the proxy agents for our http(s) requests. - */ - -export interface UiControllerArgs { - readonly actionDelayInMs?: number; - readonly binaryVersion?: string, - readonly display?: number; - readonly host?: string; - readonly port?: number; - readonly minimize?: boolean; - readonly overWriteBinary?: boolean; - readonly logLevel?: LogLevels; - readonly logFilePath?: string; - readonly proxyAgents?: ProxyAgentArgs -} - -export interface UiControllerArgsWithDefaults extends UiControllerArgs { - readonly actionDelayInMs: number; - readonly binaryVersion: string; - readonly display: number; - readonly overWriteBinary: boolean; - readonly port: number; - readonly host: string; - readonly logLevel?: LogLevels; -} - -export function createArgsWithDefaults( - args?: UiControllerArgs, -): UiControllerArgsWithDefaults { - const defaults = { - actionDelayInMs: 1000, - binaryVersion: 'latest', - display: 0, - host: '127.0.0.1', - logLevel: 'debug', - minimize: true, - overWriteBinary: false, - port: 6769, - }; - return Object.assign(defaults, args); -} - -export function createCliFlagsFromArgs(args: UiControllerArgsWithDefaults): string[] { - return [ - `-d ${args.display.toString()}`, - args?.port ? `-p ${args.port.toString()}` : '', - args?.actionDelayInMs ? `--action_wait_time ${args.actionDelayInMs.toString()}` : '', - args?.host ? `--host ${args.host}` : '', - args?.minimize ? '-m ' : '', - args?.logLevel ? `--log-level ${args.logLevel}` : '', - args?.logFilePath ? `--log-file ${args.logFilePath}` : '', - ].filter((arg) => !!arg); -} diff --git a/packages/askui-nodejs/src/lib/ui-controller-darwin.ts b/packages/askui-nodejs/src/lib/ui-controller-darwin.ts deleted file mode 100644 index a9f6b572851..00000000000 --- a/packages/askui-nodejs/src/lib/ui-controller-darwin.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { exec, execSync } from 'child_process'; -import fs from 'fs-extra'; -import path from 'path'; -import { UiControllerFacade } from './ui-controller-facade'; -import { logger } from './logger'; - -export class UiControllerDarwin extends UiControllerFacade { - protected override makeBinaryExecutable(): void { - this.makeDiskImageExecutable(); - } - - protected override getStartingCommand(): string { - return `"${path.dirname(this.binaryFilePath)}/askui-ui-controller.app/Contents/MacOS/askui-ui-controller"`; - } - - private makeDiskImageExecutable() { - const mountPoint = '/Volumes/askui-ui-controller.dmg'; - execSync([ - 'hdiutil attach', - '-nobrowse', - '-quiet', - '-noautofsck', - '-noautoopen', - `-mountpoint "${mountPoint}"`, - `"${this.binaryFilePath}"`, - ].join(' ')); - const appBaseName = 'askui-ui-controller.app'; - const appSrcPath = `${mountPoint}/${appBaseName}`; - const appDestPath = `${path.dirname(this.binaryFilePath)}/${appBaseName}`; - fs.removeSync(appDestPath); - fs.copySync(appSrcPath, appDestPath); - exec(`hdiutil detach "${mountPoint}"`, (_exception, stdout) => logger.debug(stdout)); - } -} diff --git a/packages/askui-nodejs/src/lib/ui-controller-facade.ts b/packages/askui-nodejs/src/lib/ui-controller-facade.ts deleted file mode 100644 index 42eb94d6028..00000000000 --- a/packages/askui-nodejs/src/lib/ui-controller-facade.ts +++ /dev/null @@ -1,147 +0,0 @@ -import { spawn } from 'child_process'; -import fs from 'fs-extra'; -import waitPort from 'wait-port'; -import fkill from 'fkill'; -import os from 'os'; -import path from 'path'; -import http from 'http'; -import https from 'https'; -import { - UiControllerArgs, - UiControllerArgsWithDefaults, - createArgsWithDefaults, - createCliFlagsFromArgs, -} from './ui-controller-args'; -import { downloadServerBinaries, getBinaryFilePath } from './download-binaries'; -import { logger } from './logger'; -import { TimeoutError } from './timeout-error'; -import { UnkownError } from './unkown-error'; -import { envProxyAgents } from '../utils/proxy/proxy-builder'; - -export abstract class UiControllerFacade { - protected binaryFilePath = getBinaryFilePath('latest'); - - protected serverLogFile!: string; - - protected readonly DefaultmaxWaitingForStartingInMs = 30 * 1000; - - async start(args?: UiControllerArgs, maxWaitingForStartingInSeconds?: number) { - await this.runPreStartChecks(); - const argsWithDefaults = createArgsWithDefaults(args); - const argsWithLogPath = this.serverLogFilePath(argsWithDefaults); - this.binaryFilePath = getBinaryFilePath(argsWithLogPath.binaryVersion); - await this.getBinary( - argsWithLogPath.binaryVersion, - argsWithLogPath.overWriteBinary, - argsWithLogPath.proxyAgents || await envProxyAgents(), - ); - this.makeBinaryExecutable(); - logger.debug(`UI Controller log path "${this.serverLogFile}"`); - await this.startWithDefaults(argsWithLogPath, maxWaitingForStartingInSeconds); - } - - async stop(args?: UiControllerArgs, forceStop?: boolean): Promise { - try { - const argsWithDefaults = createArgsWithDefaults(args); - await this.killPort(argsWithDefaults.port, forceStop); - } catch (err) { - throw new Error(`An unknown error occured while closing of the UI Controller. Log file: "${this.serverLogFile}". ErrorReason: ${err}`); - } - } - - protected serverLogFilePath( - args?: UiControllerArgsWithDefaults, - ): UiControllerArgsWithDefaults { - if (args?.logFilePath) { - this.serverLogFile = args.logFilePath; - return args; - } - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'askui')); - this.serverLogFile = path.join(tmpDir, 'askui-ui-controller.log'); - const argPath = { logFilePath: this.serverLogFile }; - return Object.assign(argPath, args); - } - - // eslint-disable-next-line class-methods-use-this - protected killPort(port: number, forceStop?: boolean): Promise { - return fkill( - `:${port}`, - { - force: forceStop || false, - silent: true, - }, - ); - } - - protected getStartingCommand(): string { - return `"${this.binaryFilePath}"`; - } - - // eslint-disable-next-line class-methods-use-this - protected makeBinaryExecutable() { - /* Executable out of the box */ - } - - // eslint-disable-next-line class-methods-use-this - protected runPreStartChecks(): Promise { - return Promise.resolve(); - } - - // eslint-disable-next-line class-methods-use-this - protected waitUntilStarted( - args: UiControllerArgsWithDefaults, - maxWaitingForStartingInSeconds?: number, - ): Promise { - return new Promise((resolve, reject) => { - try { - const timeoutInMs = maxWaitingForStartingInSeconds - ? maxWaitingForStartingInSeconds * 1000 : this.DefaultmaxWaitingForStartingInMs; - waitPort({ - host: args.host, - output: process?.env['LOG_LEVEL'] === 'verbose' ? 'dots' : 'silent', - port: args.port, - timeout: timeoutInMs, - }).then((returnObject) => { - if (returnObject.open) { - logger.info('The Control UI Server has been started.'); - return resolve(); - } - return reject(new TimeoutError('Starting time limit has been reached')); - }); - } catch (err) { - reject(new UnkownError(`An unknown error occured while waiting for the UI Controller: ${err}`)); - } - }); - } - - private async getBinary( - binaryVersion: string, - overWriteBinary = false, - proxyAgent?: { http: http.Agent, https: https.Agent }, - ): Promise { - if (!fs.existsSync(this.binaryFilePath) || overWriteBinary) { - logger.debug(`Currently, no binary of the UI Controller is available at "${this.binaryFilePath}"`); - await downloadServerBinaries(binaryVersion, proxyAgent); - } else { - logger.debug(`Binary of UI Controller is already present at "${this.binaryFilePath}".`); - } - } - - private async startWithDefaults( - args: UiControllerArgsWithDefaults, - maxWaitingForStartingInSeconds?: number, - ) { - try { - logger.debug('Starting the UI Controller...'); - spawn( - this.getStartingCommand(), - createCliFlagsFromArgs(args), - { shell: true }, - ); - await this.waitUntilStarted(args, maxWaitingForStartingInSeconds); - } catch (err) { - throw new Error(`The UI Controller could not be started. Log file : ${this.serverLogFile}. ErrorReason: ${err} - Check this website for more information: https://docs.askui.com/docs/general/Troubleshooting`); - } - } -} diff --git a/packages/askui-nodejs/src/lib/ui-controller-linux.ts b/packages/askui-nodejs/src/lib/ui-controller-linux.ts deleted file mode 100644 index 6bf8acfe487..00000000000 --- a/packages/askui-nodejs/src/lib/ui-controller-linux.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { exec } from 'child_process'; -import { promisify } from 'util'; -import { UiControllerFacade } from './ui-controller-facade'; -import { logger } from './logger'; -import { WaylandError } from './wayland-error'; -import { LibfuseError } from './libfuse-error'; - -export class UiControllerLinux extends UiControllerFacade { - // eslint-disable-next-line class-methods-use-this - protected override makeBinaryExecutable(): void { - exec(`chmod +x "${this.binaryFilePath}"`, (_exception, stdout) => logger.debug(stdout)); - } - - // eslint-disable-next-line class-methods-use-this - protected override async runPreStartChecks(): Promise { - const runCommand = promisify(exec); - const waylandStatus = await runCommand('echo $WAYLAND_DISPLAY'); - - if (waylandStatus.stdout.trim().includes('wayland')) { - throw new WaylandError('Wayland is not supported: https://docs.askui.com/docs/general/Troubleshooting/linux#wayland'); - } - - /* First we want to check if the user is using a debian distribution. - * and in the following if libfuse2 is installed. - * With Ubunutu 22.04 libfuse2 is not installed per default. - * For more information: https://discourse.joplinapp.org/t/appimage-incompatibility-in-ubuntu-22-04/25173 - */ - try { - await runCommand('dpkg --version'); - } catch (err) { - return; - } - - try { - await runCommand('dpkg -s libfuse2 | grep Status'); - } catch { - throw new LibfuseError('Libfuse2 package is missing: https://docs.askui.com/docs/general/Troubleshooting/linux#libfuse2'); - } - } -} diff --git a/packages/askui-nodejs/src/lib/ui-controller-win32.ts b/packages/askui-nodejs/src/lib/ui-controller-win32.ts deleted file mode 100644 index 7d482b66e40..00000000000 --- a/packages/askui-nodejs/src/lib/ui-controller-win32.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { UiControllerFacade } from './ui-controller-facade'; - -export class UiControllerWin32 extends UiControllerFacade {} diff --git a/packages/askui-nodejs/src/lib/ui-controller.ts b/packages/askui-nodejs/src/lib/ui-controller.ts deleted file mode 100644 index cf1e601ee2a..00000000000 --- a/packages/askui-nodejs/src/lib/ui-controller.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { UiControllerFacade } from './ui-controller-facade'; -import { UiControllerArgs } from './ui-controller-args'; -import { UiControllerDarwin } from './ui-controller-darwin'; -import { UiControllerLinux } from './ui-controller-linux'; -import { UiControllerWin32 } from './ui-controller-win32'; -import { platform } from './download-binaries'; - -/** - * @deprecated Will be removed soon because the AskUI Controller - * gets started as an operating system service. - */ -export class UiController { - private server: UiControllerFacade; - - constructor(private args?: UiControllerArgs) { - switch (platform()) { - case 'darwin': - this.server = new UiControllerDarwin(); - break; - case 'linux': - this.server = new UiControllerLinux(); - break; - case 'win32': - this.server = new UiControllerWin32(); - break; - default: - throw new Error(`Platform "${platform()}" not supported.`); - } - } - - async start(timeoutInSeconds?:number) { - await this.server.start(this.args, timeoutInSeconds); - } - - async stop(forceStop?: boolean) { - await this.server.stop(this.args, forceStop); - } -} diff --git a/packages/askui-nodejs/src/lib/unkown-error.ts b/packages/askui-nodejs/src/lib/unkown-error.ts deleted file mode 100644 index 822434b3f24..00000000000 --- a/packages/askui-nodejs/src/lib/unkown-error.ts +++ /dev/null @@ -1 +0,0 @@ -export class UnkownError extends Error { } diff --git a/packages/askui-nodejs/src/lib/wayland-error.ts b/packages/askui-nodejs/src/lib/wayland-error.ts deleted file mode 100644 index 73f369cc810..00000000000 --- a/packages/askui-nodejs/src/lib/wayland-error.ts +++ /dev/null @@ -1 +0,0 @@ -export class WaylandError extends Error { } diff --git a/packages/askui-nodejs/src/main.ts b/packages/askui-nodejs/src/main.ts index c7bd6188f90..1532799868c 100644 --- a/packages/askui-nodejs/src/main.ts +++ b/packages/askui-nodejs/src/main.ts @@ -1,5 +1,19 @@ -export { UiController } from './lib'; export * from './execution'; +export { DeviceClient } from './execution/device-client'; +export { AgentOsClient } from './execution/agent-os/agent-os-client'; +export { AgentOsError } from './execution/agent-os/agent-os-error'; +export { AgentOsNotConnectedError } from './execution/agent-os/agent-os-not-connected-error'; +export { AgentOsActionNotSupportedError } from './execution/agent-os/agent-os-action-not-supported-error'; +export { AndroidAdbClient, AndroidAdbClientArgs, AndroidKeyboardMode } from './execution/android/android-adb-client'; +export { AndroidError } from './execution/android/android-error'; +export { NoAndroidDeviceError } from './execution/android/no-android-device-error'; +export { AndroidNotConnectedError } from './execution/android/android-not-connected-error'; +export { LegacyAndroidClient, LegacyAndroidClientArgs } from './execution/legacy-controller/legacy-android-client'; +export { LegacyControllerError } from './execution/legacy-controller/legacy-controller-error'; +export { LegacyControllerNotConnectedError } from './execution/legacy-controller/legacy-controller-not-connected-error'; +export { + Runtime, AndroidTransport, AndroidArgs, ClientArgs, +} from './execution/ui-controller-client-interface'; export { Instruction, Reporter, diff --git a/packages/askui-nodejs/src/utils/proxy/http-client-got-proxy.spec.ts b/packages/askui-nodejs/src/utils/proxy/http-client-got-proxy.spec.ts index 8b4f10cd212..6086b30bc1a 100644 --- a/packages/askui-nodejs/src/utils/proxy/http-client-got-proxy.spec.ts +++ b/packages/askui-nodejs/src/utils/proxy/http-client-got-proxy.spec.ts @@ -2,7 +2,7 @@ import https from 'https'; import http from 'http'; import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; -import { AddressInfo } from 'ws'; +import { AddressInfo } from 'net'; import { HttpClientGot } from '../http/http-client-got'; import { buildProxy, buildSecureServer, SERVER_HOSTNAME } from '../../../test/proxy/proxy-utils'; diff --git a/packages/askui-nodejs/test-android.ts b/packages/askui-nodejs/test-android.ts new file mode 100644 index 00000000000..3958a8fde52 --- /dev/null +++ b/packages/askui-nodejs/test-android.ts @@ -0,0 +1,218 @@ +/* eslint-disable no-console */ +/** + * Manual end-to-end test for the Android device clients. + * + * Two transports (select with TRANSPORT): + * - adb (TRANSPORT=adb): needs `adb` on PATH (or ADB_PATH) and a connected + * device (`adb devices`). + * - legacy-controller (default): needs the AskUI legacy UI Controller running + * in Android mode (`AskUI-StartController ... -r android`). + * + * For the typing step with Unicode: ADBKeyboard installed and set as the default + * IME (adb transport: set KEYBOARD=adb-keyboard). + * + * Run with: + * npm run test:android # legacy controller (default) + * TRANSPORT=adb npm run test:android # direct adb + * + * Environment variables: + * TRANSPORT=adb|legacy-controller Select the transport (default legacy-controller). + * CONTROLLER_URL=ws://127.0.0.1:6769 Legacy controller address. + * ADB_PATH=/path/to/adb Override the adb executable. + * DEVICE_ID=emulator-5554 Select a specific device (adb, default: first). + * KEYBOARD=adb-keyboard Use the ADBKeyboard IME for Unicode typing (adb). + * SKIP_INPUT=1 Only connect/screenshot/info, skip input actions. + */ +import fs from 'fs'; +import path from 'path'; +import { AndroidAdbClient } from './src/execution/android/android-adb-client'; +import { AndroidError } from './src/execution/android/android-error'; +import { AndroidNotConnectedError } from './src/execution/android/android-not-connected-error'; +import { LegacyAndroidClient } from './src/execution/legacy-controller/legacy-android-client'; +import { LegacyControllerNotConnectedError } from './src/execution/legacy-controller/legacy-controller-not-connected-error'; +import { DeviceClient } from './src/execution/device-client'; +import { + Action, + ControlCommand, + ControlCommandCode, + InputEvent, +} from './src/core/ui-control-commands'; +import { UiControllerClientConnectionState } from './src/execution/ui-controller-client-connection-state'; + +const SKIP_INPUT = process.env['SKIP_INPUT'] === '1'; +const TRANSPORT = process.env['TRANSPORT'] === 'adb' ? 'adb' : 'legacy-controller'; +const SCREENSHOT_PATH = path.join(__dirname, 'android-test-screenshot.png'); + +interface StepResult { + name: string; + ok: boolean; + detail: string; +} + +const results: StepResult[] = []; + +async function step(name: string, fn: () => Promise): Promise { + process.stdout.write(`→ ${name} ... `); + try { + const detail = (await fn()) ?? ''; + results.push({ detail, name, ok: true }); + console.log(`✓ ${detail}`); + return true; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + results.push({ detail, name, ok: false }); + console.log(`✗ ${detail}`); + return false; + } +} + +function pngDimensions(base64Png: string): { width: number; height: number } { + const buffer = Buffer.from(base64Png.split(',')[1] ?? '', 'base64'); + return { + height: buffer.readUInt32BE(20), + width: buffer.readUInt32BE(16), + }; +} + +function printSummary(): void { + console.log('='.repeat(60)); + const passed = results.filter((result) => result.ok).length; + results + .filter((result) => !result.ok) + .forEach((result) => console.log(`FAILED: ${result.name}\n ${result.detail}`)); + console.log(`${passed}/${results.length} steps passed`); +} + +function buildClient(): DeviceClient { + if (TRANSPORT === 'adb') { + return new AndroidAdbClient({ + ...(process.env['ADB_PATH'] ? { adbPath: process.env['ADB_PATH'] } : {}), + ...(process.env['DEVICE_ID'] ? { id: process.env['DEVICE_ID'] } : {}), + keyboard: process.env['KEYBOARD'] === 'adb-keyboard' ? 'adb-keyboard' : 'input-text', + }); + } + return new LegacyAndroidClient({ + ...(process.env['CONTROLLER_URL'] ? { controllerUrl: process.env['CONTROLLER_URL'] } : {}), + }); +} + +async function main(): Promise { + console.log(`Android end-to-end test (transport: ${TRANSPORT})`); + console.log('='.repeat(60)); + + const client = buildClient(); + const execute = (...actions: Action[]) => client.requestControl( + new ControlCommand(ControlCommandCode.OK, actions), + ); + + let screenWidth = 0; + let screenHeight = 0; + + const connected = await step(`connect (${TRANSPORT})`, async () => { + const state = await client.connect(); + if (state !== UiControllerClientConnectionState.CONNECTED) { + throw new AndroidError(`Unexpected connection state: ${state}`); + } + return `state=${state}`; + }); + + if (!connected) { + printSummary(); + process.exit(1); + } + + await step('getStartingArguments (runtime=android)', async () => { + const args = await client.getStartingArguments(); + if (args['runtime'] !== 'android') { + throw new AndroidError(`Unexpected runtime: ${args['runtime']}`); + } + return JSON.stringify(args); + }); + + await step('requestScreenshot (screencap → PNG)', async () => { + const screenshot = await client.requestScreenshot(); + if (!screenshot.startsWith('data:image/png;base64,')) { + throw new AndroidError('Screenshot is not a base64 PNG data URI'); + } + const { width, height } = pngDimensions(screenshot); + if (width < 1 || height < 1) { + throw new AndroidError(`Implausible dimensions ${width}x${height}`); + } + screenWidth = width; + screenHeight = height; + fs.writeFileSync( + SCREENSHOT_PATH, + Buffer.from(screenshot.split(',')[1] ?? '', 'base64'), + ); + return `${width}x${height}, saved to ${SCREENSHOT_PATH}`; + }); + + if (!SKIP_INPUT) { + await step('PRESS_ANDROID_SINGLE_KEY (HOME)', () => execute( + new Action(InputEvent.PRESS_ANDROID_SINGLE_KEY, { x: 0, y: 0 }, 'home', {}), + )); + + await step('EXECUTE_COMMAND (input tap, center)', () => execute( + new Action( + InputEvent.EXECUTE_COMMAND, + { x: 0, y: 0 }, + `input tap ${Math.round(screenWidth / 2)} ${Math.round(screenHeight / 2)}`, + {}, + ), + )); + + await step('EXECUTE_COMMAND (input swipe, scroll up)', () => execute( + new Action( + InputEvent.EXECUTE_COMMAND, + { x: 0, y: 0 }, + `input swipe ${Math.round(screenWidth / 2)} ${Math.round(screenHeight * 0.7)} ` + + `${Math.round(screenWidth / 2)} ${Math.round(screenHeight * 0.3)}`, + {}, + ), + )); + + await step('MOUSE_MOVE + MOUSE_CLICK_LEFT (tap at stored position)', () => execute( + new Action(InputEvent.MOUSE_MOVE, { x: Math.round(screenWidth / 2), y: Math.round(screenHeight / 2) }, '', {}), + new Action(InputEvent.MOUSE_CLICK_LEFT, { x: 0, y: 0 }, '', {}), + )); + + await step('TYPE (into focused field, if any)', () => execute( + new Action(InputEvent.TYPE, { x: 0, y: 0 }, 'askui android test', {}), + )); + + await step('PRESS_ANDROID_KEY_SEQUENCE (BACK)', () => execute( + new Action(InputEvent.PRESS_ANDROID_KEY_SEQUENCE, { x: 0, y: 0 }, 'back', {}), + )); + } + + await step('disconnect', async () => { + client.disconnect(); + if (client.connectionState !== UiControllerClientConnectionState.NOT_CONNECTED) { + throw new AndroidError(`Unexpected state after disconnect: ${client.connectionState}`); + } + return `state=${client.connectionState}`; + }); + + await step('requests after disconnect are rejected', async () => { + const ExpectedError = TRANSPORT === 'adb' + ? AndroidNotConnectedError + : LegacyControllerNotConnectedError; + try { + await client.requestScreenshot(); + } catch (error) { + if (error instanceof ExpectedError) { + return `threw ${error.name} as expected`; + } + throw error; + } + throw new AndroidError('Expected an error but none was thrown'); + }); + + printSummary(); + process.exit(results.every((result) => result.ok) ? 0 : 1); +} + +main().catch((error) => { + console.error('Unexpected failure:', error); + process.exit(1); +}); diff --git a/packages/askui-nodejs/test.ts b/packages/askui-nodejs/test.ts new file mode 100644 index 00000000000..e7dc7fd0628 --- /dev/null +++ b/packages/askui-nodejs/test.ts @@ -0,0 +1,241 @@ +/* eslint-disable no-console */ +/** + * Manual end-to-end test for the AskUI AgentOS gRPC client. + * + * Requires a running AskUI AgentOS (standalone on localhost:23000 or managed by + * the AskuiCoreService on localhost:26000). IMPORTANT: The AgentOS only allows a + * single session — close the AskUI Desktop app (and other SDK sessions) before + * running this script. + * + * Run with: + * npm run test:agentos + * + * Environment variables: + * AGENT_OS_URL=localhost:23000 Override the AgentOS address. + * SKIP_INPUT=1 Only test connect/screenshot/info, skip all + * mouse/keyboard/command actions. + * + * The script opens a maximized Notepad window (Windows) as a safe target for + * mouse/keyboard actions and force-closes it afterwards. + */ +import fs from 'fs'; +import path from 'path'; +import { AgentOsClient } from './src/execution/agent-os/agent-os-client'; +import { AgentOsActionNotSupportedError } from './src/execution/agent-os/agent-os-action-not-supported-error'; +import { AgentOsError } from './src/execution/agent-os/agent-os-error'; +import { + Action, + ControlCommand, + ControlCommandCode, + InputEvent, +} from './src/core/ui-control-commands'; +import { UiControllerClientConnectionState } from './src/execution/ui-controller-client-connection-state'; + +const AGENT_OS_URL = process.env['AGENT_OS_URL'] ?? 'localhost:26000'; +const SKIP_INPUT = process.env['SKIP_INPUT'] === '1'; +const SCREENSHOT_PATH = path.join(__dirname, 'agentos-test-screenshot.png'); + +interface StepResult { + name: string; + ok: boolean; + detail: string; +} + +const results: StepResult[] = []; + +async function step(name: string, fn: () => Promise): Promise { + process.stdout.write(`→ ${name} ... `); + try { + const detail = (await fn()) ?? ''; + results.push({ detail, name, ok: true }); + console.log(`✓ ${detail}`); + return true; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + results.push({ detail, name, ok: false }); + console.log(`✗ ${detail}`); + return false; + } +} + +function delay(milliseconds: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, milliseconds); + }); +} + +function pngDimensions(base64Png: string): { width: number; height: number } { + const buffer = Buffer.from(base64Png.split(',')[1] ?? '', 'base64'); + return { + height: buffer.readUInt32BE(20), + width: buffer.readUInt32BE(16), + }; +} + +function printSummary(): void { + console.log('='.repeat(60)); + const passed = results.filter((result) => result.ok).length; + results + .filter((result) => !result.ok) + .forEach((result) => console.log(`FAILED: ${result.name}\n ${result.detail}`)); + console.log(`${passed}/${results.length} steps passed`); +} + +async function main(): Promise { + console.log(`AgentOS end-to-end test against "${AGENT_OS_URL}"`); + console.log('='.repeat(60)); + + const client = new AgentOsClient(AGENT_OS_URL); + const execute = (...actions: Action[]) => client.requestControl( + new ControlCommand(ControlCommandCode.OK, actions), + ); + + let screenWidth = 0; + let screenHeight = 0; + + const connected = await step('connect (with service detection)', async () => { + const state = await client.connect(); + if (state !== UiControllerClientConnectionState.CONNECTED) { + throw new AgentOsError(`Unexpected connection state: ${state}`); + } + return `state=${state}`; + }); + + if (!connected) { + printSummary(); + process.exit(1); + } + + await step('getStartingArguments', async () => { + const args = await client.getStartingArguments(); + if (args['runtime'] !== 'desktop') { + throw new AgentOsError(`Unexpected runtime: ${args['runtime']}`); + } + return JSON.stringify(args); + }); + + await step('requestScreenshot (BGRA → PNG conversion)', async () => { + const screenshot = await client.requestScreenshot(); + if (!screenshot.startsWith('data:image/png;base64,')) { + throw new AgentOsError('Screenshot is not a base64 PNG data URI'); + } + const { width, height } = pngDimensions(screenshot); + if (width < 1 || height < 1) { + throw new AgentOsError(`Implausible dimensions ${width}x${height}`); + } + screenWidth = width; + screenHeight = height; + fs.writeFileSync( + SCREENSHOT_PATH, + Buffer.from(screenshot.split(',')[1] ?? '', 'base64'), + ); + return `${width}x${height}, saved to ${SCREENSHOT_PATH}`; + }); + + if (!SKIP_INPUT && process.platform === 'win32') { + await step('EXECUTE_COMMAND (open maximized Notepad)', async () => { + await execute(new Action(InputEvent.EXECUTE_COMMAND, { x: 0, y: 0 }, 'cmd /c start /MAX notepad', {})); + await delay(2000); + }); + + await step('TYPE (unicode text)', () => execute( + new Action(InputEvent.TYPE, { x: 0, y: 0 }, 'AskUI AgentOS gRPC test — äöü ± 😀', {}), + )); + + await step('PRESS_KEY_SEQUENCE (enter)', () => execute( + new Action(InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, 'enter', {}), + )); + + await step('TYPE_TEXT (second line)', () => execute( + new Action(InputEvent.TYPE_TEXT, { x: 0, y: 0 }, 'second line 1234', {}), + )); + + await step('MOUSE_MOVE (screen center)', () => execute( + new Action( + InputEvent.MOUSE_MOVE, + { x: Math.round(screenWidth / 2), y: Math.round(screenHeight / 2) }, + '', + {}, + ), + )); + + await step('MOUSE_MOVE_RELATIVELY (+50, +30)', () => execute( + new Action(InputEvent.MOUSE_MOVE_RELATIVELY, { x: 50, y: 30 }, '', {}), + )); + + await step('MOUSE_SCROLL (vertical −120, horizontal +120)', () => execute( + new Action(InputEvent.MOUSE_SCROLL, { x: 120, y: -120 }, '', {}), + )); + + await step('MOUSE_CLICK_LEFT', () => execute( + new Action(InputEvent.MOUSE_CLICK_LEFT, { x: 0, y: 0 }, '', {}), + )); + + await step('MOUSE_CLICK_DOUBLE_LEFT (selects a word)', () => execute( + new Action(InputEvent.MOUSE_CLICK_DOUBLE_LEFT, { x: 0, y: 0 }, '', {}), + )); + + await step('MOUSE_DOWN + MOUSE_UP', () => execute( + new Action(InputEvent.MOUSE_DOWN, { x: 0, y: 0 }, '', {}), + new Action(InputEvent.MOUSE_UP, { x: 0, y: 0 }, '', {}), + )); + + await step('KEY_PRESS + KEY_RELEASE (hold and release shift)', () => execute( + new Action(InputEvent.KEY_PRESS, { x: 0, y: 0 }, '', { key: 'shift', modifiers: [] }), + new Action(InputEvent.KEY_RELEASE, { x: 0, y: 0 }, '', { key: 'shift', modifiers: [] }), + )); + + await step('PRESS_KEY_SEQUENCE with modifier (control+a, delete)', () => execute( + new Action(InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, 'control+a', {}), + new Action(InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, 'delete', {}), + )); + + await step('EXECUTE_COMMAND (close Notepad)', () => execute( + new Action(InputEvent.EXECUTE_COMMAND, { x: 0, y: 0 }, 'taskkill /IM notepad.exe /F', {}), + )); + } else if (!SKIP_INPUT) { + console.log(' (input action steps are Windows-only, skipping on this platform)'); + } + + await step('PRESS_ANDROID_SINGLE_KEY rejects (Android unsupported)', async () => { + try { + await execute(new Action(InputEvent.PRESS_ANDROID_SINGLE_KEY, { x: 0, y: 0 }, 'home', {})); + } catch (error) { + if (error instanceof AgentOsActionNotSupportedError) { + return 'threw AgentOsActionNotSupportedError as expected'; + } + throw new AgentOsError(`Wrong error type: ${error}`); + } + throw new AgentOsError('Expected an AgentOsActionNotSupportedError but none was thrown'); + }); + + await step('disconnect', async () => { + client.disconnect(); + if (client.connectionState !== UiControllerClientConnectionState.NOT_CONNECTED) { + throw new AgentOsError(`Unexpected state after disconnect: ${client.connectionState}`); + } + // Give the background session teardown a moment to release the session. + await delay(500); + return `state=${client.connectionState}`; + }); + + await step('requests after disconnect are rejected', async () => { + try { + await client.requestScreenshot(); + } catch (error) { + if (error instanceof AgentOsError) { + return 'threw AgentOsNotConnectedError as expected'; + } + throw error; + } + throw new AgentOsError('Expected an error but none was thrown'); + }); + + printSummary(); + process.exit(results.every((result) => result.ok) ? 0 : 1); +} + +main().catch((error) => { + console.error('Unexpected failure:', error); + process.exit(1); +});