fix: CDN URLs with credentials; complete the v8 upgrade docs

README.md

@@ -31,8 +31,13 @@ From geckodriver 8 and edgedriver 8, `@wdio/logger` is a peer dependency, so the
 - If you use `geckodriver` or `edgedriver` without WebdriverIO and install with Yarn, add `@wdio/logger` to your dependencies. npm and pnpm install it for you.
 - `geckodriver` no longer reads `GECKODRIVER_FILEPATH`. Use `GECKODRIVER_PATH`.
 - `require()` now returns every export of the package, for example `findEdgePath` from `edgedriver`, with CommonJS types.
+- With `require('safaridriver')`, `start()` returns the `ChildProcess` and `stop()` returns nothing, as in ESM: before, both returned a Promise. `await start()` still works; `start().then(...)` does not.
+- `HTTPS_PROXY` and `HTTP_PROXY` now apply to downloads; 6.x and 7.x ignored them. If a proxy is set in your environment but the CDN must be reached directly, add its host to `NO_PROXY`.
 - `edgedriver`'s `start()` resolves to a `ChildProcess` (it was typed `ChildProcessWithoutNullStreams`): its `stdout` and `stderr` are `null` when you pass `spawnOpts: { stdio: 'ignore' }`.
-- The rest of the API, the CLI and the options did not change.
+- `safaridriver` now pipes the driver output like the other drivers: read `stdout` and `stderr`, or pass `spawnOpts: { stdio: 'ignore' }`. Before, the output was buffered and the driver was killed after 1 MB of it.
+- The `edgedriver` and `geckodriver` CLIs exit with code 1 when a signal kills the driver; they exited with 0.
+- `findEdgePath()` on Windows returns `undefined` when Edge is not installed, as on macOS and Linux; it threw.
+- `spawnOpts` is a new option of `edgedriver` and `safaridriver`. The rest of the API, the CLI and the options did not change.
 
 ---
 
@@ -84,14 +89,17 @@ By default, binaries download when initialized via CLI or API. To download them
 * **Custom Driver Version:**
 * `GECKODRIVER_VERSION="0.31.0"`
 * `EDGEDRIVER_VERSION="114.0.1823.18"`
+* `EDGE_BINARY_PATH=/path/to/msedge`: the Edge binary whose version selects the Edgedriver download when `EDGEDRIVER_VERSION` is not set. `findEdgePath()` returns it.
 
 
 * **Custom CDN URL:**
 * `GECKODRIVER_CDNURL=https://INTERNAL_CDN/geckodriver/download`
 * `EDGEDRIVER_CDNURL=https://INTERNAL_CDN/edgedriver/download`
 
 
-* **HTTP/HTTPS Proxy:** Standard `HTTP_PROXY` and `HTTPS_PROXY` environment variables are supported for downloads.
+* **CDN credentials:** a CDN URL can carry them, for example `https://user:password@INTERNAL_CDN`. They are sent as a Basic `Authorization` header and kept out of the logs; percent-encode special characters (`@` is `%40`, `%` is `%25`). Use `https://`: over `http://`, Basic credentials travel in clear text. A redirect to another host does not receive them.
+
+* **HTTP/HTTPS Proxy:** `HTTPS_PROXY` and `HTTP_PROXY` apply to downloads, and `NO_PROXY` lists the hosts that skip the proxy. The lower-case forms work too and win over the upper-case ones.
 
 #### Windows Setup (`selenium-webdriver` note)
 
@@ -191,6 +199,7 @@ Passed into the `start(options)` method:
 | `cacheDir` | `string` | `process.env.EDGEDRIVER_CACHE_DIR \|\| os.tmpdir()` | Root directory for caching downloaded binaries. |
 | `allowedIps` | `string[]` | `['']` | List of remote IP addresses allowed to connect. |
 | `allowedOrigins` | `string[]` | `['*']` | List of allowed request origins. Using `*` to allow any origin is dangerous! |
+| `spawnOpts` | `object` | `undefined` | Spawn options passed directly to Node.js `child_process.spawn`, e.g. `{ stdio: 'ignore' }` if you don't read the driver output. |
 
 See the [edgedriver README](packages/node-edgedriver/README.md) for the full list of options.
 
@@ -205,6 +214,7 @@ Passed into `safaridriver.start(options)`:
 | `useTechnologyPreview` | `boolean` | `false` | Enables Safari Technology Preview driver binary. |
 | `enable` | `boolean` | `false` | Configures macOS permissions ("Enable Remote Automation") and exits immediately. |
 | `diagnose` | `boolean` | `false` | Enables diagnostic log output for driver sessions. |
+| `spawnOpts` | `object` | `undefined` | Spawn options passed directly to Node.js `child_process.spawn`, e.g. `{ stdio: 'ignore' }` if you don't read the driver output. |
 
 See the [safaridriver README](packages/node-safaridriver/README.md) for more details.
 

packages/node-edgedriver/.npmignore

@@ -5,4 +5,4 @@ coverage
 *.tsbuildinfo
 /.*
 /*.mjs
-/tsconfig.json
+/tsconfig*.json

packages/node-edgedriver/README.md

@@ -124,6 +124,8 @@ edgedriver_cdnurl=https://msedgedriver.microsoft.com
 
 The default location is set to https://msedgedriver.microsoft.com
 
+A CDN URL can carry credentials, for example `https://user:password@INTERNAL_CDN`: they are sent as a Basic `Authorization` header and kept out of the logs. Percent-encode special characters in them (`@` is `%40`, `%` is `%25`). Use an `https://` URL: over `http://`, Basic credentials travel in clear text. A redirect to another host does not receive them.
+
 ## Options
 
 The `start` method offers the following options to be passed on to the actual Edgedriver CLI.

packages/node-edgedriver/src/install.ts

@@ -58,16 +58,25 @@ export async function download (
     return binaryFilePath
 }
 
+/**
+ * Every CDN request goes through here: fetch rejects a URL with `user:pass@`, so the
+ * credentials of `EDGEDRIVER_CDNURL` move into an Authorization header. `url` has none,
+ * so it is the one to log.
+ */
+function fetchCdn (rawUrl: string) {
+    const { url, authHeader } = extractBasicAuthFromUrl(rawUrl)
+    const opts: RequestInit = { ...fetchOpts }
+    if (authHeader) {
+        opts.headers = { Authorization: authHeader }
+    }
+    return { url, response: fetch(url, opts) }
+}
+
 async function downloadDriver(version: string) {
     try {
-        const rawDownloadUrl = format(DOWNLOAD_URL, version, getNameByArchitecture())
-        const { url: downloadUrl, authHeader } = extractBasicAuthFromUrl(rawDownloadUrl)
+        const { url: downloadUrl, response } = fetchCdn(format(DOWNLOAD_URL, version, getNameByArchitecture()))
         log.info(`Downloading Edgedriver from ${downloadUrl}`)
-        const opts: RequestInit = { ...fetchOpts }
-        if (authHeader) {
-            opts.headers = { ...opts.headers, Authorization: authHeader }
-        }
-        const res = await fetch(downloadUrl, opts)
+        const res = await response
 
         if (!res.body || !res.ok || res.status !== 200) {
             throw new Error(`Failed to download binary from ${downloadUrl} (statusCode ${res.status})`)
@@ -181,7 +190,7 @@ export async function fetchVersion (edgeVersion: string) {
             return productVersion
         }
 
-        const res = await fetch(format(TAGGED_VERSION_URL, edgeVersion.toUpperCase()), fetchOpts)
+        const res = await fetchCdn(format(TAGGED_VERSION_URL, edgeVersion.toUpperCase())).response
         if (res.status !== 200) {
             throw new Error(`Couldn't fetch the latest ${edgeVersion} version (statusCode ${res.status}): ${res.statusText}`)
         }
@@ -195,9 +204,9 @@ export async function fetchVersion (edgeVersion: string) {
     const versionMatch = edgeVersion.match(MATCH_VERSION)
     if (versionMatch) {
         const [major] = versionMatch
-        const url = format(LATEST_RELEASE_URL, major.toString().toUpperCase(), platform.toUpperCase())
+        const { url, response } = fetchCdn(format(LATEST_RELEASE_URL, major.toString().toUpperCase(), platform.toUpperCase()))
         log.info(`Fetching latest version from ${url}`)
-        const res = await fetch(url, fetchOpts)
+        const res = await response
         if (!res.ok || res.status !== 200) {
             throw new Error(`Couldn't detect version for ${edgeVersion}`)
         }

packages/node-edgedriver/src/types.ts

@@ -4,7 +4,7 @@ export type LogLevel = 'ALL' | 'DEBUG' | 'INFO' | 'WARNING' | 'SEVERE' | 'OFF'
 
 export interface EdgedriverParameters {
     /**
-     * Version of Edgedriver to start. See https://msedgedriver.azureedge.net/ for all available versions, platforms and architecture.
+     * Version of Edgedriver to start. See https://msedgedriver.microsoft.com/ for all available versions, platforms and architecture.
      */
     edgeDriverVersion?: string
     /**

packages/node-edgedriver/src/utils.ts

@@ -125,6 +125,18 @@ export interface BasicAuthResult {
     authHeader?: string
 }
 
+/**
+ * A bare `%` (`p%ss`) is not percent-encoding: keep it as text. If decodeURIComponent threw here,
+ * the caller would get back, and log, the URL with the password in it.
+ */
+function decodeCredential (value: string) {
+    try {
+        return decodeURIComponent(value)
+    } catch {
+        return value
+    }
+}
+
 /**
  * Extract Basic Auth credentials from a URL and return the cleaned URL with auth header.
  * This is needed because fetch() doesn't support URLs with embedded credentials.
@@ -135,7 +147,8 @@ export function extractBasicAuthFromUrl(urlString: string): BasicAuthResult {
     try {
         const url = new URL(urlString)
         if (url.username || url.password) {
-            const credentials = btoa(`${url.username}:${url.password}`)
+            // URL keeps them percent-encoded (`p%40ss`); the header needs the real `p@ss`
+            const credentials = Buffer.from(`${decodeCredential(url.username)}:${decodeCredential(url.password)}`).toString('base64')
             url.username = ''
             url.password = ''
             return {

packages/node-edgedriver/tests/cdnCredentials.test.ts

@@ -0,0 +1,66 @@
+import fs from 'node:fs'
+import http from 'node:http'
+import os from 'node:os'
+import path from 'node:path'
+import type { AddressInfo } from 'node:net'
+import { BlobWriter, TextReader, ZipWriter } from '@zip.js/zip.js'
+import { afterAll, beforeAll, expect, test, vi } from 'vitest'
+
+/**
+ * A local CDN behind `EDGEDRIVER_CDNURL=http://user:p%40ss@…`: version .91 is missing,
+ * so download() falls back to LATEST_RELEASE_120_*, which points at .92.
+ */
+const requests: { url: string, auth?: string }[] = []
+let server: http.Server
+let cacheDir = ''
+
+beforeAll(async () => {
+    const { getNameByArchitecture } = await import('../src/utils.js')
+    const { BINARY_FILE } = await import('../src/constants.js')
+    const zipWriter = new ZipWriter(new BlobWriter('application/zip'))
+    await zipWriter.add(BINARY_FILE, new TextReader('#!/bin/sh\n'))
+    const zip = Buffer.from(await (await zipWriter.close()).arrayBuffer())
+    // the real LATEST_* files are UTF-16LE with a byte order mark
+    const latest = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from('120.0.2210.92', 'utf16le')])
+
+    server = http.createServer((req, res) => {
+        requests.push({ url: req.url ?? '', auth: req.headers.authorization })
+        if (req.url?.startsWith('/LATEST_RELEASE_120_')) {
+            return res.end(latest)
+        }
+        if (req.url === `/120.0.2210.92/${getNameByArchitecture()}.zip`) {
+            return res.end(zip)
+        }
+        res.writeHead(404).end()
+    })
+    await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
+    process.env.EDGEDRIVER_CDNURL = `http://user:p%40ss@127.0.0.1:${(server.address() as AddressInfo).port}`
+    cacheDir = fs.mkdtempSync(path.join(os.tmpdir(), 'edgedriver-cdn-'))
+    // the CDN URL is read when the module loads
+    vi.resetModules()
+})
+
+afterAll(() => {
+    delete process.env.EDGEDRIVER_CDNURL
+    server.close()
+    fs.rmSync(cacheDir, { recursive: true, force: true })
+})
+
+test('sends CDN credentials as a header on every request, also for the version fallback', async () => {
+    const { log } = await import('../src/constants.js')
+    const info = vi.spyOn(log, 'info')
+    const { download } = await import('../src/install.js')
+
+    const binary = await download('120.0.2210.91', cacheDir)
+
+    expect(fs.existsSync(binary)).toBe(true)
+    expect(requests.map((r) => r.url)).toEqual([
+        expect.stringMatching(/^\/120\.0\.2210\.91\//),
+        expect.stringMatching(/^\/LATEST_RELEASE_120_/),
+        expect.stringMatching(/^\/120\.0\.2210\.92\//),
+    ])
+    // the password is `p@ss`: percent-encoding is undone before it goes into the header
+    const expected = `Basic ${Buffer.from('user:p@ss').toString('base64')}`
+    expect(requests.every((r) => r.auth === expected)).toBe(true)
+    expect(JSON.stringify(info.mock.calls)).not.toMatch(/user|p%40ss|p@ss/)
+})

packages/node-edgedriver/tests/utils.test.ts

@@ -55,6 +55,12 @@ describe('extractBasicAuthFromUrl', () => {
         expect(result.authHeader).toBe('Basic ' + Buffer.from('myuser:').toString('base64'))
     })
 
+    test('a bare % in a credential is kept as text, not left in the URL', () => {
+        const result = extractBasicAuthFromUrl('https://us%er:p%ss@cdn.example.com/path/file.zip')
+        expect(result.url).toBe('https://cdn.example.com/path/file.zip')
+        expect(result.authHeader).toBe('Basic ' + Buffer.from('us%er:p%ss').toString('base64'))
+    })
+
     test('with invalid URL returns original', () => {
         const result = extractBasicAuthFromUrl('not-a-valid-url')
         expect(result.url).toBe('not-a-valid-url')

packages/node-geckodriver/.npmignore

@@ -5,5 +5,5 @@ coverage
 *.tsbuildinfo
 /.*
 /*.mjs
-/tsconfig.json
+/tsconfig*.json
 CHANGELOG.md

packages/node-geckodriver/README.md

@@ -61,10 +61,16 @@ GECKODRIVER_CDNURL=https://INTERNAL_CDN/geckodriver/download
 
 Binaries on your CDN should be located in a subdirectory of the above base URL. For example, `/vxx.xx.xx/*.tar.gz` should be located under `/geckodriver/download` above.
 
-Alternatively, you can add the same property to your .npmrc file.
+Alternatively, add it to your [`.npmrc`](https://docs.npmjs.com/cli/configuring-npm/npmrc) file:
+
+```
+geckodriver_cdnurl=https://INTERNAL_CDN/geckodriver/download
+```
 
 The default location is set to https://github.com/mozilla/geckodriver/releases/download
 
+A CDN URL can carry credentials, for example `https://user:password@INTERNAL_CDN`: they are sent as a Basic `Authorization` header and kept out of the logs. Percent-encode special characters in them (`@` is `%40`, `%` is `%25`). Use an `https://` URL: over `http://`, Basic credentials travel in clear text. A redirect to another host does not receive them.
+
 ## Setting a PROXY URL
 
 Use `HTTPS_PROXY` or `HTTP_PROXY` to set your proxy URL, and `NO_PROXY` for the hosts that skip it. The lower-case forms (`https_proxy`, `http_proxy`, `no_proxy`) work too and win over the upper-case ones.

packages/node-geckodriver/src/install.ts

@@ -10,7 +10,7 @@ import { unpackTar } from 'modern-tar/fs'
 import { BlobReader, BlobWriter, ZipReader, type FileEntry } from '@zip.js/zip.js'
 
 import { BINARY_FILE, GECKODRIVER_CARGO_YAML } from './constants.js'
-import { hasAccess, getDownloadUrl, retryFetch } from './utils.js'
+import { hasAccess, getDownloadUrl, retryFetch, extractBasicAuthFromUrl } from './utils.js'
 
 const log = logger('geckodriver')
 
@@ -72,9 +72,9 @@ export async function download (
     }
 
     const binaryFilePath = path.resolve(cacheDir, getBinaryFilename(geckodriverVersion))
-    const url = getDownloadUrl(geckodriverVersion)
+    const { url, authHeader } = extractBasicAuthFromUrl(getDownloadUrl(geckodriverVersion))
     log.info(`Downloading Geckodriver from ${url}`)
-    const res = await retryFetch(url, fetchOpts)
+    const res = await retryFetch(url, authHeader ? { ...fetchOpts, headers: { Authorization: authHeader } } : fetchOpts)
 
     if (!res.body || res.status !== 200) {
         throw new Error(`Failed to download binary (statusCode ${res.status}): ${res.statusText}`)

packages/node-geckodriver/src/utils.ts

@@ -61,6 +61,38 @@ export async function retryFetch(url: string, opts: RequestInit = {}, retry = 3)
     throw new Error('Failed to fetch after retries')
 }
 
+/**
+ * A bare `%` (`p%ss`) is not percent-encoding: keep it as text. If decodeURIComponent threw here,
+ * the caller would get back, and log, the URL with the password in it.
+ */
+function decodeCredential (value: string) {
+    try {
+        return decodeURIComponent(value)
+    } catch {
+        return value
+    }
+}
+
+/**
+ * fetch rejects a URL with `user:pass@`: move the credentials of `GECKODRIVER_CDNURL` into an
+ * Authorization header. The returned `url` has none, so it is the one to log.
+ */
+export function extractBasicAuthFromUrl(urlString: string): { url: string, authHeader?: string } {
+    try {
+        const url = new URL(urlString)
+        if (url.username || url.password) {
+            // URL keeps them percent-encoded (`p%40ss`); the header needs the real `p@ss`
+            const credentials = Buffer.from(`${decodeCredential(url.username)}:${decodeCredential(url.password)}`).toString('base64')
+            url.username = ''
+            url.password = ''
+            return { url: url.toString(), authHeader: `Basic ${credentials}` }
+        }
+    } catch {
+        // not a valid URL: let fetch report it
+    }
+    return { url: urlString }
+}
+
 function sleep(ms: number) {
     return new Promise(resolve => setTimeout(resolve, ms))
 }

packages/node-geckodriver/tests/cdnCredentials.test.ts

@@ -0,0 +1,56 @@
+import fs from 'node:fs'
+import http from 'node:http'
+import os from 'node:os'
+import path from 'node:path'
+import { execFileSync } from 'node:child_process'
+import type { AddressInfo } from 'node:net'
+import logger from '@wdio/logger'
+import { afterAll, beforeAll, describe, expect, test, vi } from 'vitest'
+
+/**
+ * A local CDN behind `GECKODRIVER_CDNURL=http://user:p%40ss@…` serving a real .tar.gz
+ */
+describe.skipIf(process.platform === 'win32')('CDN credentials', () => {
+    const requests: { url: string, auth?: string }[] = []
+    let server: http.Server
+    let root = ''
+
+    beforeAll(async () => {
+        root = fs.mkdtempSync(path.join(os.tmpdir(), 'geckodriver-cdn-'))
+        fs.mkdirSync(path.join(root, 'archive'))
+        fs.writeFileSync(path.join(root, 'archive', 'geckodriver'), '#!/bin/sh\n', { mode: 0o755 })
+        execFileSync('tar', ['-czf', path.join(root, 'geckodriver.tar.gz'), '-C', path.join(root, 'archive'), 'geckodriver'])
+        const archive = fs.readFileSync(path.join(root, 'geckodriver.tar.gz'))
+
+        server = http.createServer((req, res) => {
+            requests.push({ url: req.url ?? '', auth: req.headers.authorization })
+            if (req.url?.startsWith('/v0.36.0/geckodriver-v0.36.0-')) {
+                return res.end(archive)
+            }
+            res.writeHead(404).end()
+        })
+        await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
+        process.env.GECKODRIVER_CDNURL = `http://user:p%40ss@127.0.0.1:${(server.address() as AddressInfo).port}`
+        // the CDN URL is read when the module loads
+        vi.resetModules()
+    })
+
+    afterAll(() => {
+        delete process.env.GECKODRIVER_CDNURL
+        server.close()
+        fs.rmSync(root, { recursive: true, force: true })
+    })
+
+    test('sends CDN credentials as a header and keeps them out of the log', async () => {
+        const info = vi.spyOn(logger('geckodriver'), 'info')
+        const { download } = await import('../src/install.js')
+
+        const binary = await download('0.36.0', path.join(root, 'cache'))
+
+        expect(fs.existsSync(binary)).toBe(true)
+        expect(requests).toHaveLength(1)
+        // the password is `p@ss`: percent-encoding is undone before it goes into the header
+        expect(requests[0].auth).toBe(`Basic ${Buffer.from('user:p@ss').toString('base64')}`)
+        expect(JSON.stringify(info.mock.calls)).not.toMatch(/user|p%40ss|p@ss/)
+    })
+})