Skip to content

Commit 09e0de0

Browse files
jakebaileyjakebailey
authored
Build all platforms in parallel in release pipeline (#64563)
1 parent b82cd13 commit 09e0de0

2 files changed

Lines changed: 239 additions & 88 deletions

File tree

‎Herebyfile.mjs‎

Lines changed: 153 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ const { values: rawOptions } = parseArgs({
8888
forRelease: { type: "boolean" },
8989
respectGoEnv: { type: "boolean" },
9090
vscodeTypescriptRelease: { type: "boolean" },
91+
platform: { type: "string" },
9192

9293
race: { type: "boolean", default: parseEnvBoolean("RACE") },
9394
noembed: { type: "boolean", default: parseEnvBoolean("NOEMBED") },
@@ -2289,46 +2290,79 @@ async function runBuildNativePreviewPackages() {
22892290
await rimraf(builtNpm);
22902291

22912292
const platforms = getPlatforms();
2293+
await buildMainNativePreviewPackage(platforms);
22922294

2293-
const inputDir = "./packages/typescript";
2294-
2295-
const inputPackageJson = JSON.parse(fs.readFileSync(path.join(inputDir, "package.json"), "utf8"));
2296-
inputPackageJson.version = getVersion();
2297-
delete inputPackageJson.private;
2298-
inputPackageJson.files = [...new Set([...(inputPackageJson.files ?? []), "NOTICE.txt"])];
2299-
if (publishAsTypescript) {
2300-
inputPackageJson.bin = {
2301-
tsc: "./bin/tsc",
2302-
};
2303-
inputPackageJson.description = "TypeScript is a language for application scale JavaScript development";
2304-
inputPackageJson.homepage = "https://www.typescriptlang.org/";
2305-
inputPackageJson.keywords = [
2306-
"TypeScript",
2307-
"Microsoft",
2308-
"compiler",
2309-
"language",
2310-
"javascript",
2311-
];
2312-
inputPackageJson.bugs = {
2313-
url: "https://github.com/microsoft/TypeScript/issues",
2314-
};
2315-
inputPackageJson.repository = {
2316-
type: "git",
2317-
url: "https://github.com/microsoft/TypeScript.git",
2318-
};
2319-
delete inputPackageJson.scripts;
2320-
delete inputPackageJson.devDependencies;
2295+
if (isCI) {
2296+
for (const platform of platforms) {
2297+
await buildNativePreviewPlatform(platform);
2298+
// Build machines have too little space.
2299+
// Clear the Go build cache between platforms.
2300+
await run("go", ["clean", "-cache"]);
2301+
}
23212302
}
2322-
stripSourceConditions(inputPackageJson);
2323-
2324-
if (fs.existsSync(".git")) {
2325-
const { stdout: gitHead } = await runOutput("git", ["rev-parse", "HEAD"]);
2326-
inputPackageJson.gitHead = gitHead.trim();
2303+
else {
2304+
await runWithConcurrencyLimit(platforms.map(platform => () => buildNativePreviewPlatform(platform)), os.availableParallelism());
23272305
}
2328-
inputPackageJson.publishConfig = {
2329-
access: "public",
2330-
tag: getPublishTag(),
2331-
};
2306+
}
2307+
2308+
export const buildMainNativePreviewPackageTask = task({
2309+
name: "typescript:build-main",
2310+
hiddenFromTaskList: true,
2311+
run: async () => {
2312+
if (usePublishedPlatformPackagesForVsix) {
2313+
checkPublishedPlatformPackagesForVsix();
2314+
await rimraf(builtNpm);
2315+
console.log("Skipping npm package build; VSIX packaging will use published platform packages.");
2316+
return;
2317+
}
2318+
2319+
await rimraf(builtNpm);
2320+
await buildMainNativePreviewPackage(getPlatforms());
2321+
},
2322+
});
2323+
2324+
export const buildNativePreviewPlatformTask = task({
2325+
name: "typescript:build-platform",
2326+
hiddenFromTaskList: true,
2327+
run: async () => {
2328+
if (usePublishedPlatformPackagesForVsix) {
2329+
throw new Error("Platform packages are not built when VSIX packaging uses published platform packages.");
2330+
}
2331+
if (!options.platform) {
2332+
throw new Error("typescript:build-platform requires --platform.");
2333+
}
2334+
2335+
const platform = getPlatforms().find(({ nodeOs, nodeArch }) => `${nodeOs}-${nodeArch}` === options.platform);
2336+
if (!platform) {
2337+
throw new Error(`Unknown release platform: ${options.platform}`);
2338+
}
2339+
2340+
await rimraf(platform.npmDir);
2341+
await buildNativePreviewPlatform(platform);
2342+
},
2343+
});
2344+
2345+
export const buildNativePreviewPlatformMatrix = task({
2346+
name: "typescript:build-platform-matrix",
2347+
hiddenFromTaskList: true,
2348+
run: () => {
2349+
const matrix = Object.fromEntries(
2350+
getPlatforms().map(({ nodeOs, nodeArch, npmDirName }) => {
2351+
const platform = `${nodeOs}-${nodeArch}`;
2352+
return [platform.replaceAll("-", "_"), { platform, npmDirName }];
2353+
}),
2354+
);
2355+
console.log(`##vso[task.setvariable variable=matrix;isOutput=true]${JSON.stringify(matrix)}`);
2356+
console.log(`##vso[task.setvariable variable=packageBaseName;isOutput=true]${path.basename(mainNativePreviewPackage.npmDir)}`);
2357+
},
2358+
});
2359+
2360+
/**
2361+
* @param {ReturnType<typeof getPlatforms>} platforms
2362+
*/
2363+
async function buildMainNativePreviewPackage(platforms) {
2364+
const inputDir = "./packages/typescript";
2365+
const inputPackageJson = await createNativePreviewInputPackageJson();
23322366

23332367
const mainPackage = {
23342368
...inputPackageJson,
@@ -2381,65 +2415,98 @@ async function runBuildNativePreviewPackages() {
23812415
if (importErrors.length) {
23822416
throw new Error(`Found external imports in .d.ts files:\n${importErrors.map(e => " " + e).join("\n")}`);
23832417
}
2418+
}
23842419

2385-
const extraFlags = options.respectGoEnv
2386-
? []
2387-
: getReleaseBuildFlags(options.setPrerelease || nativePreviewReleaseVersion ? getVersion() : undefined);
2388-
2389-
const platformBuilders = platforms.map(({ npmDir, npmPackageName, nodeOs, nodeArch, goos, goarch }) => async () => {
2390-
const packageJson = {
2391-
...inputPackageJson,
2392-
bin: undefined,
2393-
files: ["lib", "NOTICE.txt"],
2394-
imports: undefined,
2395-
dependencies: undefined,
2396-
name: npmPackageName,
2397-
os: [nodeOs],
2398-
cpu: [nodeArch],
2399-
exports: {
2400-
"./package.json": "./package.json",
2401-
},
2402-
};
2420+
/**
2421+
* @param {ReturnType<typeof getPlatforms>[number]} platform
2422+
*/
2423+
async function buildNativePreviewPlatform({ npmDir, npmPackageName, nodeOs, nodeArch, goos, goarch }) {
2424+
const inputPackageJson = await createNativePreviewInputPackageJson();
24032425

2404-
const out = path.join(npmDir, "lib");
2405-
await fs.promises.mkdir(out, { recursive: true });
2406-
await fs.promises.writeFile(path.join(npmDir, "package.json"), JSON.stringify(packageJson, undefined, 4));
2407-
await fs.promises.copyFile("LICENSE.txt", path.join(npmDir, "LICENSE"));
2408-
await fs.promises.copyFile("NOTICE.txt", path.join(npmDir, "NOTICE.txt"));
2426+
const packageJson = {
2427+
...inputPackageJson,
2428+
bin: undefined,
2429+
files: ["lib", "NOTICE.txt"],
2430+
imports: undefined,
2431+
dependencies: undefined,
2432+
name: npmPackageName,
2433+
os: [nodeOs],
2434+
cpu: [nodeArch],
2435+
exports: {
2436+
"./package.json": "./package.json",
2437+
},
2438+
};
24092439

2410-
const readme = [
2411-
`# \`${npmPackageName}\``,
2412-
"",
2413-
`This package provides ${nodeOs}-${nodeArch} support for [${mainNativePreviewPackage.npmPackageName}](https://www.npmjs.com/package/${mainNativePreviewPackage.npmPackageName}).`,
2414-
];
2440+
const out = path.join(npmDir, "lib");
2441+
await fs.promises.mkdir(out, { recursive: true });
2442+
await fs.promises.writeFile(path.join(npmDir, "package.json"), JSON.stringify(packageJson, undefined, 4));
2443+
await fs.promises.copyFile("LICENSE.txt", path.join(npmDir, "LICENSE"));
2444+
await fs.promises.copyFile("NOTICE.txt", path.join(npmDir, "NOTICE.txt"));
2445+
2446+
const readme = [
2447+
`# \`${npmPackageName}\``,
2448+
"",
2449+
`This package provides ${nodeOs}-${nodeArch} support for [${mainNativePreviewPackage.npmPackageName}](https://www.npmjs.com/package/${mainNativePreviewPackage.npmPackageName}).`,
2450+
];
24152451

2416-
await fs.promises.writeFile(path.join(npmDir, "README.md"), readme.join("\n") + "\n");
2452+
await fs.promises.writeFile(path.join(npmDir, "README.md"), readme.join("\n") + "\n");
24172453

2418-
await generateLibs(out);
2454+
await generateLibs(out);
24192455

2420-
const exeName = nativePreviewExeName(nodeOs);
2421-
await buildTsc({
2422-
out: publishAsTypescript ? path.join(out, exeName) : out,
2423-
env: {
2424-
GOOS: goos,
2425-
GOARCH: goarch,
2426-
...(options.respectGoEnv ? {} : { GOARM: "6", CGO_ENABLED: "0" }),
2427-
},
2428-
extraFlags,
2429-
});
2456+
const exeName = nativePreviewExeName(nodeOs);
2457+
await buildTsc({
2458+
out: publishAsTypescript ? path.join(out, exeName) : out,
2459+
env: {
2460+
GOOS: goos,
2461+
GOARCH: goarch,
2462+
...(options.respectGoEnv ? {} : { GOARM: "6", CGO_ENABLED: "0" }),
2463+
},
2464+
extraFlags: options.respectGoEnv
2465+
? []
2466+
: getReleaseBuildFlags(options.setPrerelease || nativePreviewReleaseVersion ? getVersion() : undefined),
24302467
});
2468+
}
24312469

2432-
if (isCI) {
2433-
for (const build of platformBuilders) {
2434-
await build();
2435-
// Build machines have too little space.
2436-
// Clear the Go build cache between platforms.
2437-
await run("go", ["clean", "-cache"]);
2438-
}
2470+
async function createNativePreviewInputPackageJson() {
2471+
const inputPackageJson = JSON.parse(fs.readFileSync("./packages/typescript/package.json", "utf8"));
2472+
inputPackageJson.version = getVersion();
2473+
delete inputPackageJson.private;
2474+
inputPackageJson.files = [...new Set([...(inputPackageJson.files ?? []), "NOTICE.txt"])];
2475+
if (publishAsTypescript) {
2476+
inputPackageJson.bin = {
2477+
tsc: "./bin/tsc",
2478+
};
2479+
inputPackageJson.description = "TypeScript is a language for application scale JavaScript development";
2480+
inputPackageJson.homepage = "https://www.typescriptlang.org/";
2481+
inputPackageJson.keywords = [
2482+
"TypeScript",
2483+
"Microsoft",
2484+
"compiler",
2485+
"language",
2486+
"javascript",
2487+
];
2488+
inputPackageJson.bugs = {
2489+
url: "https://github.com/microsoft/TypeScript/issues",
2490+
};
2491+
inputPackageJson.repository = {
2492+
type: "git",
2493+
url: "https://github.com/microsoft/TypeScript.git",
2494+
};
2495+
delete inputPackageJson.scripts;
2496+
delete inputPackageJson.devDependencies;
24392497
}
2440-
else {
2441-
await runWithConcurrencyLimit(platformBuilders, os.availableParallelism());
2498+
stripSourceConditions(inputPackageJson);
2499+
2500+
if (fs.existsSync(".git")) {
2501+
const { stdout: gitHead } = await runOutput("git", ["rev-parse", "HEAD"]);
2502+
inputPackageJson.gitHead = gitHead.trim();
24422503
}
2504+
inputPackageJson.publishConfig = {
2505+
access: "public",
2506+
tag: getPublishTag(),
2507+
};
2508+
2509+
return inputPackageJson;
24432510
}
24442511

24452512
/**

‎tools/pipelines/typescript-build.yml‎

Lines changed: 86 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,68 @@ extends:
5151
- stage: Build
5252
displayName: Build and Sign
5353
jobs:
54+
- job: SetupBuildMatrix
55+
displayName: Set up platform build matrix
56+
57+
templateContext:
58+
mb:
59+
signing:
60+
enabled: false
61+
62+
steps:
63+
- checkout: self
64+
clean: true
65+
submodules: false
66+
fetchDepth: 1
67+
fetchTags: false
68+
- template: /tools/pipelines/steps/setup-node-npm-ci.yml@self
69+
70+
- bash: npx hereby typescript:build-platform-matrix --forRelease --setPrerelease dev.$(Build.BuildNumber)
71+
displayName: Set build matrix
72+
name: buildMatrix
73+
74+
- job: BuildPackages
75+
displayName: Build platform package
76+
dependsOn: SetupBuildMatrix
77+
78+
strategy:
79+
matrix: $[ dependencies.SetupBuildMatrix.outputs['buildMatrix.matrix'] ]
80+
81+
templateContext:
82+
mb:
83+
signing:
84+
enabled: false
85+
outputs:
86+
- output: pipelineArtifact
87+
targetPath: $(Build.SourcesDirectory)/built/npm/$(npmDirName)
88+
artifactName: $(npmDirName)
89+
isProduction: false
90+
sbomEnabled: false
91+
92+
steps:
93+
- checkout: self
94+
clean: true
95+
submodules: false
96+
fetchDepth: 1
97+
fetchTags: false
98+
- template: /tools/pipelines/steps/setup-node-npm-ci.yml@self
99+
- template: /tools/pipelines/steps/setup-go.yml@self
100+
101+
- bash: npx hereby typescript:build-platform --platform $(platform) --forRelease --setPrerelease dev.$(Build.BuildNumber)
102+
displayName: Build $(platform)
103+
54104
- job: Build
55105
displayName: Build and Sign
106+
dependsOn:
107+
- SetupBuildMatrix
108+
- BuildPackages
56109
timeoutInMinutes: 180
57110
pool:
58111
name: AzurePipelines-EO
59112
image: 1ESPT-Ubuntu22.04
60113
os: linux
114+
variables:
115+
packageBaseName: $[ dependencies.SetupBuildMatrix.outputs['buildMatrix.packageBaseName'] ]
61116

62117
templateContext:
63118
mb:
@@ -136,8 +191,37 @@ extends:
136191
137192
- template: /tools/pipelines/steps/setup-go.yml@self
138193

139-
- bash: npx hereby typescript:build --forRelease --setPrerelease dev.$(initialBuildNumber)
140-
displayName: 'Build packages'
194+
- bash: npx hereby typescript:build-main --forRelease --setPrerelease dev.$(initialBuildNumber)
195+
displayName: 'Build main package'
196+
197+
- task: DownloadPipelineArtifact@2
198+
displayName: 'Download platform packages'
199+
inputs:
200+
buildType: current
201+
targetPath: $(Pipeline.Workspace)/platform-packages
202+
patterns: '$(packageBaseName)-*/**'
203+
204+
- bash: |
205+
set -euo pipefail
206+
shopt -s nullglob
207+
platformArtifacts=("$(Pipeline.Workspace)"/platform-packages/"$PACKAGE_BASE_NAME"-*)
208+
if (( ${#platformArtifacts[@]} == 0 )); then
209+
echo "No platform package artifacts were downloaded." >&2
210+
exit 1
211+
fi
212+
for artifact in "${platformArtifacts[@]}"; do
213+
packageDir="built/npm/$(basename "$artifact")"
214+
cp -a "$artifact" "$packageDir"
215+
executable="$(find "$packageDir/lib" -maxdepth 1 -type f \( -name tsc -o -name tsc.exe -o -name tsgo -o -name tsgo.exe \) -print -quit)"
216+
if [[ -z "$executable" ]]; then
217+
echo "Missing platform executable in $packageDir." >&2
218+
exit 1
219+
fi
220+
chmod +x "$executable"
221+
done
222+
displayName: 'Assemble platform packages'
223+
env:
224+
PACKAGE_BASE_NAME: $(packageBaseName)
141225
142226
- bash: npx hereby typescript:test-package --forRelease --setPrerelease dev.$(initialBuildNumber)
143227
displayName: 'Test packages'

0 commit comments

Comments
 (0)