improvement(build-tools): Convert build-cli to ESM (#20972)

Updates the build-cli project to be ESM instead of CJS. For the
oclif-related changes I followed this guide:
https://oclif.io/docs/esm/#migrating-a-cjs-plugin-to-esm

Summary of changes:

- Ran ts2esm to add extensions to source file import statements.
- Upgraded chalk, fs-extra, and node-fetch to more recent versions with
better ESM support.
- jssm's types aren't set up quite right for TypeScript to resolve them
when using Node16/NodeNext module resolution, so I patched in support. I
opened StoneCypher/fsl#1295 to discuss options
for upstreaming the changes.
- I've limited the patched files to package.json, rollup
configs, and the two new rolled up declaration files. Critically, this
means the patch has _no runtime changes_. The only changes are in the
types and config files. This should make it almost impossible for there
to have been runtime bugs introduced by these changes.
- Updated the install build tools step in the CI workflows to use npm
link to make flub globally available. I removed the hack shim we only
used in CI.
- Updated the syncpack config to account for new dependencies.
- Regenerated the packlists. This should be done as a part of the build
but it isn't yet so I updated them opportunistically.
- Renamed the dangerfile to .cts so it's always CommonJS like it used to
be. Seemed the safest approach. If we want it to be ESM we can make that
change separately.

build-tools/package.json

@@ -69,9 +69,9 @@
 		"@commitlint/cli": "^17.6.6",
 		"@commitlint/config-conventional": "^17.6.6",
 		"@commitlint/cz-commitlint": "^17.5.0",
-		"@fluid-tools/build-cli": "~0.29.0",
+		"@fluid-tools/build-cli": "~0.38.0",
 		"@fluidframework/build-common": "^2.0.3",
-		"@fluidframework/build-tools": "~0.29.0",
+		"@fluidframework/build-tools": "~0.38.0",
 		"@microsoft/api-documenter": "^7.22.24",
 		"@microsoft/api-extractor": "^7.43.1",
 		"c8": "^7.14.0",
@@ -91,7 +91,7 @@
 	},
 	"packageManager": "pnpm@8.15.8+sha512.d1a029e1a447ad90bc96cd58b0fad486d2993d531856396f7babf2d83eb1823bb83c5a3d0fc18f675b2d10321d49eb161fece36fe8134aa5823ecd215feed392",
 	"engines": {
-		"node": ">=14.17.0"
+		"node": ">=18.17.1"
 	},
 	"dependenciesComments": {
 		"@fluidframework/build-tools": "Provides fluid-build which is used by the `build:fast` script. To allow this to work when the workspace version of build-tools has not been built yet, a prepackaged version is depended on. If this self-dependency becomes problematic, the `build` script can be used which does not require it."
@@ -113,6 +113,9 @@
 			"qs": "^6.11.0",
 			"sharp": "^0.33.2"
 		},
+		"patchedDependencies": {
+			"jssm@5.98.2": "patches/jssm@5.98.2.patch"
+		},
 		"updateConfig": {
 			"ignoreDependencies": [
 				"latest-version",

build-tools/packages/build-cli/.eslintrc.cjs

@@ -22,12 +22,26 @@ module.exports = {
 		// oclif uses default exports for commands
 		"import/no-default-export": "off",
 
-		// This package uses interfaces and types that are not exposed directly by npm-check-updates.
-		// We also call commands' run method directly in some cases, so these are all excluded.
 		"import/no-internal-modules": [
 			"error",
 			{
-				allow: ["npm-check-updates/build/src/types/**", "**/commands/**"],
+				allow: [
+					// fs-extra's ./esm export is needed for ESM support.
+					"fs-extra/esm",
+
+					// This package uses interfaces and types that are not exposed directly by npm-check-updates.
+					"npm-check-updates/build/src/types/**",
+
+					// We call oclif commands' run method directly in some cases, so these are all excluded.
+					"**/commands/**",
+
+					// These are all excluded because they're "submodules" used for organization.
+					// AB#8118 tracks removing the barrel files and importing directly from the submodules.
+					"**/library/index.js",
+					"**/handlers/index.js",
+					"**/machines/index.js",
+					"**/repoPolicyCheck/index.js",
+				],
 			},
 		],
 
@@ -50,9 +64,6 @@ module.exports = {
 		// In commands, destructuring is useful in some places but makes others less legible, so consistency isn't preferred.
 		"unicorn/consistent-destructuring": "off",
 
-		// This package is currently CJS-only.
-		"unicorn/prefer-module": "off",
-
 		// Deprecated in 2018: https://eslint.org/blog/2018/11/jsdoc-end-of-life/
 		"valid-jsdoc": "off",
 

build-tools/packages/build-cli/.mocharc.json

@@ -1,7 +1,8 @@
 {
 	"require": ["ts-node/register"],
-	"watch-extensions": ["ts"],
+	"watch-extensions": ["ts", "cts", "mts"],
 	"recursive": true,
 	"reporter": "spec",
-	"timeout": 60000
+	"timeout": 60000,
+	"node-option": ["loader=ts-node/esm"]
 }

build-tools/packages/build-cli/bin/dev.js

@@ -1,11 +1,9 @@
-#!/usr/bin/env ts-node
+#!/usr/bin/env -S node --loader ts-node/esm --no-warnings=ExperimentalWarning
 /*!
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
 
-// eslint-disable-next-line node/shebang, unicorn/prefer-top-level-await
-(async () => {
-	const oclif = await import("@oclif/core");
-	await oclif.execute({ development: true, dir: __dirname });
-})();
+import { execute } from "@oclif/core";
+
+await execute({ development: true, dir: import.meta.url });

build-tools/packages/build-cli/bin/run.js

@@ -4,8 +4,6 @@
  * Licensed under the MIT License.
  */
 
-// eslint-disable-next-line unicorn/prefer-top-level-await
-(async () => {
-	const oclif = await import("@oclif/core");
-	await oclif.execute({ development: false, dir: __dirname });
-})();
+import { execute } from "@oclif/core";
+
+await execute({ dir: import.meta.url });

build-tools/packages/build-cli/package.json

@@ -11,7 +11,7 @@
 	"license": "MIT",
 	"author": "Microsoft and contributors",
 	"sideEffects": false,
-	"type": "commonjs",
+	"type": "module",
 	"exports": {
 		".": {
 			"default": {
@@ -76,22 +76,22 @@
 		"@octokit/core": "^4.2.4",
 		"@rushstack/node-core-library": "^3.59.5",
 		"async": "^3.2.4",
-		"chalk": "^2.4.2",
+		"chalk": "^5.3.0",
 		"change-case": "^3.1.0",
 		"danger": "^11.3.0",
 		"date-fns": "^2.30.0",
 		"debug": "^4.3.4",
 		"execa": "^5.1.1",
-		"fs-extra": "^9.1.0",
+		"fs-extra": "^11.2.0",
 		"globby": "^11.1.0",
 		"gray-matter": "^4.0.3",
 		"human-id": "^4.0.0",
 		"inquirer": "^8.2.5",
 		"json5": "^2.2.3",
-		"jssm": "^5.89.2",
+		"jssm": "5.98.2",
 		"latest-version": "^5.1.0",
 		"minimatch": "^7.4.6",
-		"node-fetch": "^2.6.9",
+		"node-fetch": "^3.3.2",
 		"npm-check-updates": "^16.14.20",
 		"oclif": "^4.10.1",
 		"prettier": "~3.2.4",
@@ -112,13 +112,13 @@
 	"devDependencies": {
 		"@fluid-private/readme-command": "workspace:~",
 		"@fluidframework/build-common": "^2.0.3",
-		"@fluidframework/eslint-config-fluid": "^5.2.0",
+		"@fluidframework/eslint-config-fluid": "^5.3.0",
 		"@oclif/test": "^3.2.12",
 		"@types/async": "^3.2.20",
 		"@types/chai": "^4.3.5",
 		"@types/chai-arrays": "^2.0.0",
 		"@types/debug": "^4.1.7",
-		"@types/fs-extra": "^8.1.2",
+		"@types/fs-extra": "^11.0.4",
 		"@types/inquirer": "^8.2.6",
 		"@types/mocha": "^9.1.1",
 		"@types/node": "^18.18.6",
@@ -137,7 +137,7 @@
 		"eslint-config-oclif": "^5.2.0",
 		"eslint-config-oclif-typescript": "^3.1.7",
 		"eslint-config-prettier": "~9.1.0",
-		"jssm-viz-cli": "^5.89.2",
+		"jssm-viz-cli": "5.97.1",
 		"mocha": "^10.2.0",
 		"mocha-json-output-reporter": "^2.1.0",
 		"mocha-multi-reporters": "^1.5.1",
@@ -148,7 +148,7 @@
 		"typescript": "~5.1.6"
 	},
 	"engines": {
-		"node": ">=14.17.0"
+		"node": ">=18.17.1"
 	},
 	"fluidBuild": {
 		"tasks": {

build-tools/packages/build-cli/packlist.txt

@@ -4,7 +4,6 @@ oclif.manifest.json
 package.json
 bin/dev.cmd
 bin/dev.js
-bin/flub
 bin/run.cmd
 bin/run.js
 lib/BasePackageCommand.d.ts
@@ -15,10 +14,6 @@ lib/args.d.ts
 lib/args.d.ts.map
 lib/args.js
 lib/args.js.map
-lib/base.d.ts
-lib/base.d.ts.map
-lib/base.js
-lib/base.js.map
 lib/commands/bump.d.ts
 lib/commands/bump.d.ts.map
 lib/commands/bump.js
@@ -75,6 +70,10 @@ lib/commands/generate/packlist.d.ts
 lib/commands/generate/packlist.d.ts.map
 lib/commands/generate/packlist.js
 lib/commands/generate/packlist.js.map
+lib/commands/generate/typetests.d.ts
+lib/commands/generate/typetests.d.ts.map
+lib/commands/generate/typetests.js
+lib/commands/generate/typetests.js.map
 lib/commands/generate/upcoming.d.ts
 lib/commands/generate/upcoming.d.ts.map
 lib/commands/generate/upcoming.js
@@ -99,6 +98,10 @@ lib/commands/modify/fluid-imports.d.ts
 lib/commands/modify/fluid-imports.d.ts.map
 lib/commands/modify/fluid-imports.js
 lib/commands/modify/fluid-imports.js.map
+lib/commands/modify/lockfile.d.ts
+lib/commands/modify/lockfile.d.ts.map
+lib/commands/modify/lockfile.js
+lib/commands/modify/lockfile.js.map
 lib/commands/release.d.ts
 lib/commands/release.d.ts.map
 lib/commands/release.js
@@ -211,14 +214,30 @@ lib/library/collections.d.ts
 lib/library/collections.d.ts.map
 lib/library/collections.js
 lib/library/collections.js.map
+lib/library/commands/base.d.ts
+lib/library/commands/base.d.ts.map
+lib/library/commands/base.js
+lib/library/commands/base.js.map
+lib/library/commands/constants.d.ts
+lib/library/commands/constants.d.ts.map
+lib/library/commands/constants.js
+lib/library/commands/constants.js.map
+lib/library/commands/generateEntrypoints.d.ts
+lib/library/commands/generateEntrypoints.d.ts.map
+lib/library/commands/generateEntrypoints.js
+lib/library/commands/generateEntrypoints.js.map
+lib/library/commands/index.d.ts
+lib/library/commands/index.d.ts.map
+lib/library/commands/index.js
+lib/library/commands/index.js.map
 lib/library/context.d.ts
 lib/library/context.d.ts.map
 lib/library/context.js
 lib/library/context.js.map
-lib/library/dangerfile.d.ts
-lib/library/dangerfile.d.ts.map
-lib/library/dangerfile.js
-lib/library/dangerfile.js.map
+lib/library/dangerfile.cjs
+lib/library/dangerfile.cjs.map
+lib/library/dangerfile.d.cts
+lib/library/dangerfile.d.cts.map
 lib/library/dates.d.ts
 lib/library/dates.d.ts.map
 lib/library/dates.js
@@ -243,6 +262,10 @@ lib/library/package.d.ts
 lib/library/package.d.ts.map
 lib/library/package.js
 lib/library/package.js.map
+lib/library/packageExports.d.ts
+lib/library/packageExports.d.ts.map
+lib/library/packageExports.js
+lib/library/packageExports.js.map
 lib/library/release.d.ts
 lib/library/release.d.ts.map
 lib/library/release.js
@@ -263,6 +286,10 @@ lib/library/repoPolicyCheck/dockerfilePackages.d.ts
 lib/library/repoPolicyCheck/dockerfilePackages.d.ts.map
 lib/library/repoPolicyCheck/dockerfilePackages.js
 lib/library/repoPolicyCheck/dockerfilePackages.js.map
+lib/library/repoPolicyCheck/fluidBuildDatabase.d.ts
+lib/library/repoPolicyCheck/fluidBuildDatabase.d.ts.map
+lib/library/repoPolicyCheck/fluidBuildDatabase.js
+lib/library/repoPolicyCheck/fluidBuildDatabase.js.map
 lib/library/repoPolicyCheck/fluidBuildTasks.d.ts
 lib/library/repoPolicyCheck/fluidBuildTasks.d.ts.map
 lib/library/repoPolicyCheck/fluidBuildTasks.js

build-tools/packages/build-cli/src/BasePackageCommand.ts

@@ -15,9 +15,9 @@ import {
 	parsePackageFilterFlags,
 	parsePackageSelectionFlags,
 	selectAndFilterPackages,
-} from "./filter";
-import { type PackageSelectionDefault, filterFlags, selectionFlags } from "./flags";
-import { BaseCommand } from "./library";
+} from "./filter.js";
+import { type PackageSelectionDefault, filterFlags, selectionFlags } from "./flags.js";
+import { BaseCommand } from "./library/index.js";
 
 /**
  * Commands that run operations per project.

build-tools/packages/build-cli/src/args.ts

@@ -6,7 +6,7 @@
 import { MonoRepo, Package } from "@fluidframework/build-tools";
 import { Args } from "@oclif/core";
 // eslint-disable-next-line import/no-deprecated
-import { Context, isMonoRepoKind } from "./library";
+import { Context, isMonoRepoKind } from "./library/index.js";
 
 /**
  * A re-usable CLI argument for package or release group names.

build-tools/packages/build-cli/src/commands/bump.ts

@@ -23,14 +23,14 @@ import {
 	isInterdependencyRange,
 } from "@fluid-tools/version-tools";
 
-import { findPackageOrReleaseGroup, packageOrReleaseGroupArg } from "../args";
-import { bumpTypeFlag, checkFlags, skipCheckFlag, versionSchemeFlag } from "../flags";
+import { findPackageOrReleaseGroup, packageOrReleaseGroupArg } from "../args.js";
+import { bumpTypeFlag, checkFlags, skipCheckFlag, versionSchemeFlag } from "../flags.js";
 import {
 	BaseCommand,
 	generateBumpVersionBranchName,
 	generateBumpVersionCommitMessage,
 	setVersion,
-} from "../library";
+} from "../library/index.js";
 
 export default class BumpCommand extends BaseCommand<typeof BumpCommand> {
 	static readonly summary =

build-tools/packages/build-cli/src/commands/bump/deps.ts

@@ -10,15 +10,15 @@ import stripAnsi from "strip-ansi";
 
 import { FluidRepo, MonoRepo } from "@fluidframework/build-tools";
 
-import { findPackageOrReleaseGroup, packageOrReleaseGroupArg } from "../../args";
+import { findPackageOrReleaseGroup, packageOrReleaseGroupArg } from "../../args.js";
 import {
 	checkFlags,
 	dependencyUpdateTypeFlag,
 	packageSelectorFlag,
 	releaseGroupFlag,
 	skipCheckFlag,
 	testModeFlag,
-} from "../../flags";
+} from "../../flags.js";
 import {
 	BaseCommand,
 	// eslint-disable-next-line import/no-deprecated
@@ -28,10 +28,10 @@ import {
 	indentString,
 	isDependencyUpdateType,
 	npmCheckUpdates,
-} from "../../library";
+} from "../../library/index.js";
 // eslint-disable-next-line import/no-internal-modules
-import { npmCheckUpdatesHomegrown } from "../../library/package";
-import { ReleaseGroup } from "../../releaseGroups";
+import { npmCheckUpdatesHomegrown } from "../../library/package.js";
+import { ReleaseGroup } from "../../releaseGroups.js";
 
 /**
  * Update the dependency version of a specified package or release group. That is, if one or more packages in the repo

build-tools/packages/build-cli/src/commands/check/buildVersion.ts

@@ -7,7 +7,7 @@ import path from "node:path";
 import { Package } from "@fluidframework/build-tools";
 import { Flags } from "@oclif/core";
 
-import { PackageCommand } from "../../BasePackageCommand";
+import { PackageCommand } from "../../BasePackageCommand.js";
 
 export default class CheckBuildVersionCommand extends PackageCommand<
 	typeof CheckBuildVersionCommand

build-tools/packages/build-cli/src/commands/check/changeset.ts

@@ -7,7 +7,7 @@ import { Flags } from "@oclif/core";
 import chalk from "chalk";
 import { sortPackageJson as sortJson } from "sort-package-json";
 
-import { BaseCommand, Repository } from "../../library";
+import { BaseCommand, Repository } from "../../library/index.js";
 
 export default class CheckChangesetCommand extends BaseCommand<typeof CheckChangesetCommand> {
 	static readonly summary =

build-tools/packages/build-cli/src/commands/check/layers.ts

@@ -9,7 +9,7 @@ import path from "node:path";
 import { Timer } from "@fluidframework/build-tools";
 import { Flags } from "@oclif/core";
 
-import { BaseCommand, LayerGraph } from "../../library";
+import { BaseCommand, LayerGraph } from "../../library/index.js";
 
 const packagesMdFileName = "PACKAGES.md";