[7.x] [shared-ui-deps] use a single global version of lodash (#78100) (…

…#78454)

Co-authored-by: spalger <spalger@users.noreply.github.com>

packages/kbn-pm/src/commands/bootstrap.test.ts

@@ -19,6 +19,7 @@
 
 jest.mock('../utils/scripts');
 jest.mock('../utils/link_project_executables');
+jest.mock('../utils/validate_yarn_lock');
 
 import { resolve } from 'path';
 

packages/kbn-pm/src/commands/bootstrap.ts

@@ -25,6 +25,8 @@ import { Project } from '../utils/project';
 import { ICommand } from './';
 import { getAllChecksums } from '../utils/project_checksums';
 import { BootstrapCacheFile } from '../utils/bootstrap_cache_file';
+import { readYarnLock } from '../utils/yarn_lock';
+import { validateYarnLock } from '../utils/validate_yarn_lock';
 
 export const BootstrapCommand: ICommand = {
   description: 'Install dependencies and crosslink projects',
@@ -54,6 +56,10 @@ export const BootstrapCommand: ICommand = {
       }
     }
 
+    const yarnLock = await readYarnLock(kbn);
+
+    await validateYarnLock(kbn, yarnLock);
+
     await linkProjectExecutables(projects, projectGraph);
 
     /**
@@ -63,7 +69,7 @@ export const BootstrapCommand: ICommand = {
      * have to, as it will slow down the bootstrapping process.
      */
 
-    const checksums = await getAllChecksums(kbn, log);
+    const checksums = await getAllChecksums(kbn, log, yarnLock);
     const caches = new Map<Project, { file: BootstrapCacheFile; valid: boolean }>();
     let cachedProjectCount = 0;
 

packages/kbn-pm/src/utils/fs.ts

@@ -25,6 +25,7 @@ import { promisify } from 'util';
 
 const lstat = promisify(fs.lstat);
 export const readFile = promisify(fs.readFile);
+export const writeFile = promisify(fs.writeFile);
 const symlink = promisify(fs.symlink);
 export const chmod = promisify(fs.chmod);
 const cmdShim = promisify<string, string>(cmdShimCb);

packages/kbn-pm/src/utils/kibana.ts

@@ -22,6 +22,8 @@ import Path from 'path';
 import multimatch from 'multimatch';
 import isPathInside from 'is-path-inside';
 
+import { resolveDepsForProject, YarnLock } from './yarn_lock';
+import { Log } from './log';
 import { ProjectMap, getProjects, includeTransitiveProjects } from './projects';
 import { Project } from './project';
 import { getProjectPaths } from '../config';
@@ -133,4 +135,26 @@ export class Kibana {
   isOutsideRepo(project: Project) {
     return !this.isPartOfRepo(project);
   }
+
+  resolveAllProductionDependencies(yarnLock: YarnLock, log: Log) {
+    const kibanaDeps = resolveDepsForProject({
+      project: this.kibanaProject,
+      yarnLock,
+      kbn: this,
+      includeDependentProject: true,
+      productionDepsOnly: true,
+      log,
+    })!;
+
+    const xpackDeps = resolveDepsForProject({
+      project: this.getProject('x-pack')!,
+      yarnLock,
+      kbn: this,
+      includeDependentProject: true,
+      productionDepsOnly: true,
+      log,
+    })!;
+
+    return new Map([...kibanaDeps.entries(), ...xpackDeps.entries()]);
+  }
 }

packages/kbn-pm/src/utils/project_checksums.ts

@@ -24,7 +24,7 @@ import { promisify } from 'util';
 
 import execa from 'execa';
 
-import { readYarnLock, YarnLock } from './yarn_lock';
+import { YarnLock, resolveDepsForProject } from './yarn_lock';
 import { ProjectMap } from '../utils/projects';
 import { Project } from '../utils/project';
 import { Kibana } from '../utils/kibana';
@@ -145,51 +145,6 @@ async function getLatestSha(project: Project, kbn: Kibana) {
   return stdout.trim() || undefined;
 }
 
-/**
- * Get a list of the absolute dependencies of this project, as resolved
- * in the yarn.lock file, does not include other projects in the workspace
- * or their dependencies
- */
-function resolveDepsForProject(project: Project, yarnLock: YarnLock, kbn: Kibana, log: Log) {
-  /** map of [name@range, name@resolved] */
-  const resolved = new Map<string, string>();
-
-  const queue: Array<[string, string]> = Object.entries(project.allDependencies);
-
-  while (queue.length) {
-    const [name, versionRange] = queue.shift()!;
-    const req = `${name}@${versionRange}`;
-
-    if (resolved.has(req)) {
-      continue;
-    }
-
-    if (!kbn.hasProject(name)) {
-      const pkg = yarnLock[req];
-      if (!pkg) {
-        log.warning(
-          'yarn.lock file is out of date, please run `yarn kbn bootstrap` to re-enable caching'
-        );
-        return;
-      }
-
-      const res = `${name}@${pkg.version}`;
-      resolved.set(req, res);
-
-      const allDepsEntries = [
-        ...Object.entries(pkg.dependencies || {}),
-        ...Object.entries(pkg.optionalDependencies || {}),
-      ];
-
-      for (const [childName, childVersionRange] of allDepsEntries) {
-        queue.push([childName, childVersionRange]);
-      }
-    }
-  }
-
-  return Array.from(resolved.values()).sort((a, b) => a.localeCompare(b));
-}
-
 /**
  * Get the checksum for a specific project in the workspace
  */
@@ -224,11 +179,22 @@ async function getChecksum(
       })
   );
 
-  const deps = await resolveDepsForProject(project, yarnLock, kbn, log);
-  if (!deps) {
+  const depMap = resolveDepsForProject({
+    project,
+    yarnLock,
+    kbn,
+    log,
+    includeDependentProject: false,
+    productionDepsOnly: false,
+  });
+  if (!depMap) {
     return;
   }
 
+  const deps = Array.from(depMap.values())
+    .map(({ name, version }) => `${name}@${version}`)
+    .sort((a, b) => a.localeCompare(b));
+
   log.verbose(`[${project.name}] resolved %d deps`, deps.length);
 
   const checksum = JSON.stringify(
@@ -256,10 +222,9 @@ async function getChecksum(
  *  - un-committed changes
  *  - resolved dependencies from yarn.lock referenced by project package.json
  */
-export async function getAllChecksums(kbn: Kibana, log: Log) {
+export async function getAllChecksums(kbn: Kibana, log: Log, yarnLock: YarnLock) {
   const projects = kbn.getAllProjects();
   const changesByProject = await getChangesForProjects(projects, kbn, log);
-  const yarnLock = await readYarnLock(kbn);
 
   /** map of [project.name, cacheKey] */
   const cacheKeys: ChecksumMap = new Map();

packages/kbn-pm/src/utils/validate_yarn_lock.ts

@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// @ts-expect-error published types are useless
+import { stringify as stringifyLockfile } from '@yarnpkg/lockfile';
+import dedent from 'dedent';
+
+import { writeFile } from './fs';
+import { Kibana } from './kibana';
+import { YarnLock } from './yarn_lock';
+import { log } from './log';
+
+export async function validateYarnLock(kbn: Kibana, yarnLock: YarnLock) {
+  // look through all of the packages in the yarn.lock file to see if
+  // we have accidentally installed multiple lodash v4 versions
+  const lodash4Versions = new Set<string>();
+  const lodash4Reqs = new Set<string>();
+  for (const [req, dep] of Object.entries(yarnLock)) {
+    if (req.startsWith('lodash@') && dep.version.startsWith('4.')) {
+      lodash4Reqs.add(req);
+      lodash4Versions.add(dep.version);
+    }
+  }
+
+  // if we find more than one lodash v4 version installed then delete
+  // lodash v4 requests from the yarn.lock file and prompt the user to
+  // retry bootstrap so that a single v4 version will be installed
+  if (lodash4Versions.size > 1) {
+    for (const req of lodash4Reqs) {
+      delete yarnLock[req];
+    }
+
+    await writeFile(kbn.getAbsolute('yarn.lock'), stringifyLockfile(yarnLock), 'utf8');
+
+    log.error(dedent`
+
+      Multiple version of lodash v4 were detected, so they have been removed
+      from the yarn.lock file. Please rerun yarn kbn bootstrap to coalese the
+      lodash versions installed.
+
+      If you still see this error when you re-bootstrap then you might need
+      to force a new dependency to use the latest version of lodash via the
+      "resolutions" field in package.json.
+
+      If you have questions about this please reach out to the operations team.
+
+    `);
+
+    process.exit(1);
+  }
+
+  // look through all the dependencies of production packages and production
+  // dependencies of those packages to determine if we're shipping any versions
+  // of lodash v3 in the distributable
+  const prodDependencies = kbn.resolveAllProductionDependencies(yarnLock, log);
+  const lodash3Versions = new Set<string>();
+  for (const dep of prodDependencies.values()) {
+    if (dep.name === 'lodash' && dep.version.startsWith('3.')) {
+      lodash3Versions.add(dep.version);
+    }
+  }
+
+  // if any lodash v3 packages were found we abort and tell the user to fix things
+  if (lodash3Versions.size) {
+    log.error(dedent`
+
+      Due to changes in the yarn.lock file and/or package.json files a version of
+      lodash 3 is now included in the production dependencies. To reduce the size of
+      our distributable and especially our front-end bundles we have decided to
+      prevent adding any new instances of lodash 3.
+
+      Please inspect the changes to yarn.lock or package.json files to identify where
+      the lodash 3 version is coming from and remove it.
+
+      If you have questions about this please reack out to the operations team.
+
+    `);
+
+    process.exit(1);
+  }
+
+  log.success('yarn.lock analysis completed without any issues');
+}

packages/kbn-pm/src/utils/yarn_lock.ts

@@ -17,14 +17,16 @@
  * under the License.
  */
 
-// @ts-ignore published types are worthless
+// @ts-expect-error published types are worthless
 import { parse as parseLockfile } from '@yarnpkg/lockfile';
 
 import { readFile } from '../utils/fs';
 import { Kibana } from '../utils/kibana';
+import { Project } from '../utils/project';
+import { Log } from '../utils/log';
 
 export interface YarnLock {
-  /** a simple map of version@versionrange tags to metadata about a package */
+  /** a simple map of name@versionrange tags to metadata about a package */
   [key: string]: {
     /** resolved version installed for this pacakge */
     version: string;
@@ -61,3 +63,82 @@ export async function readYarnLock(kbn: Kibana): Promise<YarnLock> {
 
   return {};
 }
+
+/**
+ * Get a list of the absolute dependencies of this project, as resolved
+ * in the yarn.lock file, does not include other projects in the workspace
+ * or their dependencies
+ */
+export function resolveDepsForProject({
+  project: rootProject,
+  yarnLock,
+  kbn,
+  log,
+  productionDepsOnly,
+  includeDependentProject,
+}: {
+  project: Project;
+  yarnLock: YarnLock;
+  kbn: Kibana;
+  log: Log;
+  productionDepsOnly: boolean;
+  includeDependentProject: boolean;
+}) {
+  /** map of [name@range, { name, version }] */
+  const resolved = new Map<string, { name: string; version: string }>();
+
+  const seenProjects = new Set<Project>();
+  const projectQueue: Project[] = [rootProject];
+  const depQueue: Array<[string, string]> = [];
+
+  while (projectQueue.length) {
+    const project = projectQueue.shift()!;
+    if (seenProjects.has(project)) {
+      continue;
+    }
+    seenProjects.add(project);
+
+    const projectDeps = Object.entries(
+      productionDepsOnly ? project.productionDependencies : project.allDependencies
+    );
+    for (const [name, versionRange] of projectDeps) {
+      depQueue.push([name, versionRange]);
+    }
+
+    while (depQueue.length) {
+      const [name, versionRange] = depQueue.shift()!;
+      const req = `${name}@${versionRange}`;
+
+      if (resolved.has(req)) {
+        continue;
+      }
+
+      if (includeDependentProject && kbn.hasProject(name)) {
+        projectQueue.push(kbn.getProject(name)!);
+      }
+
+      if (!kbn.hasProject(name)) {
+        const pkg = yarnLock[req];
+        if (!pkg) {
+          log.warning(
+            'yarn.lock file is out of date, please run `yarn kbn bootstrap` to re-enable caching'
+          );
+          return;
+        }
+
+        resolved.set(req, { name, version: pkg.version });
+
+        const allDepsEntries = [
+          ...Object.entries(pkg.dependencies || {}),
+          ...Object.entries(pkg.optionalDependencies || {}),
+        ];
+
+        for (const [childName, childVersionRange] of allDepsEntries) {
+          depQueue.push([childName, childVersionRange]);
+        }
+      }
+    }
+  }
+
+  return resolved;
+}