feat: add support for melos_overrides.yaml + `command/bootstrap/dep…

…endencyOverridePaths` (#410)

Co-authored-by: Gabriel Terwesten <gabriel@terwesten.net>

docs/configuration/overview.mdx

@@ -8,6 +8,10 @@ description: Configure Melos using the `melos.yaml` file.
 Every project requires a `melos.yaml` project in the root. The below outlines
 all the configurable fields and their purpose.
 
+Additionally, projects may include a `melos_overrides.yaml` file to override any
+`melos.yaml` field. This is useful for making untracked customizations to a
+project.
+
 ## `name`
 
 > required
@@ -155,6 +159,22 @@ Configuration relating to specific Melos commands such as versioning.
 
 Configuration for the `bootstrap` command.
 
+### `command/bootstrap/dependencyOverridePaths`
+
+> optional
+
+A list of paths to local packages realtive to the workspace directory that
+should be added to each workspace package's dependency overrides. Each entry can
+be a specific path or a [glob] pattern.
+
+**Tip:** External local packages can be referenced using paths relative to the
+workspace root.
+
+```yaml
+dependencyOverridePaths:
+  - '../external_project/packages/**'
+```
+
 ### `command/bootstrap/runPubGetInParallel`
 
 Whether to run `pub get` in parallel during bootstrapping.

packages/melos/lib/src/commands/bootstrap.dart

@@ -101,18 +101,30 @@ mixin _BootstrapMixin on _CleanMixin {
   ) async {
     final allTransitiveDependencies =
         package.allTransitiveDependenciesInWorkspace;
-    final melosDependencyOverrides = {...package.pubSpec.dependencyOverrides};
+    final melosDependencyOverrides = <String, DependencyReference>{};
 
     // Traversing all packages so that transitive dependencies for the
-    // bootstraped packages are setup properly.
+    // bootstrapped packages are setup properly.
     for (final otherPackage in workspace.allPackages.values) {
-      if (allTransitiveDependencies.containsKey(otherPackage.name) &&
-          !melosDependencyOverrides.containsKey(otherPackage.name)) {
+      if (allTransitiveDependencies.containsKey(otherPackage.name)) {
         melosDependencyOverrides[otherPackage.name] =
             PathReference(utils.relativePath(otherPackage.path, package.path));
       }
     }
 
+    // Add custom workspace overrides.
+    for (final dependencyOverride
+        in workspace.dependencyOverridePackages.values) {
+      melosDependencyOverrides[dependencyOverride.name] = PathReference(
+        utils.relativePath(dependencyOverride.path, package.path),
+      );
+    }
+
+    // Add existing dependency overrides from pubspec.yaml last, overwriting
+    // overrides that would be made by Melos, to provide granular control at a
+    // package level.
+    melosDependencyOverrides.addAll(package.pubSpec.dependencyOverrides);
+
     // Load current pubspec_overrides.yaml.
     final pubspecOverridesFile =
         utils.pubspecOverridesPathForDirectory(package.path);

packages/melos/lib/src/common/utils.dart

@@ -297,6 +297,9 @@ String melosYamlPathForDirectory(String directory) =>
 String melosStatePathForDirectory(String directory) =>
     p.join(directory, '.melos');
 
+String melosOverridesYamlPathForDirectory(String directory) =>
+    p.join(directory, 'melos_overrides.yaml');
+
 String pubspecPathForDirectory(String directory) =>
     p.join(directory, 'pubspec.yaml');
 
@@ -347,6 +350,53 @@ String listAsPaddedTable(List<List<String>> table, {int paddingSize = 1}) {
   return output.join('\n');
 }
 
+extension YamlUtils on YamlNode {
+  /// Converts a YAML node to a regular mutable Dart object.
+  Object? toPlainObject() {
+    final node = this;
+    if (node is YamlScalar) {
+      return node.value;
+    }
+    if (node is YamlMap) {
+      return {
+        for (final entry in node.nodes.entries)
+          (entry.key as YamlNode).toPlainObject(): entry.value.toPlainObject(),
+      };
+    }
+    if (node is YamlList) {
+      return node.nodes.map((node) => node.toPlainObject()).toList();
+    }
+    throw FormatException(
+      'Unsupported YAML node type encountered: ${node.runtimeType}',
+      this,
+    );
+  }
+}
+
+/// Merges two maps together, overriding any values in [base] with those
+/// with the same key in [overlay].
+void mergeMap(Map<Object?, Object?> base, Map<Object?, Object?> overlay) {
+  for (final entry in overlay.entries) {
+    final overlayValue = entry.value;
+    final baseValue = base[entry.key];
+    if (overlayValue is Map<Object?, Object?>) {
+      if (baseValue is Map<Object?, Object?>) {
+        mergeMap(baseValue, overlayValue);
+      } else {
+        base[entry.key] = overlayValue;
+      }
+    } else if (overlayValue is List<Object?>) {
+      if (baseValue is List<Object?>) {
+        baseValue.addAll(overlayValue);
+      } else {
+        base[entry.key] = overlayValue;
+      }
+    } else {
+      base[entry.key] = overlayValue;
+    }
+  }
+}
+
 /// Generate a link for display in a terminal.
 ///
 /// Similar to `<a href="$url">$text</a>` in HTML.

packages/melos/lib/src/workspace.dart

@@ -48,6 +48,7 @@ class MelosWorkspace {
     required this.config,
     required this.allPackages,
     required this.filteredPackages,
+    required this.dependencyOverridePackages,
     required this.sdkPath,
     required this.logger,
   });
@@ -65,6 +66,12 @@ class MelosWorkspace {
       ignore: workspaceConfig.ignore,
       logger: logger,
     );
+    final dependencyOverridePackages = await PackageMap.resolvePackages(
+      workspacePath: workspaceConfig.path,
+      packages: workspaceConfig.commands.bootstrap.dependencyOverridePaths,
+      ignore: const [],
+      logger: logger,
+    );
 
     final filteredPackages = await allPackages.applyFilter(filter);
 
@@ -75,6 +82,7 @@ class MelosWorkspace {
       allPackages: allPackages,
       logger: logger,
       filteredPackages: filteredPackages,
+      dependencyOverridePackages: dependencyOverridePackages,
       sdkPath: resolveSdkPath(
         configSdkPath: workspaceConfig.sdkPath,
         envSdkPath: currentPlatform.environment[utils.envKeyMelosSdkPath],
@@ -96,13 +104,21 @@ class MelosWorkspace {
   /// Configuration as defined in the "melos.yaml" file if it exists.
   final MelosWorkspaceConfig config;
 
-  /// All packages according to [MelosWorkspaceConfig].
+  /// All packages managed in this Melos workspace.
   ///
-  /// Packages filtered by [MelosWorkspaceConfig.ignore] are not included.
+  /// Packages specified in [MelosWorkspaceConfig.packages] are included,
+  /// except for those specified in [MelosWorkspaceConfig.ignore].
   final PackageMap allPackages;
 
+  /// The packages in this Melos workspace after applying filters.
+  ///
+  /// Filters are typically specified on the command line.
   final PackageMap filteredPackages;
 
+  /// The packages specified in
+  /// [BootstrapCommandConfigs.dependencyOverridePaths].
+  final PackageMap dependencyOverridePackages;
+
   late final IdeWorkspace ide = IdeWorkspace._(this);
 
   /// Returns true if this workspace contains ANY Flutter package.

packages/melos/lib/src/workspace_configs.dart

@@ -168,7 +168,10 @@ class CommandConfigs {
     );
 
     return CommandConfigs(
-      bootstrap: BootstrapCommandConfigs.fromYaml(bootstrapMap ?? const {}),
+      bootstrap: BootstrapCommandConfigs.fromYaml(
+        bootstrapMap ?? const {},
+        workspacePath: workspacePath,
+      ),
       version: VersionCommandConfigs.fromYaml(
         versionMap ?? const {},
         workspacePath: workspacePath,
@@ -216,9 +219,13 @@ class BootstrapCommandConfigs {
   const BootstrapCommandConfigs({
     this.runPubGetInParallel = true,
     this.runPubGetOffline = false,
+    this.dependencyOverridePaths = const [],
   });
 
-  factory BootstrapCommandConfigs.fromYaml(Map<Object?, Object?> yaml) {
+  factory BootstrapCommandConfigs.fromYaml(
+    Map<Object?, Object?> yaml, {
+    required String workspacePath,
+  }) {
     final runPubGetInParallel = assertKeyIsA<bool?>(
           key: 'runPubGetInParallel',
           map: yaml,
@@ -233,9 +240,26 @@ class BootstrapCommandConfigs {
         ) ??
         false;
 
+    final dependencyOverridePaths = assertListIsA<String>(
+      key: 'dependencyOverridePaths',
+      map: yaml,
+      isRequired: false,
+      assertItemIsA: (index, value) => assertIsA<String>(
+        value: value,
+        index: index,
+        path: 'dependencyOverridePaths',
+      ),
+    );
+
     return BootstrapCommandConfigs(
       runPubGetInParallel: runPubGetInParallel,
       runPubGetOffline: runPubGetOffline,
+      dependencyOverridePaths: dependencyOverridePaths
+          .map(
+            (override) =>
+                createGlob(override, currentDirectoryPath: workspacePath),
+          )
+          .toList(),
     );
   }
 
@@ -252,10 +276,17 @@ class BootstrapCommandConfigs {
   /// The default is `false`.
   final bool runPubGetOffline;
 
+  /// A list of [Glob]s for paths that contain packages to be used as dependency
+  /// overrides for all packages managed in the Melos workspace.
+  final List<Glob> dependencyOverridePaths;
+
   Map<String, Object?> toJson() {
     return {
       'runPubGetInParallel': runPubGetInParallel,
       'runPubGetOffline': runPubGetOffline,
+      if (dependencyOverridePaths.isNotEmpty)
+        'dependencyOverridePaths':
+            dependencyOverridePaths.map((path) => path.toString()).toList(),
     };
   }
 
@@ -264,20 +295,25 @@ class BootstrapCommandConfigs {
       other is BootstrapCommandConfigs &&
       runtimeType == other.runtimeType &&
       other.runPubGetInParallel == runPubGetInParallel &&
-      other.runPubGetOffline == runPubGetOffline;
+      other.runPubGetOffline == runPubGetOffline &&
+      const DeepCollectionEquality(_GlobEquality())
+          .equals(other.dependencyOverridePaths, dependencyOverridePaths);
 
   @override
   int get hashCode =>
       runtimeType.hashCode ^
       runPubGetInParallel.hashCode ^
-      runPubGetOffline.hashCode;
+      runPubGetOffline.hashCode ^
+      const DeepCollectionEquality(_GlobEquality())
+          .hash(dependencyOverridePaths);
 
   @override
   String toString() {
     return '''
 BootstrapCommandConfigs(
   runPubGetInParallel: $runPubGetInParallel,
   runPubGetOffline: $runPubGetOffline,
+  dependencyOverridePaths: $dependencyOverridePaths,
 )''';
   }
 }
@@ -755,12 +791,21 @@ You must have one of the following to be a valid Melos workspace:
 
     final melosYamlPath =
         melosYamlPathForDirectory(melosWorkspaceDirectory.path);
-    final yamlContents = await loadYamlFile(melosYamlPath);
+    final yamlContents = (await loadYamlFile(melosYamlPath))?.toPlainObject()
+        as Map<Object?, Object?>?;
 
     if (yamlContents == null) {
       throw MelosConfigException('Failed to parse the melos.yaml file');
     }
 
+    final melosOverridesYamlPath =
+        melosOverridesYamlPathForDirectory(melosWorkspaceDirectory.path);
+    final overridesYamlContents = (await loadYamlFile(melosOverridesYamlPath))
+        ?.toPlainObject() as Map<Object?, Object?>?;
+    if (overridesYamlContents != null) {
+      mergeMap(yamlContents, overridesYamlContents);
+    }
+
     return MelosWorkspaceConfig.fromYaml(
       yamlContents,
       path: melosWorkspaceDirectory.path,
@@ -833,8 +878,10 @@ You must have one of the following to be a valid Melos workspace:
       other.path == path &&
       other.name == name &&
       other.repository == repository &&
-      const DeepCollectionEquality().equals(other.packages, packages) &&
-      const DeepCollectionEquality().equals(other.ignore, ignore) &&
+      const DeepCollectionEquality(_GlobEquality())
+          .equals(other.packages, packages) &&
+      const DeepCollectionEquality(_GlobEquality())
+          .equals(other.ignore, ignore) &&
       other.scripts == scripts &&
       other.ide == ide &&
       other.commands == commands;
@@ -845,8 +892,8 @@ You must have one of the following to be a valid Melos workspace:
       path.hashCode ^
       name.hashCode ^
       repository.hashCode ^
-      const DeepCollectionEquality().hash(packages) &
-          const DeepCollectionEquality().hash(ignore) ^
+      const DeepCollectionEquality(_GlobEquality()).hash(packages) &
+          const DeepCollectionEquality(_GlobEquality()).hash(ignore) ^
       scripts.hashCode ^
       ide.hashCode ^
       commands.hashCode;
@@ -879,3 +926,17 @@ MelosWorkspaceConfig(
 )''';
   }
 }
+
+class _GlobEquality implements Equality<Glob> {
+  const _GlobEquality();
+
+  @override
+  bool equals(Glob e1, Glob e2) =>
+      e1.pattern == e2.pattern && e1.context.current == e2.context.current;
+
+  @override
+  int hash(Glob e) => e.pattern.hashCode ^ e.context.current.hashCode;
+
+  @override
+  bool isValidKey(Object? o) => true;
+}

packages/melos/test/utils.dart

@@ -294,6 +294,7 @@ class VirtualWorkspaceBuilder {
       config: config,
       allPackages: packageMap,
       filteredPackages: packageMap,
+      dependencyOverridePackages: _buildVirtualPackageMap(const [], logger),
       logger: logger,
       sdkPath: sdkPath,
     );

packages/melos/test/utils_test.dart

@@ -100,4 +100,35 @@ void main() {
       );
     });
   });
+
+  group('mergeYaml', () {
+    test('correctly handles value overriding', () {
+      final base = {
+        'abc': 123,
+        'def': [4, 5, 6],
+        'ghi': {'j': 'k', 'l': 'm', 'n': 'o'},
+        'pqr': ['1', '2', '3'],
+        'stu': 'aStringValue',
+        'vwx': true,
+      };
+      const overlay = {
+        'abc': 098,
+        'def': [7],
+        'ghi': {'j': 'i', 'l': 'm', 'n': 'o', 'p': 'q'},
+        'pqr': 'differentType',
+        'stu': ['another', 'different', 'type'],
+        'yza': false,
+      };
+      mergeMap(base, overlay);
+      expect(base, const {
+        'abc': 098,
+        'def': [4, 5, 6, 7],
+        'ghi': {'j': 'i', 'l': 'm', 'n': 'o', 'p': 'q'},
+        'pqr': 'differentType',
+        'stu': ['another', 'different', 'type'],
+        'vwx': true,
+        'yza': false,
+      });
+    });
+  });
 }