[FEATURE] depCache bundling mode (#951)

JIRA: CPOUI5FOUNDATION-744

Dependency cache information that lists modules and their dependencies
of all types: JS, declarative views/fragments.
Only the dependencies of the modules are stored as 'depCache'
configuration.

---------

Co-authored-by: Matthias Oßwald <mat.osswald@sap.com>

lib/lbt/bundle/AutoSplitter.js

@@ -41,6 +41,8 @@ class AutoSplitter {
 		const numberOfParts = options.numberOfParts;
 		let totalSize = 0;
 		const moduleSizes = Object.create(null);
+		const depCacheSizes = [];
+		let depCacheLoaderSize = 0;
 		this.optimize = !!options.optimize;
 
 		// ---- resolve module definition
@@ -74,6 +76,27 @@ class AutoSplitter {
 					totalSize += "sap.ui.requireSync('');".length + toRequireJSName(module).length;
 				});
 				break;
+			case SectionType.DepCache:
+				depCacheLoaderSize = "sap.ui.loader.config({depCacheUI5:{}});".length;
+				totalSize += depCacheLoaderSize;
+
+				section.modules.forEach( (module) => {
+					promises.push((async () => {
+						const resource = await this.pool.findResourceWithInfo(module);
+						const deps = resource.info.dependencies.filter(
+							(dep) =>
+								!resource.info.isConditionalDependency(dep) &&
+								!resource.info.isImplicitDependency(dep)
+						);
+						if (deps.length > 0) {
+							const depSize = `"${module}": [${deps.map((dep) => `"${dep}"`).join(",")}],`.length;
+							totalSize += depSize;
+
+							depCacheSizes.push({size: depSize, module});
+						}
+					})());
+				});
+				break;
 			default:
 				break;
 			}
@@ -180,6 +203,36 @@ class AutoSplitter {
 					totalSize += 21 + toRequireJSName(module).length;
 				});
 				break;
+			case SectionType.DepCache:
+				currentSection = {
+					mode: SectionType.DepCache,
+					filters: []
+				};
+				currentModule.sections.push( currentSection );
+				totalSize += depCacheLoaderSize;
+
+				depCacheSizes.forEach((depCache) => {
+					if ( part + 1 < numberOfParts && totalSize + depCache.size / 2 > partSize ) {
+						part++;
+						// start a new module
+						totalSize = depCacheLoaderSize;
+						currentSection = {
+							mode: SectionType.DepCache,
+							filters: []
+						};
+						currentModule = {
+							name: moduleNameWithPart.replace(/__part__/, part),
+							sections: [currentSection]
+						};
+						splittedModules.push(currentModule);
+					}
+
+					if (!currentSection.filters.includes(depCache.module)) {
+						currentSection.filters.push(depCache.module);
+						totalSize += depCache.size;
+					}
+				});
+				break;
 			default:
 				break;
 			}

lib/lbt/bundle/Builder.js

@@ -71,6 +71,14 @@ const EVOBundleFormat = {
 			resolvedModule.executes(MODULE__UI5LOADER_AUTOCONFIG) ||
 			resolvedModule.executes(MODULE__JQUERY_SAP_GLOBAL) ||
 			resolvedModule.executes(MODULE__SAP_UI_CORE_CORE);
+	},
+
+	beforeDepCache(outW) {
+		outW.writeln(`sap.ui.loader.config({depCacheUI5:{`);
+	},
+
+	afterDepCache(outW) {
+		outW.writeln(`}});`);
 	}
 };
 
@@ -220,6 +228,8 @@ class BundleBuilder {
 			return this.writeBundleInfos([section]);
 		case SectionType.Require:
 			return this.writeRequires(section);
+		case SectionType.DepCache:
+			return this.writeDepCache(section);
 		default:
 			throw new Error("unknown section mode " + section.mode);
 		}
@@ -549,6 +559,59 @@ class BundleBuilder {
 		});
 	}
 
+	// When AutoSplit is enabled for depCache, we need to ensure that modules
+	// are not duplicated across files. This might happen due to the filters provided.
+	// So, certain modules that are included in depCache could be dependencies of another
+	// module in the next file. This will also duplicate its dependency definition if we do not filter.
+	#depCacheSet = new Set();
+	async writeDepCache(section) {
+		const outW = this.outW;
+		let hasDepCache = false;
+
+		const sequence = section.modules.slice().sort();
+
+		if (sequence.length > 0) {
+			for (const module of sequence) {
+				if (this.#depCacheSet.has(module)) {
+					continue;
+				}
+
+				this.#depCacheSet.add(module);
+				let resource = null;
+				try {
+					resource = await this.pool.findResourceWithInfo(module);
+				} catch (e) {
+					log.error(`    couldn't find ${module}`);
+				}
+
+				if (resource != null) {
+					const deps = resource.info.dependencies.filter(
+						(dep) =>
+							!resource.info.isConditionalDependency(dep) &&
+							!resource.info.isImplicitDependency(dep)
+					);
+					if (deps.length > 0) {
+						if (!hasDepCache) {
+							hasDepCache = true;
+							outW.ensureNewLine();
+							this.targetBundleFormat.beforeDepCache(outW, section);
+						}
+
+						outW.writeln(
+							`"${module}": [${deps.map((dep) => `"${dep}"`).join(",")}],`
+						);
+					} else {
+						log.verbose(`    skipped ${module}, no dependencies`);
+					}
+				}
+			}
+
+			if (hasDepCache) {
+				this.targetBundleFormat.afterDepCache(outW, section);
+			}
+		}
+	}
+
 	async getSourceMapForModule({moduleName, moduleContent, resourcePath}) {
 		let moduleSourceMap = null;
 		let newModuleContent = moduleContent;

lib/lbt/bundle/BundleDefinition.js

@@ -27,5 +27,12 @@ export const SectionType = {
 	 * Usually used as the last section in a merged module to enforce loading and
 	 * execution of some specific module or modules.
 	 */
-	Require: "require"
+	Require: "require",
+
+	/**
+	 * Dependency cache information that lists modules and their dependencies
+	 * of all types: JS, declarative views/fragments.
+	 * Only the dependencies of the modules are stored as 'depCache' configuration.
+	 */
+	DepCache: "depCache"
 };

lib/lbt/bundle/ResolvedBundleDefinition.js

@@ -64,7 +64,11 @@ class ResolvedBundleDefinition {
 				return Promise.all(
 					modules.map( (submodule) => {
 						return pool.getModuleInfo(submodule).then(
-							(subinfo) => bundleInfo.addSubModule(subinfo)
+							(subinfo) => {
+								if (!bundleInfo.subModules.includes(subinfo.name)) {
+									bundleInfo.addSubModule(subinfo);
+								}
+							}
 						);
 					})
 				);

lib/lbt/bundle/Resolver.js

@@ -198,7 +198,7 @@ class BundleResolver {
 			let oldIgnoredResources;
 			let oldSelectedResourcesSequence;
 
-			if ( section.mode == SectionType.Require ) {
+			if ( [SectionType.Require, SectionType.DepCache].includes(section.mode) ) {
 				oldSelectedResources = selectedResources;
 				oldIgnoredResources = visitedResources;
 				oldSelectedResourcesSequence = selectedResourcesSequence;
@@ -254,7 +254,7 @@ class BundleResolver {
 			});
 
 			return Promise.all(promises).then( function() {
-				if ( section.mode == SectionType.Require ) {
+				if ( [SectionType.Require, SectionType.DepCache].includes(section.mode) ) {
 					newKeys = selectedResourcesSequence;
 					selectedResources = oldSelectedResources;
 					visitedResources = oldIgnoredResources;

test/lib/lbt/bundle/AutoSplitter.js

@@ -75,6 +75,10 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => {
 		name: `Component-preload.js`,
 		defaultFileTypes: [".js", ".fragment.xml", ".view.xml", ".properties", ".json"],
 		sections: [{
+			mode: "depCache",
+			filters: ["*.js"],
+			modules: ["a.js", "c.js", "b.json", "c.properties", "x.view.xml"]
+		}, {
 			mode: "preload",
 			filters: ["a.js", "b.json", "x.view.xml"],
 			resolve: false,
@@ -110,12 +114,15 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => {
 	t.deepEqual(oResult[0], {
 		name: `Component-preload-0.js`,
 		sections: [{
+			filters: ["a.js", "c.js"],
+			mode: "depCache"
+		}, {
 			mode: "preload",
 			filters: ["a.js"],
 			name: undefined
 		}],
 		configuration: {}
-	}, "first part should contain only a.js since its size is only 2048");
+	}, "bundle properly and correct dependencies & sizes");
 	t.deepEqual(oResult[1], {
 		name: `Component-preload-1.js`,
 		sections: [{
@@ -138,6 +145,55 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => {
 	}, "second part should contain the other resources");
 });
 
+test("integration: Extreme AutoSplitter with numberOfParts 50", async (t) => {
+	const includedNamespace = "foo/bar/a";
+	const excludedNamespace = "fizz/buzz/b";
+	const modules = new Array(150)
+		.fill(null)
+		.map((val, index) =>
+			index % 2 ?
+				`${includedNamespace}${index}.js` :
+				`${excludedNamespace}${index}.js`
+		);
+	const pool = {
+		findResourceWithInfo: async (name) => {
+			const info = new ModuleInfo(name);
+			modules
+				.filter((moduleName) => moduleName !== name)
+				.forEach((dependency) => {
+					info.addDependency(dependency);
+				});
+			return {info};
+		},
+		resources: modules.map((res) => ({name: res}))
+	};
+	const autoSplitter = new AutoSplitter(pool, new BundleResolver(pool));
+	const bundleDefinition = {
+		name: `test-depCache-preload.js`,
+		sections: [{
+			mode: "depCache",
+			filters: ["foo/bar/**"],
+			modules
+		}]
+	};
+	const oResult = await autoSplitter.run(bundleDefinition, {numberOfParts: 50, optimize: false});
+	t.is(oResult.length, 50, "50 parts expected");
+
+	for (let i= 0; i < 50; i++) {
+		t.is(oResult[i].name, `test-depCache-preload-${i}.js`, "Correct preload bundles got created");
+	}
+
+	// Merge filters from all bundles
+	const allFilters = oResult.flatMap((res) =>
+		res.sections.flatMap((section) => section.filters)
+	).sort();
+
+	t.deepEqual(Array.from(new Set(allFilters)).sort(), allFilters, "There are no duplicate filters");
+	t.true(
+		allFilters.every((filter) => filter.startsWith("foo/bar")),
+		"Every (included) filter starts with foo/bar namespace. The rest are filtered."
+	);
+});
 
 test("_calcMinSize: compressedSize", async (t) => {
 	const pool = {