From f2cf564f0f71d635e58a743c7bdef1f427e341b2 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Wed, 6 Dec 2023 09:59:12 +0200 Subject: [PATCH] [FEATURE] depCache bundling mode (#951) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JIRA: CPOUI5FOUNDATION-744 Dependency cache information that lists modules and their dependencies of all types: JS, declarative views/fragments. Only the dependencies of the modules are stored as 'depCache' configuration. --------- Co-authored-by: Matthias Oßwald --- lib/lbt/bundle/AutoSplitter.js | 53 +++++ lib/lbt/bundle/Builder.js | 63 ++++++ lib/lbt/bundle/BundleDefinition.js | 9 +- lib/lbt/bundle/ResolvedBundleDefinition.js | 6 +- lib/lbt/bundle/Resolver.js | 4 +- test/lib/lbt/bundle/AutoSplitter.js | 58 ++++- test/lib/lbt/bundle/Builder.js | 252 +++++++++++++++++++++ 7 files changed, 440 insertions(+), 5 deletions(-) diff --git a/lib/lbt/bundle/AutoSplitter.js b/lib/lbt/bundle/AutoSplitter.js index 995d1a915..5e5f0d1f1 100644 --- a/lib/lbt/bundle/AutoSplitter.js +++ b/lib/lbt/bundle/AutoSplitter.js @@ -41,6 +41,8 @@ class AutoSplitter { const numberOfParts = options.numberOfParts; let totalSize = 0; const moduleSizes = Object.create(null); + const depCacheSizes = []; + let depCacheLoaderSize = 0; this.optimize = !!options.optimize; // ---- resolve module definition @@ -74,6 +76,27 @@ class AutoSplitter { totalSize += "sap.ui.requireSync('');".length + toRequireJSName(module).length; }); break; + case SectionType.DepCache: + depCacheLoaderSize = "sap.ui.loader.config({depCacheUI5:{}});".length; + totalSize += depCacheLoaderSize; + + section.modules.forEach( (module) => { + promises.push((async () => { + const resource = await this.pool.findResourceWithInfo(module); + const deps = resource.info.dependencies.filter( + (dep) => + !resource.info.isConditionalDependency(dep) && + !resource.info.isImplicitDependency(dep) + ); + if (deps.length > 0) { + const depSize = `"${module}": [${deps.map((dep) => `"${dep}"`).join(",")}],`.length; + totalSize += depSize; + + depCacheSizes.push({size: depSize, module}); + } + })()); + }); + break; default: break; } @@ -180,6 +203,36 @@ class AutoSplitter { totalSize += 21 + toRequireJSName(module).length; }); break; + case SectionType.DepCache: + currentSection = { + mode: SectionType.DepCache, + filters: [] + }; + currentModule.sections.push( currentSection ); + totalSize += depCacheLoaderSize; + + depCacheSizes.forEach((depCache) => { + if ( part + 1 < numberOfParts && totalSize + depCache.size / 2 > partSize ) { + part++; + // start a new module + totalSize = depCacheLoaderSize; + currentSection = { + mode: SectionType.DepCache, + filters: [] + }; + currentModule = { + name: moduleNameWithPart.replace(/__part__/, part), + sections: [currentSection] + }; + splittedModules.push(currentModule); + } + + if (!currentSection.filters.includes(depCache.module)) { + currentSection.filters.push(depCache.module); + totalSize += depCache.size; + } + }); + break; default: break; } diff --git a/lib/lbt/bundle/Builder.js b/lib/lbt/bundle/Builder.js index cd4e28c8e..71a839fb9 100644 --- a/lib/lbt/bundle/Builder.js +++ b/lib/lbt/bundle/Builder.js @@ -71,6 +71,14 @@ const EVOBundleFormat = { resolvedModule.executes(MODULE__UI5LOADER_AUTOCONFIG) || resolvedModule.executes(MODULE__JQUERY_SAP_GLOBAL) || resolvedModule.executes(MODULE__SAP_UI_CORE_CORE); + }, + + beforeDepCache(outW) { + outW.writeln(`sap.ui.loader.config({depCacheUI5:{`); + }, + + afterDepCache(outW) { + outW.writeln(`}});`); } }; @@ -220,6 +228,8 @@ class BundleBuilder { return this.writeBundleInfos([section]); case SectionType.Require: return this.writeRequires(section); + case SectionType.DepCache: + return this.writeDepCache(section); default: throw new Error("unknown section mode " + section.mode); } @@ -549,6 +559,59 @@ class BundleBuilder { }); } + // When AutoSplit is enabled for depCache, we need to ensure that modules + // are not duplicated across files. This might happen due to the filters provided. + // So, certain modules that are included in depCache could be dependencies of another + // module in the next file. This will also duplicate its dependency definition if we do not filter. + #depCacheSet = new Set(); + async writeDepCache(section) { + const outW = this.outW; + let hasDepCache = false; + + const sequence = section.modules.slice().sort(); + + if (sequence.length > 0) { + for (const module of sequence) { + if (this.#depCacheSet.has(module)) { + continue; + } + + this.#depCacheSet.add(module); + let resource = null; + try { + resource = await this.pool.findResourceWithInfo(module); + } catch (e) { + log.error(` couldn't find ${module}`); + } + + if (resource != null) { + const deps = resource.info.dependencies.filter( + (dep) => + !resource.info.isConditionalDependency(dep) && + !resource.info.isImplicitDependency(dep) + ); + if (deps.length > 0) { + if (!hasDepCache) { + hasDepCache = true; + outW.ensureNewLine(); + this.targetBundleFormat.beforeDepCache(outW, section); + } + + outW.writeln( + `"${module}": [${deps.map((dep) => `"${dep}"`).join(",")}],` + ); + } else { + log.verbose(` skipped ${module}, no dependencies`); + } + } + } + + if (hasDepCache) { + this.targetBundleFormat.afterDepCache(outW, section); + } + } + } + async getSourceMapForModule({moduleName, moduleContent, resourcePath}) { let moduleSourceMap = null; let newModuleContent = moduleContent; diff --git a/lib/lbt/bundle/BundleDefinition.js b/lib/lbt/bundle/BundleDefinition.js index d4716b8b7..bcb617267 100644 --- a/lib/lbt/bundle/BundleDefinition.js +++ b/lib/lbt/bundle/BundleDefinition.js @@ -27,5 +27,12 @@ export const SectionType = { * Usually used as the last section in a merged module to enforce loading and * execution of some specific module or modules. */ - Require: "require" + Require: "require", + + /** + * Dependency cache information that lists modules and their dependencies + * of all types: JS, declarative views/fragments. + * Only the dependencies of the modules are stored as 'depCache' configuration. + */ + DepCache: "depCache" }; diff --git a/lib/lbt/bundle/ResolvedBundleDefinition.js b/lib/lbt/bundle/ResolvedBundleDefinition.js index 013b433a7..e8cf0353a 100644 --- a/lib/lbt/bundle/ResolvedBundleDefinition.js +++ b/lib/lbt/bundle/ResolvedBundleDefinition.js @@ -64,7 +64,11 @@ class ResolvedBundleDefinition { return Promise.all( modules.map( (submodule) => { return pool.getModuleInfo(submodule).then( - (subinfo) => bundleInfo.addSubModule(subinfo) + (subinfo) => { + if (!bundleInfo.subModules.includes(subinfo.name)) { + bundleInfo.addSubModule(subinfo); + } + } ); }) ); diff --git a/lib/lbt/bundle/Resolver.js b/lib/lbt/bundle/Resolver.js index e14c40c7e..45c3e7f32 100644 --- a/lib/lbt/bundle/Resolver.js +++ b/lib/lbt/bundle/Resolver.js @@ -198,7 +198,7 @@ class BundleResolver { let oldIgnoredResources; let oldSelectedResourcesSequence; - if ( section.mode == SectionType.Require ) { + if ( [SectionType.Require, SectionType.DepCache].includes(section.mode) ) { oldSelectedResources = selectedResources; oldIgnoredResources = visitedResources; oldSelectedResourcesSequence = selectedResourcesSequence; @@ -254,7 +254,7 @@ class BundleResolver { }); return Promise.all(promises).then( function() { - if ( section.mode == SectionType.Require ) { + if ( [SectionType.Require, SectionType.DepCache].includes(section.mode) ) { newKeys = selectedResourcesSequence; selectedResources = oldSelectedResources; visitedResources = oldIgnoredResources; diff --git a/test/lib/lbt/bundle/AutoSplitter.js b/test/lib/lbt/bundle/AutoSplitter.js index c766b704b..77a8e1e1b 100644 --- a/test/lib/lbt/bundle/AutoSplitter.js +++ b/test/lib/lbt/bundle/AutoSplitter.js @@ -75,6 +75,10 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => { name: `Component-preload.js`, defaultFileTypes: [".js", ".fragment.xml", ".view.xml", ".properties", ".json"], sections: [{ + mode: "depCache", + filters: ["*.js"], + modules: ["a.js", "c.js", "b.json", "c.properties", "x.view.xml"] + }, { mode: "preload", filters: ["a.js", "b.json", "x.view.xml"], resolve: false, @@ -110,12 +114,15 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => { t.deepEqual(oResult[0], { name: `Component-preload-0.js`, sections: [{ + filters: ["a.js", "c.js"], + mode: "depCache" + }, { mode: "preload", filters: ["a.js"], name: undefined }], configuration: {} - }, "first part should contain only a.js since its size is only 2048"); + }, "bundle properly and correct dependencies & sizes"); t.deepEqual(oResult[1], { name: `Component-preload-1.js`, sections: [{ @@ -138,6 +145,55 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => { }, "second part should contain the other resources"); }); +test("integration: Extreme AutoSplitter with numberOfParts 50", async (t) => { + const includedNamespace = "foo/bar/a"; + const excludedNamespace = "fizz/buzz/b"; + const modules = new Array(150) + .fill(null) + .map((val, index) => + index % 2 ? + `${includedNamespace}${index}.js` : + `${excludedNamespace}${index}.js` + ); + const pool = { + findResourceWithInfo: async (name) => { + const info = new ModuleInfo(name); + modules + .filter((moduleName) => moduleName !== name) + .forEach((dependency) => { + info.addDependency(dependency); + }); + return {info}; + }, + resources: modules.map((res) => ({name: res})) + }; + const autoSplitter = new AutoSplitter(pool, new BundleResolver(pool)); + const bundleDefinition = { + name: `test-depCache-preload.js`, + sections: [{ + mode: "depCache", + filters: ["foo/bar/**"], + modules + }] + }; + const oResult = await autoSplitter.run(bundleDefinition, {numberOfParts: 50, optimize: false}); + t.is(oResult.length, 50, "50 parts expected"); + + for (let i= 0; i < 50; i++) { + t.is(oResult[i].name, `test-depCache-preload-${i}.js`, "Correct preload bundles got created"); + } + + // Merge filters from all bundles + const allFilters = oResult.flatMap((res) => + res.sections.flatMap((section) => section.filters) + ).sort(); + + t.deepEqual(Array.from(new Set(allFilters)).sort(), allFilters, "There are no duplicate filters"); + t.true( + allFilters.every((filter) => filter.startsWith("foo/bar")), + "Every (included) filter starts with foo/bar namespace. The rest are filtered." + ); +}); test("_calcMinSize: compressedSize", async (t) => { const pool = { diff --git a/test/lib/lbt/bundle/Builder.js b/test/lib/lbt/bundle/Builder.js index 8c24e4ceb..b28e4597b 100644 --- a/test/lib/lbt/bundle/Builder.js +++ b/test/lib/lbt/bundle/Builder.js @@ -817,6 +817,258 @@ ${SOURCE_MAPPING_URL}=library-preload.js.map ]); }); +test.serial("integration: createBundle with depCache", async (t) => { + const pool = new ResourcePool(); + pool.addResource({ + name: "a.js", + getPath: () => "a.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "sap.ui.define([\"./b\", \"./c2\"],function(b, c){return {};});" + }); + pool.addResource({ + name: "b.js", + getPath: () => "b.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "function Two(){return 2;}" + }); + pool.addResource({ + name: "c2.js", + getPath: () => "c2.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "sap.ui.define([\"./c1\", \"./c3\"],function(c1, c3){return {};});" + }); + pool.addResource({ + name: "c1.js", + getPath: () => "c1.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "function Three(){return 3.1;}" + }); + pool.addResource({ + name: "c3.js", + getPath: () => "c3.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "function Three(){return 3.3;}" + }); + pool.addResource({ + name: "a.library", + getPath: () => "a.library", + string: function() { + return this.buffer(); + }, + buffer: async () => ` + + + + + + + + +` + }); + + const bundleDefinition = { + name: `library-depCache-preload.js`, + sections: [{ + mode: "preload", + name: "preload-section", + filters: ["a.js"] + }, { + mode: "depCache", + filters: ["*.js"] + }] + }; + + const builder = new Builder(pool); + const oResult = await builder.createBundle(bundleDefinition, {}); + t.is(oResult.name, "library-depCache-preload.js"); + const expectedContent = `//@ui5-bundle library-depCache-preload.js +sap.ui.require.preload({ + "a.js":function(){ +sap.ui.define(["./b", "./c2"],function(b, c){return {};}); +} +},"preload-section"); +sap.ui.loader.config({depCacheUI5:{ +"a.js": ["b.js","c2.js"], +"c2.js": ["c1.js","c3.js"], +}}); +${SOURCE_MAPPING_URL}=library-depCache-preload.js.map +`; + t.deepEqual(oResult.content, expectedContent, "EVOBundleFormat " + + "should contain:" + + " preload part from a.js" + + " depCache part from a.js && c2.js"); + t.is(oResult.bundleInfo.name, "library-depCache-preload.js", "bundle info name is correct"); + t.deepEqual(oResult.bundleInfo.size, expectedContent.length, "bundle info size is correct"); + t.deepEqual(oResult.bundleInfo.subModules, ["a.js", "b.js", "c2.js", "c1.js", "c3.js"], + "bundle info subModules are correct"); +}); + +test.serial("integration: createBundle with depCache with splitted modules", async (t) => { + const resolvedModulesCount = 10; + const pool = new ResourcePool(); + + // Builds N resources by adding provided "dependencies" as resource dependencies. + // Also adds the remaining I resources into dependency list + const buildDependencies = function(count, namespace, dependencies = []) { + return new Array(count).fill(null).map((val, index, arr) => { + const strDeps = dependencies.map((dep) => "\"" + dep + "\""); + const deps = dependencies.map((val, i) => `b${i}`); + for (let i = index + 1; i < arr.length; i++ ) { + strDeps.push(`"${namespace}${i}"`); + deps.push(`a${i}`); + } + + const curResourceName = `${namespace}${index}`; + pool.addResource({ + name: `${curResourceName}.js`, + getPath: () => `${curResourceName}.js`, + string: function() { + return this.buffer(); + }, + buffer: async () => `sap.ui.define([${strDeps.join(", ")}],function(${deps.join(", ")}){return {};});` + }); + + return curResourceName; + }); + }; + + const nonCachedDependencies = buildDependencies(5, "fizz/buzz/b"); + const cachedDependencies = buildDependencies(resolvedModulesCount, "foo/bar/a", nonCachedDependencies); + + const bundleDefinition = { + name: `library-depCache-preload.js`, + sections: [{ + mode: "depCache", + filters: ["foo/bar/**"] + }] + }; + + const builder = new Builder(pool); + const oResult = await builder.createBundle(bundleDefinition, {numberOfParts: 2}); + t.is(oResult.length, 2, "The bundle got split into 2 parts"); + + t.falsy( + oResult[0].bundleInfo.subModules.find((module) => + oResult[1].bundleInfo.subModules.includes(module) + ), "Submodules do not overlap" + ); + + const allSubmodules = [...oResult[0].bundleInfo.subModules, ...oResult[1].bundleInfo.subModules]; + t.is(allSubmodules.length, resolvedModulesCount, `${resolvedModulesCount} of all defined modules in the pool are actually cached as the filter is only for foo/bar namespace`); + t.deepEqual( + allSubmodules.sort(), + cachedDependencies.sort().map((dep) => `${dep}.js`), + "Cached dependencies are the correct ones" + ); + t.true(allSubmodules.every((module) => module.startsWith("foo/bar")), "Every (included) submodule starts with foo/bar namespace. The rest are filtered."); +}); + +test.serial("integration: createBundle with depCache with NO dependencies", async (t) => { + const pool = new ResourcePool(); + pool.addResource({ + name: "a.js", + getPath: () => "a.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "sap.ui.define([],function(){return {};});" + }); + pool.addResource({ + name: "b.js", + getPath: () => "b.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "function Two(){return 2;}" + }); + pool.addResource({ + name: "c2.js", + getPath: () => "c2.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "sap.ui.define([],function(){return {};});" + }); + pool.addResource({ + name: "c1.js", + getPath: () => "c1.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "function Three(){return 3.1;}" + }); + pool.addResource({ + name: "c3.js", + getPath: () => "c3.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "function Three(){return 3.3;}" + }); + pool.addResource({ + name: "a.library", + getPath: () => "a.library", + string: function() { + return this.buffer(); + }, + buffer: async () => ` + + + + + + + + +` + }); + + const bundleDefinition = { + name: `library-depCache-preload.js`, + sections: [{ + mode: "preload", + name: "preload-section", + filters: ["a.js"] + }, { + mode: "depCache", + filters: ["*.js"] + }] + }; + + const builder = new Builder(pool); + const oResult = await builder.createBundle(bundleDefinition, {}); + t.is(oResult.name, "library-depCache-preload.js"); + const expectedContent = `//@ui5-bundle library-depCache-preload.js +sap.ui.require.preload({ + "a.js":function(){ +sap.ui.define([],function(){return {};}); +} +},"preload-section"); +${SOURCE_MAPPING_URL}=library-depCache-preload.js.map +`; + t.deepEqual(oResult.content, expectedContent, "EVOBundleFormat " + + "should contain:" + + " preload part from a.js" + + " depCache part from a.js && c2.js"); + t.is(oResult.bundleInfo.name, "library-depCache-preload.js", "bundle info name is correct"); + t.deepEqual(oResult.bundleInfo.size, expectedContent.length, "bundle info size is correct"); + t.deepEqual(oResult.bundleInfo.subModules, ["a.js", "b.js", "c2.js", "c1.js", "c3.js"], + "bundle info subModules are correct"); +}); + test("integration: createBundle using predefine calls with source maps and a single, simple source", async (t) => { const pool = new ResourcePool();