Merge pull request webpack#6014 from webpack/performance/merge-duplicate-chunks

sokra · web-flow · commit f1efe03b6204 · 2017-11-24T21:28:03.000+01:00
use a faster merge duplicate chunks algorithm
diff --git a/lib/optimize/MergeDuplicateChunksPlugin.js b/lib/optimize/MergeDuplicateChunksPlugin.js
@@ -9,18 +9,53 @@ class MergeDuplicateChunksPlugin {
 	apply(compiler) {
 		compiler.plugin("compilation", (compilation) => {
 			compilation.plugin("optimize-chunks-basic", (chunks) => {
-				const map = Object.create(null);
-				chunks.slice().forEach((chunk) => {
-					if(chunk.hasEntryModule()) return;
-					const ident = chunk.getModulesIdent();
-					const otherChunk = map[ident];
-					if(otherChunk) {
-						if(otherChunk.integrate(chunk, "duplicate"))
-							chunks.splice(chunks.indexOf(chunk), 1);
-						return;
+				// remember already tested chunks for performance
+				const notDuplicates = new Set();
+
+				// for each chunk
+				for(const chunk of chunks) {
+
+					// track a Set of all chunk that could be duplicates
+					let possibleDuplicates;
+					for(const module of chunk.modulesIterable) {
+						if(possibleDuplicates === undefined) {
+							// when possibleDuplicates is not yet set,
+							// create a new Set from chunks of the current module
+							// including only chunks with the same number of modules
+							for(const dup of module.chunksIterable) {
+								if(dup !== chunk && chunk.getNumberOfModules() === dup.getNumberOfModules() && !notDuplicates.has(dup)) {
+									// delay allocating the new Set until here, reduce memory pressure
+									if(possibleDuplicates === undefined)
+										possibleDuplicates = new Set();
+									possibleDuplicates.add(dup);
+								}
+							}
+							// when no chunk is possible we can break here
+							if(possibleDuplicates === undefined) break;
+						} else {
+							// validate existing possible duplicates
+							for(const dup of possibleDuplicates) {
+								// remove possible duplicate when module is not contained
+								if(!dup.containsModule(module))
+									possibleDuplicates.delete(dup);
+							}
+							// when all chunks has been removed we can break here
+							if(possibleDuplicates.size === 0) break;
+						}
 					}
-					map[ident] = chunk;
-				});
+
+					// when we found duplicates
+					if(possibleDuplicates !== undefined && possibleDuplicates.size > 0) {
+						for(const otherChunk of possibleDuplicates) {
+							// merge them
+							if(chunk.integrate(otherChunk, "duplicate"))
+								chunks.splice(chunks.indexOf(otherChunk), 1);
+						}
+					}
+
+					// don't check already processed chunks twice
+					notDuplicates.add(chunk);
+				}
 			});
 		});
 	}
