Skip to content

Commit 0b9342e

Browse files
sokrasokra
committed
better work distribution
1 parent 193173c commit 0b9342e

File tree

8 files changed

+49
-41
lines changed

8 files changed

+49
-41
lines changed

‎README.md‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -523,8 +523,7 @@ You can also save this options object in a JSON file and use it with the shell c
523523
// overhead (~100-200ms). If loaders are used they need to have the
524524
// seperable flag to work in worker process. If they havn't they work in
525525
// the main process.
526-
// In watch mode, worker processes only start once. So workers = true
527-
// is recommended for watch mode.
526+
// Pushing jobs to worker processes has an addititional overhead of ~100ms.
528527

529528
closeWorkers: false,
530529
// default: true

‎bm.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
var webpack = require("./lib/webpack");
66
var path = require("path");
77

8-
var TIMES = 2;
8+
var TIMES = 5;
99

1010
/* TESTS */
1111

‎lib/Workers.js‎

Lines changed: 19 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2,41 +2,38 @@ var child_process = require("child_process");
22

33
function Workers(moduleToFork, count) {
44
this.nextId = 1;
5-
this.jobs = [];
6-
this.freeWorkers = [];
75
this.workers = [];
6+
this.workersJobs = [];
87
this.callbacks = {};
98
for(var i = 0; i < count; i++) {
109
var worker = child_process.fork(moduleToFork);
1110
this.workers.push(worker);
12-
this.freeWorkers.push(worker);
13-
this.bindWorker(worker);
11+
this.workersJobs.push(0);
12+
this.bindWorker(worker, i);
1413
}
1514
}
1615
module.exports = Workers;
1716

1817
Workers.prototype.run = function(parameters, callback) {
19-
if(this.freeWorkers.length > 0)
20-
this.pushJob(this.freeWorkers.shift(), parameters, callback);
21-
else
22-
this.jobs.push([parameters, callback]);
18+
var worker = 0;
19+
var minJobs = -1;
20+
this.workersJobs.forEach(function(jobs, idx) {
21+
if(jobs < minJobs || minJobs == -1) {
22+
minJobs = jobs;
23+
worker = idx;
24+
}
25+
});
26+
this.workersJobs[worker]++;
27+
this.pushJob(this.workers[worker], parameters, callback);
2328
}
2429

25-
Workers.prototype.bindWorker = function(worker) {
30+
Workers.prototype.bindWorker = function(worker, idx) {
2631
worker.on("message", function(result) {
27-
if(Array.isArray(result)) {
28-
var id = result.shift();
29-
var callback = this.callbacks[id];
30-
delete this.callbacks[id];
31-
callback.apply(null, result);
32-
} else {
33-
if(this.jobs.length > 0) {
34-
var job = this.jobs.shift();
35-
this.pushJob(worker, job[0], job[1]);
36-
} else {
37-
this.freeWorkers.push(worker);
38-
}
39-
}
32+
this.workersJobs[idx]--;
33+
var id = result.shift();
34+
var callback = this.callbacks[id];
35+
delete this.callbacks[id];
36+
callback.apply(null, result);
4037
}.bind(this));
4138
}
4239

‎lib/buildDeps.js‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -104,13 +104,21 @@ function addModule(depTree, context, modu, options, reason, finalCallback) {
104104
options.events.emit("task-end");
105105
if(profile && profile.module) {
106106
profile.end = new Date();
107-
profile.module.profile = {
108-
time: profile.end - profile.start,
109-
timeResolve: profile.resolveEnd - profile.start,
110-
timeResolvePrePostLoaders: profile.resolvePrePostLoadersEnd - profile.resolveEnd,
111-
timeLoadersCheck: profile.loadersCheckEnd - profile.resolvePrePostLoadersEnd,
112-
timeBuildModule: profile.buildModuleEnd - profile.loadersCheckEnd,
113-
timeChildren: profile.end - profile.buildModuleEnd
107+
if(profile.buildModule) {
108+
profile.module.profile = {
109+
time: profile.end - profile.start,
110+
timeResolve: profile.resolveEnd - profile.start,
111+
timeResolvePrePostLoaders: profile.resolvePrePostLoadersEnd - profile.resolveEnd,
112+
timeLoadersCheck: profile.loadersCheckEnd - profile.resolvePrePostLoadersEnd,
113+
timeBuildWaiting: (profile.buildModuleEnd - profile.loadersCheckEnd) - (profile.buildModule.end - profile.buildModule.start),
114+
timeBuildModule: profile.buildModule.end - profile.buildModule.start,
115+
timeBuildModuleRead: profile.buildModule.readEnd - profile.buildModule.start,
116+
timeBuildModulePreLoaders: profile.buildModule.preLoadersEnd - profile.buildModule.readEnd,
117+
timeBuildModuleLoaders: profile.buildModule.loadersEnd - profile.buildModule.preLoadersEnd,
118+
timeBuildModulePostLoaders: profile.buildModule.postLoadersEnd - profile.buildModule.loadersEnd,
119+
timeBuildModuleParse: profile.buildModule.end - profile.buildModule.postLoadersEnd,
120+
timeChildren: profile.end - profile.buildModuleEnd
121+
}
114122
}
115123
}
116124
finalCallback(err, result);

‎lib/buildModule.js‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ function buildModule(parameters, callback) {
1212
var filename = parameters.filename;
1313
var options = parameters.options;
1414

15+
if(options.profile) var profile = {
16+
start: new Date().getTime()
17+
}
18+
1519
var dependencyInfo = {
1620
cacheable: true,
1721
files: [filename]
@@ -28,6 +32,8 @@ function buildModule(parameters, callback) {
2832
fs.readFile(filename, function(err, content) {
2933
if(err) return callback(err);
3034

35+
profile && (profile.readEnd = new Date().getTime());
36+
3137
var loaderContext = {
3238
loaders: loaders,
3339
preLoaders: preLoaders,
@@ -39,14 +45,17 @@ function buildModule(parameters, callback) {
3945
execLoaders(context, filenameWithLoaders, preLoaders, [filename], [content], loaderContext, dependencyInfo, options,
4046
function(err, result) {
4147
if(err) return callback(err);
48+
profile && (profile.preLoadersEnd = new Date().getTime());
4249
loaderContext.loaderType = "loader";
4350
execLoaders(context, filenameWithLoaders, loaders, [filename], result, loaderContext, dependencyInfo, options,
4451
function(err, result) {
4552
if(err) return callback(err);
53+
profile && (profile.loadersEnd = new Date().getTime());
4654
loaderContext.loaderType = "postLoader";
4755
execLoaders(context, filenameWithLoaders, postLoaders, [filename], result, loaderContext, dependencyInfo, options,
4856
function(err, result) {
4957
if(err) return callback(err);
58+
profile && (profile.postLoadersEnd = new Date().getTime());
5059
return processJs(result)
5160
});
5261
});
@@ -66,7 +75,8 @@ function buildModule(parameters, callback) {
6675
callback(new Error("File \"" + filenameWithLoaders + "\" parsing failed: " + e));
6776
return;
6877
}
69-
return callback(null, source, deps, dependencyInfo);
78+
profile && (profile.end = new Date().getTime());
79+
return callback(null, source, deps, dependencyInfo, profile);
7080
}
7181
}
7282
module.exports = buildModule;

‎lib/buildModuleFork.js‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,4 @@ process.on("message", function(arr) {
1111
} catch(e) {
1212
process.send([id, e]);
1313
}
14-
process.send(null);
1514
});

‎lib/formatOutput.js‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -71,12 +71,7 @@ module.exports = function(stats, options) {
7171
var valueTime = module.profile.time - module.profile.timeChildren;
7272
if(valueTime > maxTime) moduleLine += c("\033[1m\033[31m");
7373
else if(valueTime > middleTime) moduleLine += c("\033[1m\033[33m");
74-
var buildProfile = module.profile.buildModule;
75-
if(buildProfile) buildProfile = " (" +
76-
(buildProfile.loaders && buildProfile.loaders.length > 0 ? "loaders: " + buildProfile.loaders.join("ms, ") + "ms" : "") +
77-
")";
78-
else buildProfile = "";
79-
moduleLine += " [" + module.profile.time + "ms: " + (module.profile.timeResolve + module.profile.timeResolvePrePostLoaders) + "ms resolving, " + module.profile.timeBuildModule + "ms build" + buildProfile + ", " + module.profile.timeChildren + "ms children]";
74+
moduleLine += " [" + module.profile.time + "ms: " + (module.profile.timeResolve + module.profile.timeResolvePrePostLoaders) + "ms resolving, " + module.profile.timeBuildWaiting + "ms waiting" + ", " + module.profile.timeBuildModule + "ms build" + ", " + module.profile.timeChildren + "ms children]";
8075
if(valueTime > middleTime) moduleLine += c("\033[39m\033[22m");
8176
}
8277
buf.push(moduleLine);

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "webpack",
3-
"version": "0.7.0-beta2",
3+
"version": "0.7.0-beta3",
44
"author": "Tobias Koppers @sokra",
55
"description": "Packs CommonJs/AMD Modules for the browser. Allows to split your codebase into multiple bundles, which can be loaded on demand. Support loading of js, json, jade, coffee, css, ... out of the box and more with custom loaders.",
66
"dependencies": {

0 commit comments

Comments
 (0)