Skip to content

Commit fe8466b

Browse files
gmetaisgmetais
authored
Improve content type detection (#179)
1 parent f10fbc3 commit fe8466b

4 files changed

Lines changed: 248 additions & 9 deletions

File tree

‎lib/tools/weightChecker/contentTypeChecker.js‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
var debug = require('debug')('ylt:contentTypeChecker');
2+
var Q = require('q');
3+
var isJpg = require('is-jpg');
4+
var isPng = require('is-png');
5+
var isSvg = require('is-svg');
6+
var isGif = require('is-gif');
7+
var isWoff = require('is-woff');
8+
var isWoff2 = require('is-woff2');
9+
var isOtf = require('is-otf');
10+
var isTtf = require('is-ttf');
11+
var isEot = require('is-eot');
12+
13+
var ContentTypeChecker = function() {
14+
15+
function checkContentType(entry) {
16+
var deferred = Q.defer();
17+
18+
debug('Entering contentTypeChecker');
19+
20+
// Ignore very small files as they are generally tracking pixels
21+
if (entry.weightCheck && entry.weightCheck.body && entry.weightCheck.bodySize > 100) {
22+
var foundType;
23+
24+
try {
25+
foundType = findContentType(entry.weightCheck.body);
26+
27+
if (!entry.contentType || entry.contentType === '') {
28+
if (foundType === null) {
29+
debug('ContentType is empty for file %s', entry.url);
30+
} else {
31+
debug('ContentType is empty for file %s. It should be %s.', entry.url, foundType.mimes[0]);
32+
entry.oldContentType = null;
33+
rewriteContentType(entry, foundType);
34+
}
35+
} else {
36+
if (foundType !== null && foundType.mimes.indexOf(entry.contentType) === -1) {
37+
debug('ContentType %s is wrong for %s. It should be %s.', entry.contentType, entry.url, foundType.mimes[0]);
38+
entry.oldContentType = entry.contentType;
39+
rewriteContentType(entry, foundType);
40+
}
41+
}
42+
43+
} catch(err) {
44+
debug('Error while analyzing the contentType of %s', entry.url);
45+
debug(err);
46+
}
47+
}
48+
49+
deferred.resolve(entry);
50+
51+
return deferred.promise;
52+
}
53+
54+
function findContentType(body) {
55+
var buffer = new Buffer(body, 'binary');
56+
57+
if (isJpg(buffer)) {
58+
return contentTypes.jpeg;
59+
}
60+
61+
if (isPng(buffer)) {
62+
return contentTypes.png;
63+
}
64+
65+
// https://github.com/sindresorhus/is-svg/issues/7
66+
if (/<svg/.test(body) && isSvg(body)) {
67+
return contentTypes.svg;
68+
}
69+
70+
if (isGif(buffer)) {
71+
return contentTypes.gif;
72+
}
73+
74+
if (isWoff(buffer)) {
75+
return contentTypes.woff;
76+
}
77+
78+
if (isWoff2(buffer)) {
79+
return contentTypes.woff2;
80+
}
81+
82+
if (isOtf(buffer)) {
83+
return contentTypes.otf;
84+
}
85+
86+
if (isTtf(buffer)) {
87+
return contentTypes.ttf;
88+
}
89+
90+
if (isEot(buffer)) {
91+
return contentTypes.eot;
92+
}
93+
94+
return null;
95+
}
96+
97+
98+
function rewriteContentType(entry, contentTypeObj) {
99+
delete(entry.isHTML);
100+
delete(entry.isXML);
101+
delete(entry.isCSS);
102+
delete(entry.isJS);
103+
delete(entry.isJSON);
104+
delete(entry.isImage);
105+
delete(entry.isSVG);
106+
delete(entry.isVideo);
107+
delete(entry.isWebFont);
108+
delete(entry.isTTF);
109+
delete(entry.isFavicon);
110+
111+
entry.contentType = contentTypeObj.mimes[0];
112+
contentTypeObj.updateFn(entry);
113+
}
114+
115+
var contentTypes = {
116+
jpeg: {
117+
mimes: ['image/jpeg'],
118+
updateFn: function(entry) {
119+
entry.type = 'image';
120+
entry.isImage = true;
121+
}
122+
},
123+
png: {
124+
mimes: ['image/png'],
125+
updateFn: function(entry) {
126+
entry.type = 'image';
127+
entry.isImage = true;
128+
}
129+
},
130+
svg: {
131+
mimes: ['image/svg+xml'],
132+
updateFn: function(entry) {
133+
entry.type = 'image';
134+
entry.isImage = true;
135+
entry.isSVG = true;
136+
}
137+
},
138+
gif: {
139+
mimes: ['image/gif'],
140+
updateFn: function(entry) {
141+
entry.type = 'image';
142+
entry.isImage = true;
143+
}
144+
},
145+
woff: {
146+
mimes: ['application/x-font-woff', 'application/font-woff', 'font/woff'],
147+
updateFn: function(entry) {
148+
entry.type = 'webfont';
149+
entry.isWebFont = true;
150+
}
151+
},
152+
woff2: {
153+
mimes: ['font/woff2', 'application/x-font-woff2', 'application/font-woff2'],
154+
updateFn: function(entry) {
155+
entry.type = 'webfont';
156+
entry.isWebFont = true;
157+
}
158+
},
159+
otf: {
160+
mimes: ['application/x-font-otf', 'font/otf', 'font/opentype', 'application/x-font-opentype'],
161+
updateFn: function(entry) {
162+
entry.type = 'webfont';
163+
entry.isWebFont = true;
164+
}
165+
},
166+
ttf: {
167+
mimes: ['application/x-font-ttf', 'font/ttf', 'application/x-font-truetype'],
168+
updateFn: function(entry) {
169+
entry.type = 'webfont';
170+
entry.isWebFont = true;
171+
}
172+
},
173+
eot: {
174+
mimes: ['application/vnd.ms-fontobject', 'font/eot'],
175+
updateFn: function(entry) {
176+
entry.type = 'webfont';
177+
entry.isWebFont = true;
178+
}
179+
}
180+
};
181+
182+
return {
183+
checkContentType: checkContentType,
184+
findContentType: findContentType
185+
};
186+
};
187+
188+
module.exports = new ContentTypeChecker();

‎lib/tools/weightChecker/weightChecker.js‎

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,17 @@
55
*/
66

77

8-
var debug = require('debug')('ylt:weightChecker');
9-
var Q = require('q');
10-
var http = require('http');
11-
var zlib = require('zlib');
12-
var async = require('async');
13-
var request = require('request');
8+
var debug = require('debug')('ylt:weightChecker');
9+
var Q = require('q');
10+
var http = require('http');
11+
var zlib = require('zlib');
12+
var async = require('async');
13+
var request = require('request');
1414

15-
var imageOptimizer = require('./imageOptimizer');
16-
var fileMinifier = require('./fileMinifier');
17-
var gzipCompressor = require('./gzipCompressor');
15+
var imageOptimizer = require('./imageOptimizer');
16+
var fileMinifier = require('./fileMinifier');
17+
var gzipCompressor = require('./gzipCompressor');
18+
var contentTypeChecker = require('./contentTypeChecker');
1819

1920

2021
var WeightChecker = function() {
@@ -47,6 +48,8 @@ var WeightChecker = function() {
4748

4849
redownloadEntry(entry, httpAuth)
4950

51+
.then(contentTypeChecker.checkContentType)
52+
5053
.then(imageOptimizer.optimizeImage)
5154

5255
.then(fileMinifier.minifyFile)
@@ -89,6 +92,10 @@ var WeightChecker = function() {
8992
});
9093

9194

95+
// Wrong contentType
96+
offenders.incorrectContentTypes = listIncorrectContentTypes(results);
97+
metrics.incorrectContentTypes = offenders.incorrectContentTypes.length;
98+
9299
// Total weight
93100
offenders.totalWeight = listRequestWeight(results);
94101
metrics.totalWeight = offenders.totalWeight.totalWeight;
@@ -121,6 +128,21 @@ var WeightChecker = function() {
121128
return deferred.promise;
122129
}
123130

131+
function listIncorrectContentTypes(requests) {
132+
var results = [];
133+
134+
requests.forEach(function(req) {
135+
if (req.oldContentType || req.oldContentType === null) {
136+
results.push({
137+
url: req.url,
138+
current: req.oldContentType,
139+
correct: req.contentType
140+
});
141+
}
142+
});
143+
144+
return results;
145+
}
124146

125147
function listRequestWeight(requests) {
126148
var results = {

‎package.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,16 @@
4343
"imagemin-jpegtran": "5.0.2",
4444
"imagemin-optipng": "5.1.0",
4545
"imagemin-svgo": "5.1.0",
46+
"is-eot": "1.0.0",
47+
"is-gif": "1.0.0",
4648
"is-http2": "1.0.4",
49+
"is-jpg": "1.0.0",
50+
"is-otf": "0.1.2",
51+
"is-png": "1.0.0",
52+
"is-svg": "2.0.1",
53+
"is-ttf": "0.2.2",
54+
"is-woff": "1.0.3",
55+
"is-woff2": "1.0.0",
4756
"lwip": "0.0.9",
4857
"meow": "3.7.0",
4958
"minimize": "2.0.0",

‎test/core/contentTypeCheckerTest.js‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
var should = require('chai').should();
2+
var contentTypeChecker = require('../../lib/tools/weightChecker/contentTypeChecker');
3+
var fs = require('fs');
4+
var path = require('path');
5+
6+
describe('contentTypeChecker', function() {
7+
8+
var jpgImageContent = fs.readFileSync(path.resolve(__dirname, '../www/jpeg-image.jpg'));
9+
var pngImageContent = fs.readFileSync(path.resolve(__dirname, '../www/png-image.png'));
10+
var svgImageContent = fs.readFileSync(path.resolve(__dirname, '../www/svg-image.svg'));
11+
var cssFileContent = fs.readFileSync(path.resolve(__dirname, '../www/unminified-stylesheet.css'));
12+
13+
it('detect the right content type', function() {
14+
contentTypeChecker.findContentType(jpgImageContent).mimes.should.deep.equal(['image/jpeg']);
15+
contentTypeChecker.findContentType(pngImageContent).mimes.should.deep.equal(['image/png']);
16+
contentTypeChecker.findContentType(svgImageContent).mimes.should.deep.equal(['image/svg+xml']);
17+
should.equal(contentTypeChecker.findContentType(cssFileContent), null);
18+
});
19+
20+
});

0 commit comments

Comments
 (0)