Skip to content

Commit cb31b0b

Browse files
HainishHainish
committed
Adding the hsts-prune utility
1 parent 0c4d2ac commit cb31b0b

File tree

4 files changed

+358
-0
lines changed

4 files changed

+358
-0
lines changed

‎utils/hsts-prune/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
node_modules

‎utils/hsts-prune/README.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# HTTPS Everywhere - HSTS Prune
2+
3+
Running this will remove targets from the rulesets which are preloaded in Firefox {Dev, Stable, ESR} as well as the latest Chromium head. If all targets would be removed from a ruleset, the ruleset itself is instead deleted.
4+
5+
## Installing Dependencies
6+
7+
npm install
8+
9+
## Running
10+
11+
node index.js

‎utils/hsts-prune/index.js‎

Lines changed: 323 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,323 @@
1+
"use strict";
2+
3+
const _ = require('highland');
4+
const fs = require('fs');
5+
const read_dir = _.wrapCallback(fs.readdir);
6+
const read_file = _.wrapCallback(fs.readFile);
7+
const write_file = _.wrapCallback(fs.writeFile);
8+
const unlink = _.wrapCallback(fs.unlink);
9+
const parse_xml = _.wrapCallback(require('xml2js').parseString);
10+
const async = require('async');
11+
const https = require('https');
12+
const http = require('http');
13+
const request = require('request');
14+
const split = require('split');
15+
const JSONStream = require('JSONStream');
16+
const base64 = require('base64-stream');
17+
const filter = require('stream-filter');
18+
const ProgressBar = require('progress');
19+
const escape_string_regexp = require('escape-string-regexp');
20+
21+
const stable_version_url = "https://download.mozilla.org/?product=firefox-latest&os=linux64&lang=en_US";
22+
const esr_version_url = "https://download.mozilla.org/?product=firefox-esr-latest&os=linux64&lang=en_US";
23+
const chromium_version_url = "https://dl.google.com/linux/direct/google-chrome-stable_current_x86_64.rpm";
24+
const rules_dir = `${__dirname}/../../src/chrome/content/rules`;
25+
26+
let bar;
27+
28+
// Here begins the process of fetching HSTS rules and parsing the from the
29+
// relevant URLs
30+
31+
const firefox_version_fetch = version_url => {
32+
return cb => {
33+
https.get(version_url, res => {
34+
cb(null, res.headers.location
35+
.match(/firefox-(.*).tar.*/)[1].replace(/\./g, "_"));
36+
});
37+
};
38+
};
39+
40+
const chromium_version_fetch = version_url => {
41+
return cb => {
42+
_(request.get(version_url))
43+
.on('error', err => {
44+
cb(err);
45+
})
46+
.on('data', data => {
47+
cb(null, String(data).match(/google-chrome-stable-([0-9\.]+)/)[1]);
48+
})
49+
.pull(_ => _);
50+
}
51+
};
52+
53+
54+
const parse_include = include_url => {
55+
let regex = RegExp(/ "(.+?)", (true|false) /);
56+
57+
return cb => {
58+
let hsts = {};
59+
request.get(include_url)
60+
.on('error', err => {
61+
cb(err);
62+
})
63+
.pipe(split())
64+
.on('data', line => {
65+
let regex_res = line.match(regex)
66+
if(regex_res){
67+
hsts[regex_res[1]] = Boolean(regex_res[2])
68+
}
69+
})
70+
.on('end', _ => {
71+
cb(null, hsts);
72+
});
73+
};
74+
};
75+
76+
const parse_json = json_url => {
77+
return cb => {
78+
let hsts = {};
79+
request.get(json_url)
80+
.on('error', err => {
81+
cb(err);
82+
})
83+
.pipe(base64.decode())
84+
.pipe(split())
85+
.pipe(filter(line => {
86+
return !String(line).match(/\/\//);
87+
}))
88+
.pipe(JSONStream.parse('entries.*'))
89+
.on('data', entry => {
90+
if(entry.mode == "force-https"){
91+
hsts[entry.name] = entry.include_subdomains;
92+
}
93+
})
94+
.on('end', _ => {
95+
cb(null, hsts);
96+
});
97+
};
98+
}
99+
100+
const check_inclusion = (structs, domain) => {
101+
if(domain in structs.esr &&
102+
domain in structs.dev &&
103+
domain in structs.stable &&
104+
domain in structs.chromium){
105+
return [true, domain];
106+
} else {
107+
let fqdn_shards = domain.split('.');
108+
for(let x = 1; x < fqdn_shards.length; x++){
109+
let recombined_fqdn = fqdn_shards.slice(x).join('.');
110+
if(structs.esr[recombined_fqdn] == true &&
111+
structs.dev[recombined_fqdn] == true &&
112+
structs.stable[recombined_fqdn] == true &&
113+
structs.chromium[recombined_fqdn] == true){
114+
return [true, recombined_fqdn];
115+
}
116+
}
117+
}
118+
return [false, null];
119+
}
120+
121+
const check_header_directives = (check_domain, cb) => {
122+
let sent_callback = false;
123+
https.get('https://' + check_domain, res => {
124+
if('strict-transport-security' in res.headers){
125+
let preload = Boolean(
126+
res.headers['strict-transport-security'].match(/preload/i));
127+
let include_subdomains = Boolean(
128+
res.headers['strict-transport-security'].match(/includesubdomains/i));
129+
let max_age_match =
130+
res.headers['strict-transport-security'].match(/max-age=([0-9]+)/i)
131+
let max_age;
132+
if(max_age_match){
133+
max_age = Number(max_age_match[1]);
134+
}
135+
cb(null,
136+
preload && include_subdomains && max_age >= 10886400);
137+
sent_callback = true;
138+
} else {
139+
cb(null, false);
140+
sent_callback = true;
141+
}
142+
}).on('error', err => {
143+
if(!sent_callback){
144+
cb(null, false);
145+
}
146+
});
147+
};
148+
149+
const check_https_redirection_and_header_directives = (check_domain, cb) => {
150+
let sent_callback = false;
151+
http.get('http://' + check_domain, res => {
152+
let escaped_check_domain = escape_string_regexp(check_domain);
153+
let check_domain_regex = RegExp(`^https://${escaped_check_domain}`, 'i');
154+
if(Math.floor(res.statusCode / 100) == 3 &&
155+
'location' in res.headers &&
156+
res.headers.location.match(check_domain_regex)){
157+
check_header_directives(check_domain, cb);
158+
} else {
159+
cb(null, false);
160+
sent_callback = true;
161+
}
162+
}).on('error', err => {
163+
if(!sent_callback){
164+
check_header_directives(check_domain, cb);
165+
}
166+
});
167+
};
168+
169+
170+
// Here we begin parsing the XML files and modifying the targets
171+
172+
function remove_target_from_xml(source, target) {
173+
let pos;
174+
// escape the regexp for targets that have a *
175+
target = escape_string_regexp(target);
176+
177+
const target_regex = RegExp(`\n[ \t]*<target host=\\s*"${target}"\\s*/>\\s*?\n`);
178+
179+
if(!source.match(target_regex)){
180+
throw new Error(`${target_regex} was not found in ${source}`);
181+
}
182+
return source.replace(target_regex, "\n");
183+
}
184+
185+
const files =
186+
read_dir(rules_dir)
187+
.tap(rules => {
188+
bar = new ProgressBar(':bar', { total: rules.length, stream: process.stdout });
189+
})
190+
.sequence()
191+
.filter(name => name.endsWith('.xml'));
192+
193+
const sources =
194+
files.fork()
195+
.map(name => read_file(`${rules_dir}/${name}`, 'utf-8'))
196+
.parallel(10);
197+
198+
const rules =
199+
sources.fork()
200+
.flatMap(parse_xml)
201+
.errors((err, push) => {
202+
push(null, { err });
203+
})
204+
.zip(files.fork())
205+
.map(([ { ruleset, err }, name ]) => {
206+
if (err) {
207+
err.message += ` (${name})`;
208+
this.emit('error', err);
209+
} else {
210+
return ruleset;
211+
}
212+
});
213+
214+
// This async call determines the current versions of the supported browsers
215+
216+
async.parallel({
217+
stable: firefox_version_fetch(stable_version_url),
218+
esr: firefox_version_fetch(esr_version_url),
219+
chromium: chromium_version_fetch(chromium_version_url)
220+
}, (err, versions) => {
221+
versions.esr_major = versions.esr.replace(/_.*/, "");
222+
223+
let stable_url = `https://hg.mozilla.org/releases/mozilla-release/raw-file/FIREFOX_${versions.stable}_RELEASE/security/manager/ssl/nsSTSPreloadList.inc`;
224+
let dev_url = `https://hg.mozilla.org/releases/mozilla-aurora/raw-file/tip/security/manager/ssl/nsSTSPreloadList.inc`;
225+
let esr_url = `https://hg.mozilla.org/releases/mozilla-esr${versions.esr_major}/raw-file/FIREFOX_${versions.esr}_RELEASE/security/manager/ssl/nsSTSPreloadList.inc`;
226+
let chromium_url = `https://chromium.googlesource.com/chromium/src.git/+/${versions.chromium}/net/http/transport_security_state_static.json?format=TEXT`;
227+
228+
// This async call fetches and builds hash for the HSTS preloads included in
229+
// each of the supported browsers.
230+
231+
async.parallel({
232+
esr: parse_include(esr_url),
233+
dev: parse_include(dev_url),
234+
stable: parse_include(esr_url),
235+
chromium: parse_json(chromium_url)
236+
}, (err, structs) => {
237+
238+
files.fork().zipAll([ sources.fork(), rules ])
239+
.consume((err, ruleset_data, push, next) => {
240+
if(ruleset_data === _.nil){
241+
push(null, ruleset_data);
242+
return;
243+
}
244+
245+
let [name, source, ruleset] = ruleset_data;
246+
bar.tick();
247+
248+
let targets = ruleset.target.map(target => target.$.host);
249+
250+
// First, determine whether the targets are included in the preload
251+
// list for all relevant browsers. If at least one of them isn't, we
252+
// can't delete the file. If any of them are, we need to perform
253+
// additional checks on the included domain.
254+
255+
let can_be_deleted = true;
256+
let preload_check_mapping = {};
257+
for(let target of targets){
258+
let [included, included_domain] = check_inclusion(structs, target);
259+
if(included == false){
260+
can_be_deleted = false;
261+
} else {
262+
preload_check_mapping[included_domain] =
263+
preload_check_mapping[included_domain] || [];
264+
preload_check_mapping[included_domain].push(target);
265+
}
266+
}
267+
268+
// Additional checks are as follows: curl the included domain, and if
269+
// the 'preload' directive is included, remove all targets that
270+
// correspond to that domain. If the preload directive is absent, make
271+
// sure not to delete the file.
272+
//
273+
// Ref: https://github.com/EFForg/https-everywhere/pull/7081
274+
275+
let checks = [];
276+
let source_overwritten = false;
277+
for(let included_domain in preload_check_mapping){
278+
checks.push(cb => {
279+
check_https_redirection_and_header_directives(
280+
included_domain, (err, meets_header_requirements) => {
281+
if(err) return cb(err);
282+
if(meets_header_requirements){
283+
for(let target of preload_check_mapping[included_domain]){
284+
source = remove_target_from_xml(source, target);
285+
source_overwritten = true;
286+
}
287+
} else {
288+
can_be_deleted = false;
289+
}
290+
cb();
291+
}
292+
);
293+
});
294+
}
295+
296+
// After building the additonal checks in the form of the
297+
// checks array of functions, run the checks in parallel and
298+
// perform the appropriate actions based on the results.
299+
300+
async.parallel(checks, err => {
301+
if(err) return console.log(err);
302+
if(can_be_deleted){
303+
console.log(`All targets removed for ${name}, deleting file...`);
304+
push(null, unlink(`${rules_dir}/${name}`));
305+
next();
306+
} else {
307+
if(source_overwritten == false){
308+
push(null, false);
309+
next();
310+
} else {
311+
console.log(`Some targets removed for ${name}, overwriting file...`);
312+
push(null, write_file(`${rules_dir}/${name}`, source));
313+
next();
314+
}
315+
}
316+
});
317+
318+
})
319+
.filter(Boolean)
320+
.parallel(10)
321+
.each(_ => _);
322+
});
323+
});

‎utils/hsts-prune/package.json‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"name": "hsts-prune",
3+
"version": "1.0.0",
4+
"description": "",
5+
"main": "index.js",
6+
"scripts": {
7+
"test": "echo \"Error: no test specified\" && exit 1"
8+
},
9+
"author": "William Budington",
10+
"license": "GPL-3.0",
11+
"dependencies": {
12+
"JSONStream": "^1.3.0",
13+
"async": "^2.1.4",
14+
"base64-stream": "^0.1.3",
15+
"escape-string-regexp": "^1.0.5",
16+
"highland": "^2.10.1",
17+
"progress": "^1.1.8",
18+
"request": "^2.79.0",
19+
"split": "^1.0.0",
20+
"stream-filter": "^2.1.0",
21+
"xml2js": "^0.4.17"
22+
}
23+
}

0 commit comments

Comments
 (0)