"""

Copyleft 2013 Osama Khalid.

This program is free software: you can redistribute it and/or modify

it under the terms of the GNU Affero General Public License as

published by the Free Software Foundation, either version 3 of the

License, or (at your option) any later version.

This tool extracts HTTPSEverywhere rulesets and converts any given URL

to the secure version if it exists, or it returns None. Here is a

sample:

>>> import find_rules

>>> replacer = find_rules.FindRules("/path/to/default.rulesets")

>>> replacer.find("http://en.wikipedia.org/")

'https://en.wikipedia.org/'

>>> replacer.find("http://en.wikipedi.org/") # With a typo

>>>

"""

import re

import xml.etree.ElementTree as ET

class FindRules:

def __init__(self, filename):

self.extract_rulesets(filename)

def verify_target(self, target, host):

matching_target = target.strip("*.")

matching_target = matching_target.strip(".*")

if target.startswith("*."):

if host.endswith(matching_target):

#print target, "matches", host

return True

elif target.endswith(".*"):

if host.startswith(matching_target):

#print target, "matches", host

return True

else:

if host == matching_target:

#print target, "matches", host

return True

def convert_to_python(self, matching, replacement):

"""Instead of $1 that is used by Javascript,

Python uses \1."""

new_matching = matching.replace(")?", "|)") # to avoid "unmatched group" error

new_replacement = re.sub(r"\$(\d)", r"\\g<\1>", replacement)

return new_matching, new_replacement

def extract_rulesets(self, filename):

tree = ET.parse(filename)

root = tree.getroot()

self.dict = {}

for child in root:

if child.tag == "ruleset":

if "default_off" in child.attrib:

continue

ruleset_name = child.attrib['name']

ruleset = child.getchildren()

self.dict[ruleset_name] = {}

self.dict[ruleset_name]['targets'] = []

self.dict[ruleset_name]['rules'] = []

self.dict[ruleset_name]['exclusions'] = []

for rule in ruleset:

if rule.tag == "target":

self.dict[ruleset_name]['targets'].append(rule.attrib['host'])

if rule.tag == "rule":

self.dict[ruleset_name]['rules'].append((rule.attrib['from'], rule.attrib['to']))

if rule.tag == "exclusion":

self.dict[ruleset_name]['exclusions'].append(rule.attrib['pattern'])

def find(self, url):

hostname_regex = r"https?://([^/]+)"

try: #Remove

host = re.findall(hostname_regex, url)[0]

except IndexError, e:

print url

raise IndexError, e

# In HTTPSEverywhere, URLs must contain a '/'.

if url.replace("http://", "").find("/") == -1:

url += "/"

for ruleset in self.dict:

for target in self.dict[ruleset]['targets']:

if self.verify_target(target, host):

for exclusion in self.dict[ruleset]['exclusions']:

if re.findall(exclusion, url):

return None

for rule in self.dict[ruleset]['rules']:

matching_regex = rule[0] # "from"

replacement_regex = rule[1] # "to"

new_matching, new_replacement = self.convert_to_python(matching_regex, replacement_regex)

try:

replace_url = re.sub(new_matching, new_replacement, url)

except re.error, e:

print new_matching, new_replacement, url

raise re.error, e

if url != replace_url:

return replace_url

return None

if __name__ == "__main__":

import sys

filename = sys.argv[1]

url = sys.argv[2]

script = FindRules(filename)

replaced_url = script.find(url)

print replaced_url