Skip to content

Commit 011de03

Browse files
daviddrysdaledaviddrysdale
committed
Genericize buildgeocodingdata.py
There is nothing geodata-specific about it, it is easily adapted to work with arbitrary per-prefix data.
1 parent e0146ee commit 011de03

7 files changed

Lines changed: 60 additions & 43 deletions

File tree

‎python/phonenumbers/geocoder.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@
5656
import os
5757
import sys
5858
if (os.path.basename(sys.argv[0]) == "buildmetadatafromxml.py" or
59-
os.path.basename(sys.argv[0]) == "buildgeocodingdata.py"):
59+
os.path.basename(sys.argv[0]) == "buildprefixdata.py"):
6060
print >> sys.stderr, "Failed to import generated data (but OK as during autogeneration)"
6161
GEOCODE_DATA = {'1': {'en': u'United States'}}
6262
GEOCODE_LONGEST_PREFIX = 1

‎python/phonenumbers/geodata/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
"""Geocoding data, mapping each prefix to a dict of locale:locationname.
1+
"""Per-prefix data, mapping each prefix to a dict of locale:name.
22

33
Auto-generated file, do not edit by hand.
44
"""

‎python/phonenumbers/phonenumbermatcher.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848
import os
4949
import sys
5050
if (os.path.basename(sys.argv[0]) == "buildmetadatafromxml.py" or
51-
os.path.basename(sys.argv[0]) == "buildgeocodingdata.py"):
51+
os.path.basename(sys.argv[0]) == "buildprefixdata.py"):
5252
print >> sys.stderr, "Failed to import generated data (but OK as during autogeneration)"
5353
_ALT_NUMBER_FORMATS = {}
5454
else:

‎python/phonenumbers/phonenumberutil.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
import os
5050
import sys
5151
if (os.path.basename(sys.argv[0]) == "buildmetadatafromxml.py" or
52-
os.path.basename(sys.argv[0]) == "buildgeocodingdata.py"):
52+
os.path.basename(sys.argv[0]) == "buildprefixdata.py"):
5353
print >> sys.stderr, "Failed to import generated data (but OK as during autogeneration)"
5454
_COUNTRY_CODE_TO_REGION_CODE = {1: ("US",)}
5555
_AVAILABLE_SHORT_REGION_CODES = []

‎python/tests/testgeodata/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
"""Geocoding data, mapping each prefix to a dict of locale:locationname.
1+
"""Per-prefix data, mapping each prefix to a dict of locale:name.
22
33
Auto-generated file, do not edit by hand.
44
"""

tools/python/buildgeocodingdata.py renamed to tools/python/buildprefixdata.py

Lines changed: 47 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
#!/usr/bin/env python
2-
"""Script to read the libphonenumber geocoding metadata and generate Python code.
2+
"""Script to read the libphonenumber per-prefix metadata and generate Python code.
33
44
Invocation:
5-
buildgeocodingdata.py indir outfile
5+
buildprefixdata.py indir outfile
66
7-
Processes all of the geocoding data under the given input directory and emit
7+
Processes all of the per-prefix data under the given input directory and emit
88
generated Python code.
99
"""
1010

11-
# Based on original geocoding data files from libphonenumber:
12-
# resources/geocoding/*/*.txt
11+
# Based on original metadata data files from libphonenumber:
12+
# resources/geocoding/*/*.txt, resources/carrier/*/*.txt
1313
# Copyright (C) 2011 The Libphonenumber Authors
1414
#
1515
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -28,6 +28,7 @@
2828
import sys
2929
import glob
3030
import re
31+
import getopt
3132
import datetime
3233

3334
if sys.version_info >= (3, 0):
@@ -47,13 +48,13 @@ def prnt(*args, **kwargs):
4748
def u(s):
4849
return unicode(s)
4950

50-
GEODATA_SUFFIX = ".txt"
51+
PREFIXDATA_SUFFIX = ".txt"
5152
BLANK_LINE_RE = re.compile(r'^\s*$', re.UNICODE)
5253
COMMENT_LINE_RE = re.compile(r'^\s*#.*$', re.UNICODE)
53-
DATA_LINE_RE = re.compile(r'^(?P<prefix>\d+)\|(?P<location>.*)$', re.UNICODE)
54+
DATA_LINE_RE = re.compile(r'^\+?(?P<prefix>\d+)\|(?P<location>.*)$', re.UNICODE)
5455

5556
# Boilerplate header
56-
GEODATA_FILE_PROLOG = '''"""Geocoding data, mapping each prefix to a dict of locale:locationname.
57+
PREFIXDATA_FILE_PROLOG = '''"""Per-prefix data, mapping each prefix to a dict of locale:name.
5758
5859
Auto-generated file, do not edit by hand.
5960
"""
@@ -76,8 +77,8 @@ def u(s):
7677
""" % datetime.datetime.now().year
7778

7879

79-
def load_geodata_file(geodata, filename, locale, overall_prefix):
80-
"""Load geocoding data from the given file, for the given locale and prefix.
80+
def load_prefixdata_file(prefixdata, filename, locale, overall_prefix):
81+
"""Load per-prefix data from the given file, for the given locale and prefix.
8182
8283
We assume that this file:
8384
- is encoded in UTF-8
@@ -100,9 +101,9 @@ def load_geodata_file(geodata, filename, locale, overall_prefix):
100101
if not prefix.startswith(overall_prefix):
101102
raise Exception("%s:%d: Prefix %s is not within %s" %
102103
(filename, lineno, prefix, overall_prefix))
103-
if prefix not in geodata:
104-
geodata[prefix] = {}
105-
geodata[prefix][locale] = location
104+
if prefix not in prefixdata:
105+
prefixdata[prefix] = {}
106+
prefixdata[prefix][locale] = location
106107
elif BLANK_LINE_RE.match(uline):
107108
pass
108109
elif COMMENT_LINE_RE.match(uline):
@@ -112,21 +113,21 @@ def load_geodata_file(geodata, filename, locale, overall_prefix):
112113
(filename, lineno, line))
113114

114115

115-
def load_geodata(indir):
116-
"""Load geocoding data from the given top-level directory.
116+
def load_prefixdata(indir):
117+
"""Load per-prefix data from the given top-level directory.
117118
118-
Geocoding data is assumed to be held in files <indir>/<locale>/<prefix>.txt.
119+
Prefix data is assumed to be held in files <indir>/<locale>/<prefix>.txt.
119120
The same prefix may occur in multiple files, giving the location's name in
120121
different locales.
121122
"""
122-
geodata = {} # prefix => dict mapping location to location name
123+
prefixdata = {} # prefix => dict mapping location to location name
123124
for locale in os.listdir(indir):
124125
if not os.path.isdir(os.path.join(indir, locale)):
125126
continue
126-
for filename in glob.glob(os.path.join(indir, locale, "*%s" % GEODATA_SUFFIX)):
127+
for filename in glob.glob(os.path.join(indir, locale, "*%s" % PREFIXDATA_SUFFIX)):
127128
overall_prefix, ext = os.path.splitext(os.path.basename(filename))
128-
load_geodata_file(geodata, filename, locale, overall_prefix)
129-
return geodata
129+
load_prefixdata_file(prefixdata, filename, locale, overall_prefix)
130+
return prefixdata
130131

131132

132133
def _stable_dict_repr(strdict):
@@ -137,28 +138,44 @@ def _stable_dict_repr(strdict):
137138
return "{%s}" % ", ".join(lines)
138139

139140

140-
def output_geodata_code(geodata, outfilename):
141-
"""Output the geocoding data in Python form to the given file """
141+
def output_prefixdata_code(prefixdata, outfilename, varprefix):
142+
"""Output the per-prefix data in Python form to the given file """
142143
with open(outfilename, "w") as outfile:
143144
longest_prefix = 0
144-
prnt(GEODATA_FILE_PROLOG, file=outfile)
145+
prnt(PREFIXDATA_FILE_PROLOG, file=outfile)
145146
prnt(COPYRIGHT_NOTICE, file=outfile)
146-
prnt("GEOCODE_DATA = {", file=outfile)
147-
for prefix in sorted(geodata.keys()):
147+
prnt("%s_DATA = {" % varprefix, file=outfile)
148+
for prefix in sorted(prefixdata.keys()):
148149
if len(prefix) > longest_prefix:
149150
longest_prefix = len(prefix)
150-
prnt(" '%s':%s," % (prefix, _stable_dict_repr(geodata[prefix])), file=outfile)
151+
prnt(" '%s':%s," % (prefix, _stable_dict_repr(prefixdata[prefix])), file=outfile)
151152
prnt("}", file=outfile)
152-
prnt("GEOCODE_LONGEST_PREFIX = %d" % longest_prefix, file=outfile)
153+
prnt("%s_LONGEST_PREFIX = %d" % (varprefix, longest_prefix), file=outfile)
153154

154155

155156
def _standalone(argv):
156157
"""Parse the given input directory and emit generated code."""
157-
if len(argv) != 2:
158+
varprefix = "GEOCODE"
159+
try:
160+
opts, args = getopt.getopt(argv, "hv:", ("help", "var="))
161+
except getopt.GetoptError:
158162
prnt(__doc__, file=sys.stderr)
159163
sys.exit(1)
160-
geodata = load_geodata(argv[0])
161-
output_geodata_code(geodata, argv[1])
164+
for opt, arg in opts:
165+
if opt in ("-h", "--help"):
166+
prnt(__doc__, file=sys.stderr)
167+
sys.exit(1)
168+
elif opt in ("-v", "--var"):
169+
varprefix = arg
170+
else:
171+
prnt("Unknown option %s" % opt, file=sys.stderr)
172+
prnt(__doc__, file=sys.stderr)
173+
sys.exit(1)
174+
if len(args) != 2:
175+
prnt(__doc__, file=sys.stderr)
176+
sys.exit(1)
177+
prefixdata = load_prefixdata(args[0])
178+
output_prefixdata_code(prefixdata, args[1], varprefix)
162179

163180

164181
if __name__ == "__main__":

‎tools/python/makefile‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,12 @@ locale: $(PYDIR)/phonenumbers/geodata/locale.py
1515
# Generate Python files from geocoding data
1616
$(PYDIR)/phonenumbers/geodata:
1717
mkdir $@
18-
$(PYDIR)/phonenumbers/geodata/__init__.py: buildgeocodingdata.py $(TOPDIR)/resources/geocoding | $(PYDIR)/phonenumbers/geodata
19-
$(PYTHON) buildgeocodingdata.py $(TOPDIR)/resources/geocoding $@
18+
$(PYDIR)/phonenumbers/geodata/__init__.py: buildprefixdata.py $(TOPDIR)/resources/geocoding | $(PYDIR)/phonenumbers/geodata
19+
$(PYTHON) buildprefixdata.py --var GEOCODE $(TOPDIR)/resources/geocoding $@
2020
$(PYDIR)/tests/testgeodata:
2121
mkdir $@
22-
$(PYDIR)/tests/testgeodata/__init__.py: buildgeocodingdata.py $(TOPDIR)/resources/test/geocoding | $(PYDIR)/tests/testgeodata
23-
$(PYTHON) buildgeocodingdata.py $(TOPDIR)/resources/test/geocoding $@
22+
$(PYDIR)/tests/testgeodata/__init__.py: buildprefixdata.py $(TOPDIR)/resources/test/geocoding | $(PYDIR)/tests/testgeodata
23+
$(PYTHON) buildprefixdata.py --var GEOCODE $(TOPDIR)/resources/test/geocoding $@
2424
geodata: $(PYDIR)/phonenumbers/geodata/__init__.py $(PYDIR)/tests/testgeodata/__init__.py
2525

2626
# Generate Python files from metadata
@@ -133,10 +133,10 @@ $(PY3DIR)/%.py: $(PYDIR)/%.py | $(PY3DIR)
133133
$(PY3DIR)/phonenumbers/geodata/locale.py: DumpLocale.class | $(PY3DIR)/phonenumbers/geodata
134134
java DumpLocale -python3 > $@
135135
locale3: $(PY3DIR)/phonenumbers/geodata/locale.py
136-
$(PY3DIR)/phonenumbers/geodata/__init__.py: buildgeocodingdata.py $(TOPDIR)/resources/geocoding | $(PY3DIR)/phonenumbers/geodata
137-
$(PYTHON3) buildgeocodingdata.py $(TOPDIR)/resources/geocoding $@
138-
$(PY3DIR)/tests/testgeodata/__init__.py: buildgeocodingdata.py $(TOPDIR)/resources/test/geocoding | $(PY3DIR)/tests/testgeodata
139-
$(PYTHON3) buildgeocodingdata.py $(TOPDIR)/resources/test/geocoding $@
136+
$(PY3DIR)/phonenumbers/geodata/__init__.py: buildprefixdata.py $(TOPDIR)/resources/geocoding | $(PY3DIR)/phonenumbers/geodata
137+
$(PYTHON3) buildprefixdata.py --var GEOCODE $(TOPDIR)/resources/geocoding $@
138+
$(PY3DIR)/tests/testgeodata/__init__.py: buildprefixdata.py $(TOPDIR)/resources/test/geocoding | $(PY3DIR)/tests/testgeodata
139+
$(PYTHON3) buildprefixdata.py --var GEOCODE $(TOPDIR)/resources/test/geocoding $@
140140
geo3data: $(PY3DIR)/phonenumbers/geodata/__init__.py $(PY3DIR)/tests/testgeodata/__init__.py
141141

142142
$(PY3DIR)/phonenumbers/data/__init__.py: $(TOPDIR)/resources/PhoneNumberMetadata.xml $(TOPDIR)/resources/PhoneNumberAlternateFormats.xml buildmetadatafromxml.py

0 commit comments

Comments
 (0)