Use itertoolz.sliding_window.

bamos · bamos · commit 2251d949ba32 · 2014-11-15T18:05:23.000-05:00
diff --git a/python3/word-counter.py b/python3/word-counter.py
@@ -3,13 +3,7 @@
 from collections import Counter
 import argparse
 import re
-from itertools import islice
-import operator
-
-url_re = re.compile(
-  'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
-)
-
+from toolz.itertoolz import sliding_window
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--numWords',type=int,default=10)
@@ -18,29 +12,22 @@
 parser.add_argument('file',type=str)
 args = parser.parse_args()
 
-# Inspired by http://stackoverflow.com/questions/6822725
-def window(seq, n):
-  it = iter(seq)
-  result = tuple(islice(it, n))
-  if len(result) == n:
-    yield result
-  for elem in it:
-    result = result[1:] + (elem,)
-    useThisWindow = True
-    for i in result:
-      if len(i) < args.minWordLength:
-        useThisWindow = False
-        break
-      elif url_re.search(i):
-        useThisWindow = False
-        break
-    if useThisWindow:
-      yield result
+def filter_func(tup):
+  for word in tup:
+    if len(word) < args.minWordLength:
+      return False
+    elif re.search('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',word):
+      return False
+  else:
+    return True
 
+def filtered_window(seq,n):
+  return filter(filter_func, sliding_window(n,seq))
+      
 with open(args.file,'r') as f:
   content = f.read().replace('\n',' ').lower()
   words = re.findall(r'\S+', content)
   for i in range(1,args.maxTuples+1):
     print("\n=== Sliding Window: {} ===".format(i))
-    for tup in Counter(window(words,i)).most_common(args.numWords):
+    for tup in Counter(filtered_window(words,i)).most_common(args.numWords):
       print("  {}: '{}'".format(tup[1]," ".join(tup[0])))
