diff --git a/README.md b/README.md index 58d5ffd..a1e13a0 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ crawler-python ============== -crawler-python is a simple crawler fraework for collection online data from websites for **academic purpose**. +crawler-python is a simple crawler framework for collection online data from websites for **academic purpose**. ## Quick Start @@ -12,13 +12,14 @@ crawler-python is a simple crawler fraework for collection online data from webs * [Yelp](http://www.yelp.com) + It's better to work with goengent (Oops, yelp blocks it) +* [amazon](http://www.amazon.com) +* [gdgt](http://www.gdgt.com) + ## Future websites -* [amazon](http://www.amazon.com) -* [dpreviews](http://www.dpreview.com) -* [gdgt](http://www.gdgt.com) * [buzzillion](http://www.buzzillions.com/) +* [dpreviews](http://www.dpreview.com) * [sina_weibo](http://www.weibo.com) ## TODO diff --git a/amazon_crawler.py b/amazon_crawler.py index 25c04df..8af3812 100644 --- a/amazon_crawler.py +++ b/amazon_crawler.py @@ -54,18 +54,18 @@ def main(): #print amazon_camera("B00EFILPHA", proxies) - #print amazon_reviews("B00EFILPHA", proxies) + print amazon_reviews("B00EFILPHA") # amazon_prd_img("B00EFILPHA", "images") - for dirname, dirnames, filenames in os.walk('D:\workspace\camera\small'): - for filename in filenames: - file_path = os.path.join(dirname, filename) + #for dirname, dirnames, filenames in os.walk('D:\workspace\camera\small'): + # for filename in filenames: + # file_path = os.path.join(dirname, filename) - pid = filename[:-5] - if not os.path.isfile(os.path.join("D:\workspace\camera\images", pid) + ".jpg"): - amazon_prd_img(pid, "D:\workspace\camera\images") + # pid = filename[:-5] + # if not os.path.isfile(os.path.join("D:\workspace\camera\images", pid) + ".jpg"): + # amazon_prd_img(pid, "D:\workspace\camera\images") - print pid + # print pid if __name__ == "__main__": main() diff --git a/crawler/contrib/amazon.py b/crawler/contrib/amazon.py index a7698b2..9ed482c 100644 --- a/crawler/contrib/amazon.py +++ b/crawler/contrib/amazon.py @@ -138,7 +138,7 @@ def amazon_prd_img(prd_id, target_dir=".", proxies=None): save.write(data) save.close() -def amazon_reviews(prd_id, proxies): +def amazon_reviews(prd_id, proxies=None): prd_reviews = [] base_review_url = "http://www.amazon.com/product-reviews/" + prd_id + "/?ie=UTF8&showViewpoints=0&pageNumber=" + "%s" + "&sortBy=bySubmissionDateDescending" @@ -159,13 +159,19 @@ def amazon_reviews(prd_id, proxies): pagn_bar = revs_soup.find("span", {"class": "paging"}) pagn_bar_str = str(pagn_bar) + #print pagn_bar_str + # before: Oct, 2013 + #mch = re.search(r"cm_cr_pr_top_link_next_([0-9]+)", pagn_bar_str) - mch = re.search(r"cm_cr_pr_top_link_next_([0-9]+)", pagn_bar_str) + # modified on: 24th, Feb 2014 + mch = re.search(r"pageNumber=([0-9]+)\">Next", pagn_bar_str) if mch is None: break + if len(mch.groups()) > 0: pagn = mch.group(1) + print pagn revs_soup.decompose() else: revs_soup.decompose() @@ -181,25 +187,27 @@ def extract_reviews(data, pid): table_reg = re.compile(r"(.*)
") table_match = table_reg.search(data) + if table_match: table_cont = table_match.group(1) - review_reg = re.compile(r"-->(.*?)(!--|$)") + # before Oct 2013 + #review_reg = re.compile(r"-->(.*?)(!--|$)") + + # modified on 24th, Feb 2014 + review_reg = re.compile(r"
(.*?)