# -*- coding: utf-8 -*-

'''

;$$;

#############

#############;#####o

## o#########################

##### $###############################

## ###$ ######! ##########################

## ### $### ################### ######

### ### ##o#######################

###### ;### #### #####################

## ### ###### ######&&################

## ### ###### ## ############ #######

o## ######## ## ##################

##o ### #### #######o#######

## ###### ###########&#####

## #### #############!

### #########

#####& ## o####

###### ## ####*

## !## #####

## ##* ####; ##

##### #####o #####

#### ### ### $###o

### ## ####! $###

## #####

## ##

;## ### ;

##$ ##

####### ##

##### &## ##

### ### ###

### ### ##

## ;## ##

## ### ##

### ### ##

#### ##

### ##

##; ##

##$ ##&

## ##

##; ##

## ##;

### ### ##$

### ### ##

###################### #####&&&&&&&&&&&&###

### $#####$ ############&$o$&################################

# $&########&o

'''

# Build-in / Std

import os, sys, time, platform, random

import re, json, cookielib

# requirements

import requests, termcolor, html2text

try:

from bs4 import BeautifulSoup

except:

import BeautifulSoup

# module

from auth import islogin

from auth import Logging

"""

Note:

1. 身份验证由 `auth.py` 完成。

2. 身份信息保存在当前目录的 `cookies` 文件中。

3. `requests` 对象可以直接使用，身份信息已经自动加载。

By Luozijun (https://github.com/LuoZijun), 09/09 2015

"""

requests = requests.Session()

requests.cookies = cookielib.LWPCookieJar('cookies')

try:

requests.cookies.load(ignore_discard=True)

except:

Logging.error(u"你还没有登录知乎哦 ...")

Logging.info(u"执行 `python auth.py` 即可以完成登录。")

raise Exception("无权限(403)")

if islogin() != True:

Logging.error(u"你的身份信息已经失效，请重新生成身份信息( `python auth.py` )。")

raise Exception("无权限(403)")

reload(sys)

sys.setdefaultencoding('utf8')

class Question:

url = None

soup = None

def __init__(self, url, title=None):

if url[0:len(url) - 8] != "http://www.zhihu.com/question/":

raise ValueError("\"" + url + "\"" + " : it isn't a question url.")

else:

self.url = url

if title != None: self.title = title

def parser(self):

r = requests.get(self.url)

self.soup = BeautifulSoup(r.content)

def get_title(self):

if hasattr(self, "title"):

if platform.system() == 'Windows':

title = self.title.decode('utf-8').encode('gbk')

return title

else:

return self.title

else:

if self.soup == None:

self.parser()

soup = self.soup

title = soup.find("h2", class_="zm-item-title").string.encode("utf-8").replace("\n", "")

self.title = title

if platform.system() == 'Windows':

title = title.decode('utf-8').encode('gbk')

return title

else:

return title

def get_detail(self):

if self.soup == None:

self.parser()

soup = self.soup

detail = soup.find("div", id="zh-question-detail").div.get_text().encode("utf-8")

if platform.system() == 'Windows':

detail = detail.decode('utf-8').encode('gbk')

return detail

else:

return detail

def get_answers_num(self):

if self.soup == None:

self.parser()

soup = self.soup

answers_num = 0

if soup.find("h3", id="zh-question-answer-num") != None:

answers_num = int(soup.find("h3", id="zh-question-answer-num")["data-num"])

return answers_num

def get_followers_num(self):

if self.soup == None:

self.parser()

soup = self.soup

followers_num = int(soup.find("div", class_="zg-gray-normal").a.strong.string)

return followers_num

def get_topics(self):

if self.soup == None:

self.parser()

soup = self.soup

topic_list = soup.find_all("a", class_="zm-item-tag")

topics = []

for i in topic_list:

topic = i.contents[0].encode("utf-8").replace("\n", "")

if platform.system() == 'Windows':

topic = topic.decode('utf-8').encode('gbk')

topics.append(topic)

return topics

def get_all_answers(self):

answers_num = self.get_answers_num()

if answers_num == 0:

print "No answer."

return

yield

else:

error_answer_count = 0

my_answer_count = 0

for i in xrange((answers_num - 1) / 50 + 1):

if i == 0:

for j in xrange(min(answers_num, 50)):

if self.soup == None:

self.parser()

soup = BeautifulSoup(self.soup.encode("utf-8"))

is_my_answer = False

if soup.find_all("div", class_="zm-item-answer")[j].find("span", class_="count") == None:

my_answer_count += 1

is_my_answer = True

if soup.find_all("div", class_="zm-item-answer")[j].find("div", class_=" zm-editable-content clearfix") == None:

error_answer_count += 1

continue

author = None

if soup.find_all("h3", class_="zm-item-answer-author-wrap")[j].string == u"匿名用户":

author_url = None

author = User(author_url)

else:

author_tag = soup.find_all("h3", class_="zm-item-answer-author-wrap")[j].find_all("a")[1]

author_id = author_tag.string.encode("utf-8")

author_url = "http://www.zhihu.com" + author_tag["href"]

author = User(author_url, author_id)

if is_my_answer == True:

count = soup.find_all("div", class_="zm-item-answer")[j].find("a", class_="zm-item-vote-count").string

else:

count = soup.find_all("span", class_="count")[j - my_answer_count].string

if count[-1] == "K":

upvote = int(count[0:(len(count) - 1)]) * 1000

elif count[-1] == "W":

upvote = int(count[0:(len(count) - 1)]) * 10000

else:

upvote = int(count)

answer_url = "http://www.zhihu.com" + soup.find_all("a", class_="answer-date-link")[j]["href"]

answer = soup.find_all("div", class_=" zm-editable-content clearfix")[j - error_answer_count]

soup.body.extract()

soup.head.insert_after(soup.new_tag("body", **{'class': 'zhi'}))

soup.body.append(answer)

img_list = soup.find_all("img", class_="content_image lazy")

for img in img_list:

img["src"] = img["data-actualsrc"]

img_list = soup.find_all("img", class_="origin_image zh-lightbox-thumb lazy")

for img in img_list:

img["src"] = img["data-actualsrc"]

noscript_list = soup.find_all("noscript")

for noscript in noscript_list:

noscript.extract()

content = soup

answer = Answer(answer_url, self, author, upvote, content)

yield answer

else:

post_url = "http://www.zhihu.com/node/QuestionAnswerListV2"

_xsrf = self.soup.find("input", attrs={'name': '_xsrf'})["value"]

offset = i * 50

params = json.dumps(

{"url_token": int(self.url[-8:-1] + self.url[-1]), "pagesize": 50, "offset": offset})

data = {

'_xsrf': _xsrf,

'method': "next",

'params': params

}

header = {

'User-Agent': "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:34.0) Gecko/20100101 Firefox/34.0",

'Host': "www.zhihu.com",

'Referer': self.url

}

r = requests.post(post_url, data=data, headers=header)

answer_list = r.json()["msg"]

for j in xrange(min(answers_num - i * 50, 50)):

soup = BeautifulSoup(self.soup.encode("utf-8"))

answer_soup = BeautifulSoup(answer_list[j])

if answer_soup.find("div", class_=" zm-editable-content clearfix") == None:

continue

author = None

if answer_soup.find("h3", class_="zm-item-answer-author-wrap").string == u"匿名用户":

author_url = None

author = User(author_url)

else:

author_tag = answer_soup.find("h3", class_="zm-item-answer-author-wrap").find_all("a")[1]

author_id = author_tag.string.encode("utf-8")

author_url = "http://www.zhihu.com" + author_tag["href"]

author = User(author_url, author_id)

if answer_soup.find("span", class_="count") == None:

count = answer_soup.find("a", class_="zm-item-vote-count").string

else:

count = answer_soup.find("span", class_="count").string

if count[-1] == "K":

upvote = int(count[0:(len(count) - 1)]) * 1000

elif count[-1] == "W":

upvote = int(count[0:(len(count) - 1)]) * 10000

else:

upvote = int(count)

answer_url = "http://www.zhihu.com" + answer_soup.find("a", class_="answer-date-link")["href"]

answer = answer_soup.find("div", class_=" zm-editable-content clearfix")

soup.body.extract()

soup.head.insert_after(soup.new_tag("body", **{'class': 'zhi'}))

soup.body.append(answer)

img_list = soup.find_all("img", class_="content_image lazy")

for img in img_list:

img["src"] = img["data-actualsrc"]

img_list = soup.find_all("img", class_="origin_image zh-lightbox-thumb lazy")

for img in img_list:

img["src"] = img["data-actualsrc"]

noscript_list = soup.find_all("noscript")

for noscript in noscript_list:

noscript.extract()

content = soup

answer = Answer(answer_url, self, author, upvote, content)

yield answer

def get_top_i_answers(self, n):

# if n > self.get_answers_num():

# n = self.get_answers_num()

j = 0

answers = self.get_all_answers()

for answer in answers:

j = j + 1

if j > n:

break

yield answer

def get_top_answer(self):

for answer in self.get_top_i_answers(1):

return answer

def get_visit_times(self):

if self.soup == None:

self.parser()

soup = self.soup

return int(soup.find("meta", itemprop="visitsCount")["content"])

class User:

user_url = None

# session = None

soup = None

def __init__(self, user_url, user_id=None):

if user_url == None:

self.user_id = "匿名用户"

elif user_url[0:28] != "http://www.zhihu.com/people/":

raise ValueError("\"" + user_url + "\"" + " : it isn't a user url.")

else:

self.user_url = user_url

if user_id != None:

self.user_id = user_id

def parser(self):

r = requests.get(self.user_url)

soup = BeautifulSoup(r.content)

self.soup = soup

def get_user_id(self):

if self.user_url == None:

# print "I'm anonymous user."

if platform.system() == 'Windows':

return "匿名用户".decode('utf-8').encode('gbk')

else:

return "匿名用户"

else:

if hasattr(self, "user_id"):

if platform.system() == 'Windows':

return self.user_id.decode('utf-8').encode('gbk')

else:

return self.user_id

else:

if self.soup == None:

self.parser()

soup = self.soup

user_id = soup.find("div", class_="title-section ellipsis") \

.find("span", class_="name").string.encode("utf-8")

self.user_id = user_id

if platform.system() == 'Windows':

return user_id.decode('utf-8').encode('gbk')

else:

return user_id

def get_followees_num(self):

if self.user_url == None:

print "I'm anonymous user."

return 0

else:

if self.soup == None:

self.parser()

soup = self.soup

followees_num = int(soup.find("div", class_="zm-profile-side-following zg-clear") \

.find("a").strong.string)

return followees_num

def get_followers_num(self):

if self.user_url == None:

print "I'm anonymous user."

return 0

else:

if self.soup == None:

self.parser()

soup = self.soup

followers_num = int(soup.find("div", class_="zm-profile-side-following zg-clear") \

.find_all("a")[1].strong.string)

return followers_num

def get_agree_num(self):

if self.user_url == None:

print "I'm anonymous user."

return 0

else:

if self.soup == None:

self.parser()

soup = self.soup

agree_num = int(soup.find("span", class_="zm-profile-header-user-agree").strong.string)

return agree_num

def get_thanks_num(self):

if self.user_url == None:

print "I'm anonymous user."

return 0

else:

if self.soup == None:

self.parser()

soup = self.soup

thanks_num = int(soup.find("span", class_="zm-profile-header-user-thanks").strong.string)

return thanks_num

def get_asks_num(self):

if self.user_url == None:

print "I'm anonymous user."

return 0

else:

if self.soup == None:

self.parser()

soup = self.soup

asks_num = int(soup.find_all("span", class_="num")[0].string)

return asks_num

def get_answers_num(self):

if self.user_url == None:

print "I'm anonymous user."

return 0

else:

if self.soup == None:

self.parser()

soup = self.soup

answers_num = int(soup.find_all("span", class_="num")[1].string)

return answers_num

def get_collections_num(self):

if self.user_url == None:

print "I'm anonymous user."

return 0

else:

if self.soup == None:

self.parser()

soup = self.soup

collections_num = int(soup.find_all("span", class_="num")[3].string)

return collections_num

def get_followees(self):

if self.user_url == None:

print "I'm anonymous user."

return

yield

else:

followees_num = self.get_followees_num()

if followees_num == 0:

return

yield

else:

followee_url = self.user_url + "/followees"

r = requests.get(followee_url)

soup = BeautifulSoup(r.content)

for i in xrange((followees_num - 1) / 20 + 1):

if i == 0:

user_url_list = soup.find_all("h2", class_="zm-list-content-title")

for j in xrange(min(followees_num, 20)):

yield User(user_url_list[j].a["href"], user_url_list[j].a.string.encode("utf-8"))

else:

post_url = "http://www.zhihu.com/node/ProfileFolloweesListV2"

_xsrf = soup.find("input", attrs={'name': '_xsrf'})["value"]

offset = i * 20

hash_id = re.findall("hash_id": "(.*)"},", r.text)[0]

params = json.dumps({"offset": offset, "order_by": "created", "hash_id": hash_id})

data = {

'_xsrf': _xsrf,

'method': "next",

'params': params

}

header = {

'User-Agent': "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:34.0) Gecko/20100101 Firefox/34.0",

'Host': "www.zhihu.com",

'Referer': followee_url

}

r_post = requests.post(post_url, data=data, headers=header)

followee_list = r_post.json()["msg"]

for j in xrange(min(followees_num - i * 20, 20)):

followee_soup = BeautifulSoup(followee_list[j])

user_link = followee_soup.find("h2", class_="zm-list-content-title").a

yield User(user_link["href"], user_link.string.encode("utf-8"))

def get_followers(self):

if self.user_url == None:

print "I'm anonymous user."

return

yield

else:

followers_num = self.get_followers_num()

if followers_num == 0:

return

yield

else:

follower_url = self.user_url + "/followers"

r = requests.get(follower_url)

soup = BeautifulSoup(r.content)

for i in xrange((followers_num - 1) / 20 + 1):

if i == 0:

user_url_list = soup.find_all("h2", class_="zm-list-content-title")

for j in xrange(min(followers_num, 20)):

yield User(user_url_list[j].a["href"], user_url_list[j].a.string.encode("utf-8"))

else:

post_url = "http://www.zhihu.com/node/ProfileFollowersListV2"

_xsrf = soup.find("input", attrs={'name': '_xsrf'})["value"]

offset = i * 20

hash_id = re.findall("hash_id": "(.*)"},", r.text)[0]

params = json.dumps({"offset": offset, "order_by": "created", "hash_id": hash_id})

data = {

'_xsrf': _xsrf,

'method': "next",

'params': params

}

header = {

'User-Agent': "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:34.0) Gecko/20100101 Firefox/34.0",

'Host': "www.zhihu.com",

'Referer': follower_url

}

r_post = requests.post(post_url, data=data, headers=header)

follower_list = r_post.json()["msg"]

for j in xrange(min(followers_num - i * 20, 20)):

follower_soup = BeautifulSoup(follower_list[j])

user_link = follower_soup.find("h2", class_="zm-list-content-title").a

yield User(user_link["href"], user_link.string.encode("utf-8"))

def get_asks(self):

if self.user_url == None:

print "I'm anonymous user."

return

yield

else:

asks_num = self.get_asks_num()

if asks_num == 0:

return

yield

else:

for i in xrange((asks_num - 1) / 20 + 1):

ask_url = self.user_url + "/asks?page=" + str(i + 1)

r = requests.get(ask_url)

soup = BeautifulSoup(r.content)

for question in soup.find_all("a", class_="question_link"):

url = "http://www.zhihu.com" + question["href"]

title = question.string.encode("utf-8")

yield Question(url, title)

def get_answers(self):

if self.user_url == None:

print "I'm anonymous user."

return

yield

else:

answers_num = self.get_answers_num()

if answers_num == 0:

return

yield

else:

for i in xrange((answers_num - 1) / 20 + 1):

answer_url = self.user_url + "/answers?page=" + str(i + 1)

r = requests.get(answer_url)

soup = BeautifulSoup(r.content)

for answer in soup.find_all("a", class_="question_link"):

question_url = "http://www.zhihu.com" + answer["href"][0:18]

question_title = answer.string.encode("utf-8")

question = Question(question_url, question_title)

yield Answer("http://www.zhihu.com" + answer["href"], question, self)

def get_collections(self):

if self.user_url == None:

print "I'm anonymous user."

return

yield

else:

collections_num = self.get_collections_num()

if collections_num == 0:

return

yield

else:

for i in xrange((collections_num - 1) / 20 + 1):

collection_url = self.user_url + "/collections?page=" + str(i + 1)

r = requests.get(collection_url)

soup = BeautifulSoup(r.content)

for collection in soup.find_all("div", class_="zm-profile-section-item zg-clear"):

url = "http://www.zhihu.com" + \

collection.find("a", class_="zm-profile-fav-item-title")["href"]

name = collection.find("a", class_="zm-profile-fav-item-title").string.encode("utf-8")

yield Collection(url, name, self)

class Answer:

answer_url = None

# session = None

soup = None

def __init__(self, answer_url, question=None, author=None, upvote=None, content=None):

self.answer_url = answer_url

if question != None:

self.question = question

if author != None:

self.author = author

if upvote != None:

self.upvote = upvote

if content != None:

self.content = content

def parser(self):

r = requests.get(self.answer_url)

soup = BeautifulSoup(r.content)

self.soup = soup

def get_question(self):

if hasattr(self, "question"):

return self.question

else:

if self.soup == None:

self.parser()

soup = self.soup

question_link = soup.find("h2", class_="zm-item-title zm-editable-content").a

url = "http://www.zhihu.com" + question_link["href"]

title = question_link.string.encode("utf-8")

question = Question(url, title)

return question

def get_author(self):

if hasattr(self, "author"):

return self.author

else:

if self.soup == None:

self.parser()

soup = self.soup

if soup.find("h3", class_="zm-item-answer-author-wrap").string == u"匿名用户":

author_url = None

author = User(author_url)

else:

author_tag = soup.find("h3", class_="zm-item-answer-author-wrap").find_all("a")[1]

author_id = author_tag.string.encode("utf-8")

author_url = "http://www.zhihu.com" + author_tag["href"]

author = User(author_url, author_id)

return author

def get_upvote(self):

if hasattr(self, "upvote"):

return self.upvote

else:

if self.soup == None:

self.parser()

soup = self.soup

count = soup.find("span", class_="count").string

if count[-1] == "K":

upvote = int(count[0:(len(count) - 1)]) * 1000

elif count[-1] == "W":

upvote = int(count[0:(len(count) - 1)]) * 10000

else:

upvote = int(count)

return upvote

def get_content(self):

if hasattr(self, "content"):

return self.content

else:

if self.soup == None:

self.parser()

soup = BeautifulSoup(self.soup.encode("utf-8"))

answer = soup.find("div", class_=" zm-editable-content clearfix")

soup.body.extract()

soup.head.insert_after(soup.new_tag("body", **{'class': 'zhi'}))

soup.body.append(answer)

img_list = soup.find_all("img", class_="content_image lazy")

for img in img_list:

img["src"] = img["data-actualsrc"]

img_list = soup.find_all("img", class_="origin_image zh-lightbox-thumb lazy")

for img in img_list:

img["src"] = img["data-actualsrc"]

noscript_list = soup.find_all("noscript")

for noscript in noscript_list:

noscript.extract()

content = soup

self.content = content

return content

def to_txt(self):

content = self.get_content()

body = content.find("body")

br_list = body.find_all("br")

for br in br_list:

br.insert_after(content.new_string("\n"))

li_list = body.find_all("li")

for li in li_list:

li.insert_before(content.new_string("\n"))

if platform.system() == 'Windows':

anon_user_id = "匿名用户".decode('utf-8').encode('gbk')

else:

anon_user_id = "匿名用户"

if self.get_author().get_user_id() == anon_user_id:

if not os.path.isdir(os.path.join(os.path.join(os.getcwd(), "text"))):

os.makedirs(os.path.join(os.path.join(os.getcwd(), "text")))

if platform.system() == 'Windows':

file_name = self.get_question().get_title() + "--" + self.get_author().get_user_id() + "的回答.txt".decode(

'utf-8').encode('gbk')

else:

file_name = self.get_question().get_title() + "--" + self.get_author().get_user_id() + "的回答.txt"

print file_name

# if platform.system() == 'Windows':

# file_name = file_name.decode('utf-8').encode('gbk')

# print file_name

# else:

# print file_name

if os.path.exists(os.path.join(os.path.join(os.getcwd(), "text"), file_name)):

f = open(os.path.join(os.path.join(os.getcwd(), "text"), file_name), "a")

f.write("\n\n")

else:

f = open(os.path.join(os.path.join(os.getcwd(), "text"), file_name), "a")

f.write(self.get_question().get_title() + "\n\n")

else:

if not os.path.isdir(os.path.join(os.path.join(os.getcwd(), "text"))):

os.makedirs(os.path.join(os.path.join(os.getcwd(), "text")))

if platform.system() == 'Windows':

file_name = self.get_question().get_title() + "--" + self.get_author().get_user_id() + "的回答.txt".decode(

'utf-8').encode('gbk')

else:

file_name = self.get_question().get_title() + "--" + self.get_author().get_user_id() + "的回答.txt"

print file_name

# if platform.system() == 'Windows':

# file_name = file_name.decode('utf-8').encode('gbk')

# print file_name

# else:

# print file_name

f = open(os.path.join(os.path.join(os.getcwd(), "text"), file_name), "wt")

f.write(self.get_question().get_title() + "\n\n")

if platform.system() == 'Windows':

f.write("作者: ".decode('utf-8').encode('gbk') + self.get_author().get_user_id() + " 赞同: ".decode(

'utf-8').encode('gbk') + str(self.get_upvote()) + "\n\n")

f.write(body.get_text().encode("gbk"))

link_str = "原链接: ".decode('utf-8').encode('gbk')

f.write("\n" + link_str + self.answer_url.decode('utf-8').encode('gbk'))

else:

f.write("作者: " + self.get_author().get_user_id() + " 赞同: " + str(self.get_upvote()) + "\n\n")

f.write(body.get_text().encode("utf-8"))

f.write("\n" + "原链接: " + self.answer_url)

f.close()

# def to_html(self):

# content = self.get_content()

# if self.get_author().get_user_id() == "匿名用户":

# file_name = self.get_question().get_title() + "--" + self.get_author().get_user_id() + "的回答.html"

# f = open(file_name, "wt")

# print file_name

# else:

# file_name = self.get_question().get_title() + "--" + self.get_author().get_user_id() + "的回答.html"

# f = open(file_name, "wt")

# print file_name

# f.write(str(content))

# f.close()

def to_md(self):

content = self.get_content()

if platform.system() == 'Windows':

anon_user_id = "匿名用户".decode('utf-8').encode('gbk')

else:

anon_user_id = "匿名用户"

if self.get_author().get_user_id() == anon_user_id:

if platform.system() == 'Windows':

file_name = self.get_question().get_title() + "--" + self.get_author().get_user_id() + "的回答.md".decode(

'utf-8').encode('gbk')

else:

file_name = self.get_question().get_title() + "--" + self.get_author().get_user_id() + "的回答.md"

print file_name

# if platform.system() == 'Windows':

# file_name = file_name.decode('utf-8').encode('gbk')

# print file_name

# else:

# print file_name

if not os.path.isdir(os.path.join(os.path.join(os.getcwd(), "markdown"))):

os.makedirs(os.path.join(os.path.join(os.getcwd(), "markdown")))

if os.path.exists(os.path.join(os.path.join(os.getcwd(), "markdown"), file_name)):

f = open(os.path.join(os.path.join(os.getcwd(), "markdown"), file_name), "a")

f.write("\n")

else:

f = open(os.path.join(os.path.join(os.getcwd(), "markdown"), file_name), "a")

f.write("# " + self.get_question().get_title() + "\n")

else:

if not os.path.isdir(os.path.join(os.path.join(os.getcwd(), "markdown"))):

os.makedirs(os.path.join(os.path.join(os.getcwd(), "markdown")))

if platform.system() == 'Windows':

file_name = self.get_question().get_title() + "--" + self.get_author().get_user_id() + "的回答.md".decode(

'utf-8').encode('gbk')

else:

file_name = self.get_question().get_title() + "--" + self.get_author().get_user_id() + "的回答.md"

print file_name

# file_name = self.get_question().get_title() + "--" + self.get_author().get_user_id() + "的回答.md"

# if platform.system() == 'Windows':

# file_name = file_name.decode('utf-8').encode('gbk')

# print file_name

# else:

# print file_name

f = open(os.path.join(os.path.join(os.getcwd(), "markdown"), file_name), "wt")

f.write("# " + self.get_question().get_title() + "\n")

if platform.system() == 'Windows':

f.write("## 作者: ".decode('utf-8').encode('gbk') + self.get_author().get_user_id() + " 赞同: ".decode(

'utf-8').encode('gbk') + str(self.get_upvote()) + "\n")

else:

f.write("## 作者: " + self.get_author().get_user_id() + " 赞同: " + str(self.get_upvote()) + "\n")

text = html2text.html2text(content.decode('utf-8')).encode("utf-8")

r = re.findall(r'\*\*(.*?)\*\*', text)

for i in r:

if i != " ":

text = text.replace(i, i.strip())

r = re.findall(r'_(.*)_', text)

for i in r:

if i != " ":

text = text.replace(i, i.strip())

r = re.findall(r'!\[\]\((?:.*?)\)', text)

for i in r:

text = text.replace(i, i + "\n\n")

if platform.system() == 'Windows':

f.write(text.decode('utf-8').encode('gbk'))

link_str = "#### 原链接: ".decode('utf-8').encode('gbk')

f.write(link_str + self.answer_url.decode('utf-8').encode('gbk'))

else:

f.write(text)

f.write("#### 原链接: " + self.answer_url)

f.close()

def get_visit_times(self):

if self.soup == None:

self.parser()

soup = self.soup

for tag_p in soup.find_all("p"):

if "所属问题被浏览" in tag_p.contents[0].encode('utf-8'):

return int(tag_p.contents[1].contents[0])

def get_voters(self):

if self.soup == None:

self.parser()

soup = self.soup

data_aid = soup.find("div", class_="zm-item-answer ")["data-aid"]

request_url = 'http://www.zhihu.com/node/AnswerFullVoteInfoV2'

if session == None:

create_session()

s = session

r = s.get(request_url, params={"params": "{\"answer_id\":\"%d\"}" % int(data_aid)})

soup = BeautifulSoup(r.content)

voters_info = soup.find_all("span")[1:-1]

if len(voters_info) == 0:

return

yield

else:

for voter_info in voters_info:

if voter_info.string == ( u"匿名用户、" or u"匿名用户"):

voter_url = None

yield User(voter_url)

else:

voter_url = "http://www.zhihu.com" + str(voter_info.a["href"])

voter_id = voter_info.a["title"].encode("utf-8")

yield User(voter_url, voter_id)

class Collection:

url = None

# session = None

soup = None

def __init__(self, url, name=None, creator=None):

if url[0:len(url) - 8] != "http://www.zhihu.com/collection/":

raise ValueError("\"" + url + "\"" + " : it isn't a collection url.")

else:

self.url = url

# print 'collection url',url

if name != None:

self.name = name

if creator != None:

self.creator = creator

def parser(self):

r = requests.get(self.url)

soup = BeautifulSoup(r.content)

self.soup = soup

def get_name(self):

if hasattr(self, 'name'):

if platform.system() == 'Windows':

return self.name.decode('utf-8').encode('gbk')

else:

return self.name

else:

if self.soup == None:

self.parser()

soup = self.soup

self.name = soup.find("h2", id="zh-fav-head-title").string.encode("utf-8").strip()

if platform.system() == 'Windows':

return self.name.decode('utf-8').encode('gbk')

return self.name

def get_creator(self):

if hasattr(self, 'creator'):

return self.creator

else:

if self.soup == None:

self.parser()

soup = self.soup

creator_id = soup.find("h2", class_="zm-list-content-title").a.string.encode("utf-8")

creator_url = "http://www.zhihu.com" + soup.find("h2", class_="zm-list-content-title").a["href"]

creator = User(creator_url, creator_id)

self.creator = creator

return creator

def get_all_answers(self):

if self.soup == None:

self.parser()

soup = self.soup

answer_list = soup.find_all("div", class_="zm-item")

if len(answer_list) == 0:

print "the collection is empty."

return

yield

else:

question_url = None

question_title = None

for answer in answer_list:

if not answer.find("p", class_="note"):

question_link = answer.find("h2")

if question_link != None:

question_url = "http://www.zhihu.com" + question_link.a["href"]

question_title = question_link.a.string.encode("utf-8")

question = Question(question_url, question_title)

answer_url = "http://www.zhihu.com" + answer.find("span", class_="answer-date-link-wrap").a["href"]

author = None

if answer.find("h3", class_="zm-item-answer-author-wrap").string == u"匿名用户":

author_url = None

author = User(author_url)

else:

author_tag = answer.find("h3", class_="zm-item-answer-author-wrap").find_all("a")[0]

author_id = author_tag.string.encode("utf-8")

author_url = "http://www.zhihu.com" + author_tag["href"]

author = User(author_url, author_id)

yield Answer(answer_url, question, author)

i = 2

while True:

r = requests.get(self.url + "?page=" + str(i))

answer_soup = BeautifulSoup(r.content)

answer_list = answer_soup.find_all("div", class_="zm-item")

if len(answer_list) == 0:

break

else:

for answer in answer_list:

if not answer.find("p", class_="note"):

question_link = answer.find("h2")

if question_link != None:

question_url = "http://www.zhihu.com" + question_link.a["href"]

question_title = question_link.a.string.encode("utf-8")

question = Question(question_url, question_title)

answer_url = "http://www.zhihu.com" + answer.find("span", class_="answer-date-link-wrap").a[

"href"]

author = None

if answer.find("h3", class_="zm-item-answer-author-wrap").string == u"匿名用户":

# author_id = "匿名用户"

author_url = None

author = User(author_url)

else:

author_tag = answer.find("h3", class_="zm-item-answer-author-wrap").find_all("a")[0]

author_id = author_tag.string.encode("utf-8")

author_url = "http://www.zhihu.com" + author_tag["href"]

author = User(author_url, author_id)

yield Answer(answer_url, question, author)

i = i + 1

def get_top_i_answers(self, n):

j = 0

answers = self.get_all_answers()

for answer in answers:

j = j + 1

if j > n:

break

yield answer