import requests
import pandas as pd
from pymongo import MongoClient


class DataCrawler(object):
    def __init__(self):
        self.cities = list(pd.read_csv('city_data.csv')['city'])
        client = MongoClient(host='localhost', port=27017)
        db = client.Laborday
        self.col = db.ticket

    def get_city_trip(self):
        for city in self.cities:
            print('正在爬取城市:{}的数据!'.format(city))
            res = requests.get('https://travelsearch.fliggy.com/async/queryItemResult.do?searchType='
                               'product&keyword={}&category=SCENIC&pagenum=1'.format(city))
            data = res.json()
            itemPagenum = data['data']['data'].get('itemPagenum')
            if itemPagenum is not None:
                page_count = itemPagenum['data']['count']

                data_list = data['data']['data']['itemProducts']['data']['list'][0]['auctions']
                for ticket in data_list:
                    ticket['city'] = city
                    self.col.insert_one(ticket)
                print('成功爬取城市:{}的第{}页数据!'.format(city, 1))

                if page_count > 1:
                    for page in range(2, page_count+1):
                        res = requests.get('https://travelsearch.fliggy.com/async/queryItemResult.do?searchType='
                                           'product&keyword={}&category=SCENIC&pagenum={}'.format(city, page))
                        data = res.json()
                        data_list = data['data']['data']['itemProducts']['data']['list'][0]['auctions']
                        for ticket in data_list:
                            ticket['city'] = city
                            self.col.insert_one(ticket)
                        print('成功爬取城市:{}的第{}页数据!'.format(city, page))


if __name__ == '__main__':
    data_crawler = DataCrawler()
    data_crawler.get_city_trip()