请教两个关于使用 python 爬去哪儿，携程等机票网站的问题

初学 python 。

近期公司由于业务原因，需要想办法获取到携程与去哪儿的机票信息。于是我尝试用 python+urllib 对这两个网站上的信息进行抓取。

去哪儿的爬虫代码如下：（初学 python 。代码有很多不合理之处。望海涵。）

# -*- coding:utf-8 -*-

import urllib
from urllib import request


class QunaerSpider:
    __query_flights_base_url = 'http://flight.qunar.com/twelli/longwell?'

    __user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103'

    __referer_base_url = 'http://flight.qunar.com/site/interroundtrip_compare.htm?'

    __referer_end_url = '&from=fi_re_search&lowestPrice=null&isInter=true&favoriteKey=&showTotalPr=null'

    def create_flights_query_url(self, from_city, from_date, to_city, to_date):
        """根据用户传参,返回组装好的机票请求用 String 格式的 url 数据

        Args:
            from_city: 出发城市
            from_date: 出发日期
            to_city: 抵达城市
            to_date: 抵达日期

        Returns:
            返回组装好的机票请求用 String 格式的 url 数据.

        Raise:
            None

        """
        from_city_encoded = urllib.request.quote(from_city.encode('utf-8'))
        from_date_encoded = urllib.request.quote(from_date)
        to_city_encoded = urllib.request.quote(to_city.encode('utf-8'))
        to_date_encoded = urllib.request.quote(to_date)
        url = QunaerSpider.__query_flights_base_url
        # 初始化参数对象
        parameter_dict = {}
        parameter_dict['from'] = 'qunarindex'
        parameter_dict['fromCity'] = from_city_encoded
        parameter_dict['fromDate'] = from_date_encoded
        parameter_dict['isInter'] = 'true'
        parameter_dict['prePay'] = 'true'
        parameter_dict['locale'] = 'zh'
        parameter_dict['mergeFlag'] = '0'
        parameter_dict['nextNDays'] = '0'
        parameter_dict['op'] = '1'
        parameter_dict['reset'] = 'true'
        parameter_dict['searchLangs'] = 'zh'
        parameter_dict['searchType'] = 'RoundTripFlight'
        parameter_dict['toCity'] = to_city_encoded
        parameter_dict['toDate'] = to_date_encoded
        parameter_dict['version'] = 'thunder'
        parameter_dict['http://www.travelco.com/searchArrivalAirport'] = to_city_encoded
        parameter_dict['http://www.travelco.com/searchDepartureAirport'] = from_city_encoded
        parameter_dict['http://www.travelco.com/searchDepartureTime'] = from_date_encoded
        parameter_dict['http://www.travelco.com/searchReturnTime'] = to_date_encoded
        # f+时间戳
        parameter_dict['xd'] = 'f1469358871776'
        parameter_dict['www'] = 'true'
        parameter_dict['wyf'] = '0P8HfQ5%2F%2FYA%2FWldSERAyfudSERU0dUd0ERPj%3D%3D%3D%3D%7C1441321882698'
        parameter_dict['departureCity'] = from_city_encoded
        parameter_dict['arrivalCity'] = to_city_encoded
        parameter_dict['departureDate'] = from_date_encoded
        parameter_dict['returnDate'] = to_date_encoded
        # token.加不加暂时看不出来什么影响
        parameter_dict['_token'] = '6455'

        # 拼装 query_rul
        for k, v in parameter_dict.items():
            url = url + '&' + k + '=' + v

        print('请求字符串为\n%s' % url)
        return url

    def create_referer_url(self, from_city, from_date, to_city, to_date):

        from_city_encoded = urllib.request.quote(from_city.encode('utf-8'))
        from_date_encoded = urllib.request.quote(from_date)
        to_city_encoded = urllib.request.quote(to_city.encode('utf-8'))
        to_date_encoded = urllib.request.quote(to_date)
        url = QunaerSpider.__referer_base_url
        # 初始化参数对象
        parameter_dict = {}
        parameter_dict['from'] = 'qunarindex'
        parameter_dict['fromCity'] = from_city_encoded
        parameter_dict['fromDate'] = from_date_encoded
        parameter_dict['toCity'] = to_city_encoded
        parameter_dict['toDate'] = to_date_encoded
        # TODO 暂时写死
        parameter_dict['fromCode'] = 'CTU'
        parameter_dict['toCode'] = 'TYO'

        # 拼装 query_rul
        for k, v in parameter_dict.items():
            url = url + '&' + k + '=' + v

        url += QunaerSpider.__referer_end_url

        print('Referer 为\n%s' % url)

        return url

    def query_flights(self, url, referer_url):
        """根据用户传参,返回组装好的机票请求用 String 格式的 url 数据

               Args:
                   url: 机票接口

               Returns:
                   封装好的机票数据

               Raise:
                   None

        """
        req = request.Request(url)
        req.add_header('Host', 'flight.qunar.com')
        req.add_header('Accept', '*/*')
        req.add_header('User-Agent', QunaerSpider.__user_agent)
        req.add_header('Connection', 'keep-alive')
        req.add_header('Accept-Encoding', 'gzip, deflate, sdch')
        req.add_header('Content-Type', 'application/json')
        req.add_header('Accept-Language', 'zh-CN,zh;q=0.8')
        req.add_header('Referer', referer_url)

        with request.urlopen(req) as f:
            # 读取数据
            data = f.read()
            print(f.status, f.reason)
            print('Data:', data.decode('utf-8'))


qunaerSpider = QunaerSpider()
referer_url = qunaerSpider.create_referer_url('成都', '2016-08-20', '东京', '2016-09-11')
url = qunaerSpider.create_flights_query_url('成都', '2016-08-20', '东京', '2016-09-11')
qunaerSpider.query_flights(url, referer_url)

去哪儿网遇到的问题

爬虫返回的信息为

200 OK Data: {isLimit:true}

使用同样的 url ，通过浏览器却能正常访问。想知道原因以及修正办法。

携程网遇到的问题

携程网的爬虫代码还没开始写。不过大概分析了下，携程网是 post 请求。并且每次请求时会验证“ SearchKey ”与“ TransNo ”。关于这两个参数，有有经验的同学知道是在哪里，通过什么方式获取到的吗？附上携程的请求参数列表。

{
	"FlightWay": "D",
	"SegmentList": [{
		"DCityCode": "CTU",
		"ACityCode": "TYO",
		"DCity": "Chengdu|成都(CTU)|28|CTU|480",
		"ACity": "Tokyo|东京(TYO)|228|TYO|540",
		"DepartDate": "2016-8-1"
	}, {
		"DCityCode": "TYO",
		"ACityCode": "CTU",
		"DCity": "Tokyo|东京(TYO)|228|TYO|480",
		"ACity": "Chengdu|成都(CTU)|28|CTU|540",
		"DepartDate": "2016-9-10"
	}],
	"TransferCityID": 0,
	"Quantity": 1,
	// 每次请求发生变化，携程会验证此 key
	"TransNo": "5516072421000033701",
	"SearchRandomKey": "",
	"IsAsync": 1,
	"RecommendedFlightSwitch": 1,
	// 每次请求发生变化。携程会验证此 key
	"SearchKey": "BEBFB6F8C0C56B8561A9B435AE822DF4D499B75C2FFA74D481318741A7F9537EFB59C5327342DE0D1A11D1E626A03C6C843FE6E311D4819F",
	"MultiPriceUnitSwitch": 1,
	"TransferCitySwitch": false,
	"EngineScoreABTest": "B",
	"AdjacentDateKey": "",
	"SearchStrategySwitch": 1,
	"MaxSearchCount": 3,
	"TicketRemarkSwitch": 1,
	"RowNum": "1500",
	"TicketRemarkChannels": ["GDS-WS", "ZY-WS"],
	"AddSearchLogOneByOne": true,
	"TFAirlineQTE": "AA",
	"IsWifiPackage": 0
}

谢谢各位。