爬
iwjw.com 的房源，全部是空页面，肿么办？
headers = {
'Cookie':'IW_UUID2_COOKIES=6a2030687b904bb8bce1c6846c99b1a6; IW_UUID_COOKIES=e3e01670134042d494cd70c725c4698f; iw_user_last_housetype_h5=2; Hm_lvt_3a10bbf8b4afa9e5ab91d66b6944e813=1461846272,1461846860; Hm_lpvt_3a10bbf8b4afa9e5ab91d66b6944e813=1461909938; iw_user_last_housetype=2'
}
url = '
http://m.iwjw.com/sale/shanghai/?kw=%E5%8F%A4%E6%A1%90%E5%85%AC%E5%AF%93'web_data = requests.get(url,headers=headers)
soup = BeautifulSoup(web_data.text,'lxml')
得到的结果：
/Library/Frameworks/Python.framework/Versions/3.5/bin/python3.5 /Users/Emma_Tang/PycharmProjects/house_to_buy/
get_house_iwjw.py<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<meta content="IE=Edge" http-equiv="X-UA-Compatible"/>
<title>二手房列表-上海--爱屋吉屋</title>
<meta content="" name="keywords"/>
<meta content="" name="description"/>
<meta content="no-cache, no-store, must-revalidate" http-equiv="Cache-Control"/>
<meta content="no-cache" http-equiv="Pragma"/>
<meta content="0" http-equiv="Expires"/>
<meta content="爱屋吉屋" name="apple-mobile-web-app-title"/>
<meta content="width=device-width,initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, user-scalable=no" name="viewport"/>
<meta content="portrait" name="screen-orientation"/>
<meta content="portrait" name="x5-orientation"/>
<meta content="yes" name="apple-mobile-web-app-capable"/>
<link href="
http://resource.iwjw.com/iwjw-h5/img/common/screen_logo.jpg" rel="apple-touch-icon-precomposed"/>
<link href="
http://resource.iwjw.com/iwjw-h5/common_4.3.2.1.css" rel="stylesheet"/>
<link href="
http://resource.iwjw.com/iwjw-h5/list_4.2.css" rel="stylesheet"/>
<script>
window.pageConfig = {
siteUrl: "
http://www.iwjw.com",
mobileSiteUrl: '
http://m.iwjw.com',
datacollectUrl:'
http://collect.iwjwagent.com/dataCollect/',
recommendUrl:'
http://recommend.iwjw.com',
staticUrl: '
http://resource.iwjw.com/iwjw-h5/',
sell: 'true' == 'true' ? true : false,
rent: 'true' == 'true' ? true : false,
provincepy: "shanghai",
provinceid: "2",
locationsJson: [{"flag":"01","id":2,"lat":31.238092,"lineCode":0,"lon":121.479659,"name":"上海","shortSpell":"sh","spell":"shanghai"},{"flag":"02","id":12438,"lat":39.920176,"lineCode":0,"lon":116.407808,"name":"北京","shortSpell":"bj","spell":"beijing"},{"flag":"04","id":40000,"lat":23.135736,"lineCode":0,"lon":113.271143,"name":"广州","shortSpell":"gz","spell":"guangzhou"},{"flag":"03","id":56000,"lat":22.547923,"lineCode":0,"lon":114.063918,"name":"深圳","shortSpell":"sz","spell":"shenzhen"},{"flag":"","id":71049,"lat":30.281202,"lineCode":0,"lon":120.161596,"name":"杭州","shortSpell":"hz","spell":"hangzhou"},{"flag":"","id":71099,"lat":39.137214,"lineCode":0,"lon":117.186954,"name":"天津","shortSpell":"tj","spell":"tianjin"},{"flag":"","id":86724,"lat":32.058854,"lineCode":0,"lon":118.784457,"name":"南京","shortSpell":"nj","spell":"nanjing"},{"flag":"","id":86725,"lat":30.595037,"lineCode":0,"lon":114.291675,"name":"武汉","shortSpell":"wh","spell":"wuhan"},{"flag":"","id":98289,"lat":30.661125,"lineCode":0,"lon":104.071599,"name":"成都","shortSpell":"cd","spell":"chengdu"},{"flag":"","id":98290,"lat":29.561329,"lineCode":0,"lon":106.548784,"name":"重庆","shortSpell":"cq","spell":"chongqing"}],
ht: '2',
platform: ''
};
var _hmt = _hmt || [];
</script>
</head>
<body>
<script>
window.pageConfig = window.pageConfig || {};
window.pageConfig.list = {
provinceid: "2",
provincepy: "shanghai",
provincename: "上海",
housetype: "2",
siteUrl: "
http://www.iwjw.com",
prices: '[{"key":-1,"txt":"全部"},{"key":1,"txt":"100 万以下"},{"key":2,"txt":"100-150 万"},{"key":3,"txt":"150-200 万"},{"key":4,"txt":"200-300 万"},{"key":5,"txt":"300-500 万"},{"key":6,"txt":"500-700 万"},{"key":7,"txt":"700-1000 万"},{"key":8,"txt":"1000 万以上"}]',
options: {
kw: "古桐公寓",
g: "0",
areaId: "0",
stationId: "0",
ip: "-1",
ia: "-1",
sp: "-1",
ep: "-1",
sa: "-1",
ea: "-1",
rn: "-1",
fe: "-1",
dt: "-1",
o: "0"
}
};
</script><div id="wx_pic" style="position: absolute;top: -100000px;"><img src="
http://resource.iwjw.com/iwjw-h5/img/common/wx_share_logo.jpg"/></div>
<div id="iwjw">
<div class="mod-h5-container">
<div class="mod-list">
</div>
</div>
</div>
<script src="
http://resource.iwjw.com/iwjw-h5/common_4.3.4.js"></script>
<script src="
http://resource.iwjw.com/iwjw-h5/list_4.3.4.js"></script>
</body>
<script>
var _hmt = _hmt || [];
(function() {
var hm = document.createElement("script");
hm.src = "//
hm.baidu.com/hm.js?3a10bbf8b4afa9e5ab91d66b6944e813";
hm.async = 1;
var s = document.getElementsByTagName("script")[0];
s.parentNode.insertBefore(hm, s);
})();
var _bdhmProtocol = (("https:" == document.location.protocol) ? " https://" : " http://");
document.write(unescape("%3Cscript async src='" + _bdhmProtocol + "
hm.baidu.com/h.js%3Fd3a10bbf8b4afa9e5ab91d66b6944e813' type='text/javascript'%3E%3C/script%3E"));
</script></html>
Process finished with exit code 0