# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class LianjiaItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
pic = scrapy.Field()
title = scrapy.Field()
detail_url = scrapy.Field()
price = scrapy.Field()
publish_info = scrapy.Field()
pic_list = scrapy.Field()
house_code = scrapy.Field()
ucid = scrapy.Field()
agent_name = scrapy.Field()
agent_phone = scrapy.Field()
# -*- coding: utf-8 -*-
import scrapy
from LianJia.items import LianjiaItem
import re
import json
import requests
class LianjiaSpider(scrapy.Spider):
name = 'lianjia'
allowed_domains = ['lianjia.com']
start_urls = ['https://www.lianjia.com/city/']
def parse(self, response):
# 獲取到的是新房的url
city_url_list = response.xpath("//div[@class='city_province']//li/a/@href").extract()
# print(city_url_list)
city_name_list = response.xpath("//div[@class='city_province']//li/a/text()").extract()
for index in range(len(city_url_list)):
city_name = city_name_list[index]
city_url = city_url_list[index]
# print(city_url)
# 城市首字母
city_alp = re.findall(r"https://(\w*).", city_url)[0]
# print(city_alp)
# 拼接租房城市url
city_url = "https://" + city_alp + ".lianjia.com/zufang/"
# print("--------------------{}開始下載-------------------------------".format(city_name))
yield scrapy.Request(url=city_url, callback=self.get_area_url)
def get_area_url(self, response):
# print(response.body.decode("utf-8"))
# 獲取城區url
area_url_list = response.xpath("//li[@data-type='district'][position()>1]/a/@href").extract()
for area_url in area_url_list:
area_url = re.findall(r"(.*)/zufang/", response.url)[0] + area_url
# print(area_url)
yield scrapy.Request(url=area_url, callback=self.get_business_url)
def get_business_url(self, response):
# 獲取商圈url
business_url_list = response.xpath("//li[@data-type='bizcircle'][position()>1]/a/@href").extract()
# print(business_url_list)
for business_url in business_url_list:
business_url = re.findall(r"(.*)/zufang/", response.url)[0] + business_url
# print(business_url)
yield scrapy.Request(url=business_url, callback=self.get_page_url)
def get_page_url(self, response):
# 獲取最大頁碼
max_page = response.xpath("//div[@class='content__pg']/@data-totalpage").extract()
max_page = int(max_page[0]) if max_page else 0
# print(max_page)
# 遍歷最大頁 拼接完整的page_url
# ---------page=0時 不會執行下面----------
for page in range(max_page):
page_url = response.url + "pg{}/#contentList".format(page + 1)
# print(page_url)
yield scrapy.Request(url=page_url, callback=self.get_page_data)
def get_page_data(self, response):
# 縮小範圍
fang_xml_list = response.xpath("//div[@class='content__list']/div")
# print(fang_xml_list)
for fang_xml in fang_xml_list:
# 獲取圖片
pic = fang_xml.xpath(".//img/@data-src").extract()
pic = pic[0] if pic else ''
# print(pic)
# 獲取標題
title = fang_xml.xpath(".//p[@class='content__list--item--title twoline']/a/text()").extract()[0].strip()
# 獲取詳情url
detail_url = fang_xml.xpath(".//p[@class='content__list--item--title twoline']/a/@href").extract()[0]
detail_url = "https://bj.lianjia.com" + detail_url
# print(title)
# 獲取價格
price = fang_xml.xpath(".//em/text()").extract()[0]
# print(price)
item = LianjiaItem()
item["pic"] = pic
item["title"] = title
item["detail_url"] = detail_url
item["price"] = price
# print(item)
yield scrapy.Request(url=detail_url, callback=self.get_detail_data, meta={"data": item}, dont_filter=True)
def get_detail_data(self, response):
item = response.meta["data"]
# 獲取發佈信息
publish_info = response.xpath("//ul/li[contains(text(), '發佈')]/text()").extract()
publish_info = publish_info[0] if publish_info else ''
# print(publish_info)
# 獲取圖片信息
pic_list = response.xpath("//ul[@class='content__article__slide__wrapper']/div/img/@data-src").extract()
# print(pic_list)
# 獲取房源編號
house_code = re.findall(r"/zufang/(.*?).html", response.url)[0]
# print(house_code)
# 獲取ucid
ucid = response.xpath("//span[@class='contact__im im__online']/@data-im_id").extract()
# print(ucid)
if ucid:
ucid = ucid[0]
# 拼接完整的經紀人接口
brokers_url = "https://bj.lianjia.com/zufang/aj/house/brokers?"
agent_api = brokers_url + "house_codes={}&position=bottom&ucid={}".format(house_code, ucid)
# print(agent_api)
item["publish_info"] = publish_info
item["pic_list"] = pic_list
item["house_code"] = house_code
item["ucid"] = ucid
yield scrapy.Request(url=agent_api, callback=self.get_agent_data, meta={"data": item}, dont_filter=True)
# 獲取經紀人信息
def get_agent_data(self, response):
# 將response對象轉成json對象
result = response.body.decode("utf-8")
json_data = json.loads(result)
# print(json_data)
item = response.meta["data"]
house_code = item.get("house_code")
# 經紀人姓名
agent_name = json_data.get("data").get(house_code).get(house_code).get("contact_name")
# print(agent_name)
# 經紀人電話
agent_phone = json_data.get("data").get(house_code).get(house_code).get("tp_number")
# print(agent_phone)
item["agent_name"] = agent_name
item["agent_phone"] = agent_phone
yield item
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymongo
import pymysql
import redis
import json
# class LianjiaPipeline(object):
# def __init__(self):
# self.count = 1
#
# def process_item(self, item, spider):
# print(self.count, dict(item))
# self.count += 1
# return item
# 插入redis數據庫
class RedisPipeline(object):
def __init__(self):
self.count = 1
self.r = redis.Redis(host="localhost", port="6379", db=3)
def process_item(self, item, spider):
# print(item)
item_dict = dict(item)
print(self.count, item_dict)
item_str = json.dumps(item_dict)
self.r.lpush("lianjia", item_str)
# print("insert successfully")
self.count += 1
return item
# 插入mongodb
class MongodbPipeline(object):
def __init__(self):
mongo_client = pymongo.MongoClient("localhost", 27017)
self.db = mongo_client.lianjia
def process_item(self, item, spider):
item = dict(item)
self.db.lianjia.insert(item)
return item
# 插入mysql
class MysqlPipeline(object):
def __init__(self):
self.conn = pymysql.connect("localhost", "root", "123456", "lianjia")
self.cursor = self.conn.cursor()
def process_item(self, item, spider):
title = item["title"]
# print(title)
price = item["price"]
agent_name = item["agent_name"]
agent_phone = item["agent_phone"]
import time
# 時間戳 數據監控時用
refresh_time = int(time.time())
sql = "insert into fang(title, price, agent_name, agent_phone, refresh_time) values('{}', '{}', '{}', '{}', '{}')"\
.format(title, price, agent_name, agent_phone, refresh_time)
try:
self.cursor.execute(sql)
self.conn.commit()
except Exception as e:
print(e)
self.conn.rollback()
return item
def __del__(self):
self.cursor.close()
self.conn.close()
4.main.py （項目執行腳本）
from scrapy import cmdline
cmdline.execute("scrapy crawl lianjia --nolog".split())
# cmdline.execute("scrapy crawl lianjia".split())
