python爬取淘寶商品信息

本文實現了用python爬取淘寶信息的功能，主要涉及訪問網頁、正則匹配、json格式化等技術。

# -*- coding: utf-8 -*

import requests
# import bs4
import re
import json


def open(keywords, page):
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36"}

    payload = {'q': keywords, 'sort': "sale-desc", 's': (page - 1) * 44}
    url = "https://s.taobao.com/search"

    res = requests.get(url, params=payload)
    # https://s.taobao.com/search?q=羽絨服&sort=sale-desc&s=44
    return res


def get_item(res):
    g_page_config = re.search(r'g_page_config = (.*?);\n', res.text)
    page_config_json = json.loads(g_page_config.group(1))
    page_item = page_config_json['mods']['itemlist']['data']['auctions']

    result = []  # 整理出我們關注的信息(ID,標題，鏈接，售價，銷量和商家)
    for each in page_item:
        dict1 = dict.fromkeys(('id', 'title', 'link', 'price', 'sale', 'shoper'))
        dict1['id'] = each['nid']
        dict1['title'] = each['title']
        dict1['link'] = each['detail_url']
        dict1['price'] = each['view_price']
        dict1['sale'] = each['view_sales']
        dict1['shoper'] = each['nick']
        result.append(dict1)

    return result


def count_sales(items):
    count = 0
    for each in items:
        if u'黑色' in each['title']:  # 規定只取標題中‘###'的商品
            count += int(re.search(r'\d+', each['sale']).group())

    return count


def main():
    keywords = raw_input("請輸入搜索關鍵詞：")  # 可以爲各種商品名稱  羽絨服
    length = 2  # 淘寶商品頁數
    total = 0

    for each in range(length):
        res = open(keywords, each + 1)
        # print res.text
        items = get_item(res)
        print items
        total += count_sales(items)  # 銷售總量
    print(total)


if __name__ == "__main__":
    main()