已知某個網址http://www.***.com，扒出所有的新聞

app.js文件：

const fs = require('fs');
const request = require('superagent');
const cheerio = require('cheerio');
const mapLimit = require('async/mapLimit');
const url = 'http://www.hiynn.com/hy-zh';
let counter = 0

// const fetch = require('whatwg-fetch');

function getNews() {
    return new Promise((resolve, reject) => {
        request.get(`${url}/news.html`)
            .end((err, data) => {
                if (err) throw err

                let $ = cheerio.load(data.text), links = []
                $('h3 > a').each((index, item) => {
                    let $item = $(item)

                    // href="#" || href="undefined"
                    if ($item.attr('href').match('#') || !$item.attr('href')) return
                    links.push($item.attr('href'))
                })

                resolve(links)
            })
    })
        .then(links => {
            /**
             * mapLimit
             *@param arg[0]  需要遍歷的集合
             *@param arg[1]  最大併發請求數
             *@param arg[2]  迭代處理函數 -> 參數1：傳入集合中的每一項；參數2：回調函數
             *@param arg[3]  所有迭代完成後的回調函數 -> 參數1：err；參數2：結果集
             */
            mapLimit(links, 10, function (link, cb) {
                request.get(`${url}/${link}`)
                    .end((err, data) => {
                        if (err) throw err

                        let $ = cheerio.load(data.text, {
                            xmlMode: true,
                            decodeEntities: false,
                            normalizeWhitespace: true,
                            withDomLvl1: false
                        }), news = [], create_time=[]
                        let creat_time = link.slice(5, 11);
                        create_time.push(creat_time.slice(0, 2), creat_time.slice(2, 4), creat_time.slice(4, 6))
                        let createTime = create_time.join('-');

                        let title = $('#tab1').html();

                        let Title = title.match(/<h3>(.+)<\/h3>/)[1].trim();

                        let typeArr = [1, 2];

                        let allStr = $('#tab1').html();
                        //匹配圖片（g表示匹配所有結果i表示區分大小寫）
                        var imgReg = /<img.*?(?:>|\/>)/gi;
                        //匹配src屬性
                        var srcReg = /src=[\'\"]?([^\'\"]*)[\'\"]?/i;
                        var arr = allStr.match(imgReg);

                        console.log('arr------->', arr);
                        let imgSrc = [];

                        if(arr) {

                            for (var i = 0; i < arr.length; i++) {

                             imgSrc = arr[i].match(srcReg);

                            }

                        }else {

                            imgSrc = [ '<img src="./images/news160812/1.jpg" width="100%" text-align:"center"=""/>', "./images/news161222/3.png"]

                        }

                        // console.log('imgSrc[0]------->', imgSrc[0]);


                        console.log('imgSrc---->', imgSrc);


                        news.push({
                            counter:counter,
                            author:'小明',
                            content:'',
                            creat_time:`20${createTime}`,
                            link: link,
                            content_html: $('#tab1').html(),
                            deleted_flag: 0,
                            important: 2,
                            title: Title,
                            img: imgSrc[1],
                            type: Math.ceil(Math.random() * typeArr.length)
                        })

                        counter++;
                        console.log(`${counter}/${links.length}`)

                        // 限制每秒的請求個數
                        setTimeout(() => {
                            // callback 必須調用，否則不會向下執行
                            cb(null, news)
                        }, 1000)
                    })
            }, function (err, coll) {
                if (err) throw err

                // 二維數組 -> 一維數組
                let news = Array.prototype.concat.apply([], coll)
                // 寫入文件
                fs.writeFile('./links.js', JSON.stringify(news), 'utf8', function (err) {
                    if (err) throw err

                        console.log('寫入成功');



                })
            })
        })
}

getNews()

package.json

{
  "name": "Crawler",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1",
    "start": "node app.js"
  },
  "keywords": [],
  "author": "",
  "license": "ISC",
  "dependencies": {
    "async": "^2.6.0",
    "cheerio": "^1.0.0-rc.2",
    "superagent": "^3.8.1",
    "whatwg-fetch": "^2.0.3"
  }
}

最後生成一個links.js文件，文件內容是一個數組，包括所有新聞。