這個框架有很多細節等着我們去發現,本文主要介紹下。

線程池設計

對於小白來說可以好好學習下

package us.codecraft.webmagic.thread;

import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;

/**
 * Thread pool for workers.<br><br>
 * Use {@link java.util.concurrent.ExecutorService} as inner implement. <br><br>
 * New feature: <br><br>
 * 1. Block when thread pool is full to avoid poll many urls without process. <br><br>
 * 2. Count of thread alive for monitor.
 *
 * @author [email protected]
 * @since 0.5.0
 */
public class CountableThreadPool {

    private int threadNum;

    private AtomicInteger threadAlive = new AtomicInteger();

    private ReentrantLock reentrantLock = new ReentrantLock();

    private Condition condition = reentrantLock.newCondition();

    public CountableThreadPool(int threadNum) {
        this.threadNum = threadNum;
        this.executorService = Executors.newFixedThreadPool(threadNum);
    }

    public CountableThreadPool(int threadNum, ExecutorService executorService) {
        this.threadNum = threadNum;
        this.executorService = executorService;
    }

    public void setExecutorService(ExecutorService executorService) {
        this.executorService = executorService;
    }

    public int getThreadAlive() {
        return threadAlive.get();
    }

    public int getThreadNum() {
        return threadNum;
    }

    private ExecutorService executorService;

    public void execute(final Runnable runnable) {


        if (threadAlive.get() >= threadNum) {
            try {
                reentrantLock.lock();
                while (threadAlive.get() >= threadNum) {
                    try {
                        condition.await();
                    } catch (InterruptedException e) {
                    }
                }
            } finally {
                reentrantLock.unlock();
            }
        }
        threadAlive.incrementAndGet();
        executorService.execute(new Runnable() {
            @Override
            public void run() {
                try {
                    runnable.run();
                } finally {
                    try {
                        reentrantLock.lock();
                        threadAlive.decrementAndGet();
                        condition.signal();
                    } finally {
                        reentrantLock.unlock();
                    }
                }
            }
        });
    }

    public boolean isShutdown() {
        return executorService.isShutdown();
    }

    public void shutdown() {
        executorService.shutdown();
    }


}

代理池設計

這塊我覺得需要很大的改造

package us.codecraft.webmagic.proxy;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Task;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * A simple ProxyProvider. Provide proxy as round-robin without heartbeat and error check. It can be used when all proxies are stable.
 * @author [email protected]
 *         Date: 17/4/16
 *         Time: 10:18
 * @since 0.7.0
 */
public class SimpleProxyProvider implements ProxyProvider {

    private final List<Proxy> proxies;

    private final AtomicInteger pointer;

    public SimpleProxyProvider(List<Proxy> proxies) {
        this(proxies, new AtomicInteger(-1));
    }

    private SimpleProxyProvider(List<Proxy> proxies, AtomicInteger pointer) {
        this.proxies = proxies;
        this.pointer = pointer;
    }

    public static SimpleProxyProvider from(Proxy... proxies) {
        List<Proxy> proxiesTemp = new ArrayList<Proxy>(proxies.length);
        for (Proxy proxy : proxies) {
            proxiesTemp.add(proxy);
        }
        return new SimpleProxyProvider(Collections.unmodifiableList(proxiesTemp));
    }

    @Override
    public void returnProxy(Proxy proxy, Page page, Task task) {
        //Donothing
    }

    @Override
    public Proxy getProxy(Task task) {
        return proxies.get(incrForLoop());
    }

    private int incrForLoop() {
        int p = pointer.incrementAndGet();
        int size = proxies.size();
        if (p < size) {
            return p;
        }
        while (!pointer.compareAndSet(p, p % size)) {
            p = pointer.get();
        }
        return p % size;
    }
}

選擇器

這塊xpath,部分語法不支持，但是你還可以發現還有個xpath2

package us.codecraft.webmagic.selector;

import net.sf.saxon.lib.NamespaceConstant;
import net.sf.saxon.xpath.XPathEvaluator;
import org.apache.log4j.Logger;
import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.DomSerializer;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import javax.xml.namespace.NamespaceContext;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

/**
 * 支持xpath2.0的選擇器。包裝了HtmlCleaner和Saxon HE。<br>
 *
 * @author [email protected] <br>
 *         Date: 13-4-21
 *         Time: 上午9:39
 */
public class Xpath2Selector implements Selector {

    private String xpathStr;

    private XPathExpression xPathExpression;

    private Logger logger = Logger.getLogger(getClass());

    public Xpath2Selector(String xpathStr) {
        this.xpathStr = xpathStr;
        try {
            init();
        } catch (XPathExpressionException e) {
            throw new IllegalArgumentException("XPath error!", e);
        }
    }

    enum XPath2NamespaceContext implements NamespaceContext {

        INSTANCE;

        private final Map<String, String> prefix2NamespaceMap = new ConcurrentHashMap<String, String>();

        private final Map<String, List<String>> namespace2PrefixMap = new ConcurrentHashMap<String, List<String>>();

        private void put(String prefix, String namespaceURI) {
            prefix2NamespaceMap.put(prefix, namespaceURI);
            List<String> prefixes = namespace2PrefixMap.get(namespaceURI);
            if (prefixes == null) {
                prefixes = new ArrayList<String>();
                namespace2PrefixMap.put(namespaceURI, prefixes);
            }
            prefixes.add(prefix);
        }

        private XPath2NamespaceContext() {
            put("fn", NamespaceConstant.FN);
            put("xslt", NamespaceConstant.XSLT);
        }

        @Override
        public String getNamespaceURI(String prefix) {
            return prefix2NamespaceMap.get(prefix);
        }

        @Override
        public String getPrefix(String namespaceURI) {
            List<String> prefixes = namespace2PrefixMap.get(namespaceURI);
            if (prefixes == null || prefixes.size() < 1) {
                return null;
            }
            return prefixes.get(0);
        }

        @Override
        public Iterator getPrefixes(String namespaceURI) {
            List<String> prefixes = namespace2PrefixMap.get(namespaceURI);
            if (prefixes == null || prefixes.size() < 1) {
                return null;
            }
            return prefixes.iterator();
        }
    }

    private void init() throws XPathExpressionException {
        XPathEvaluator xPathEvaluator = new XPathEvaluator();
        xPathEvaluator.setNamespaceContext(XPath2NamespaceContext.INSTANCE);
        xPathExpression = xPathEvaluator.compile(xpathStr);
    }

    @Override
    public String select(String text) {
        try {
            HtmlCleaner htmlCleaner = new HtmlCleaner();
            TagNode tagNode = htmlCleaner.clean(text);
            Document document = new DomSerializer(new CleanerProperties()).createDOM(tagNode);
            Object result;
            try {
                result = xPathExpression.evaluate(document, XPathConstants.NODESET);
            } catch (XPathExpressionException e) {
                result = xPathExpression.evaluate(document, XPathConstants.STRING);
            }
            if (result instanceof NodeList) {
                NodeList nodeList = (NodeList) result;
                if (nodeList.getLength() == 0) {
                    return null;
                }
                Node item = nodeList.item(0);
                if (item.getNodeType() == Node.ATTRIBUTE_NODE || item.getNodeType() == Node.TEXT_NODE) {
                    return item.getTextContent();
                } else {
                    StreamResult xmlOutput = new StreamResult(new StringWriter());
                    Transformer transformer = TransformerFactory.newInstance().newTransformer();
                    transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
                    transformer.transform(new DOMSource(item), xmlOutput);
                    return xmlOutput.getWriter().toString();
                }
            }
            return result.toString();
        } catch (Exception e) {
            logger.error("select text error! " + xpathStr, e);
        }
        return null;
    }

    @Override
    public List<String> selectList(String text) {
        List<String> results = new ArrayList<String>();
        try {
            HtmlCleaner htmlCleaner = new HtmlCleaner();
            TagNode tagNode = htmlCleaner.clean(text);
            Document document = new DomSerializer(new CleanerProperties()).createDOM(tagNode);
            Object result;
            try {
                result = xPathExpression.evaluate(document, XPathConstants.NODESET);
            } catch (XPathExpressionException e) {
                result = xPathExpression.evaluate(document, XPathConstants.STRING);
            }
            if (result instanceof NodeList) {
                NodeList nodeList = (NodeList) result;
                Transformer transformer = TransformerFactory.newInstance().newTransformer();
                StreamResult xmlOutput = new StreamResult();
                transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
                for (int i = 0; i < nodeList.getLength(); i++) {
                    Node item = nodeList.item(i);
                    if (item.getNodeType() == Node.ATTRIBUTE_NODE || item.getNodeType() == Node.TEXT_NODE) {
                        results.add(item.getTextContent());
                    } else {
                        xmlOutput.setWriter(new StringWriter());
                        transformer.transform(new DOMSource(item), xmlOutput);
                        results.add(xmlOutput.getWriter().toString());
                    }
                }
            } else {
                results.add(result.toString());
            }
        } catch (Exception e) {
            logger.error("select text error! " + xpathStr, e);
        }
        return results;
    }
}

分佈式

這塊沒有成熟demo，很可惜

webmagic-avalon

這個項目也沒有完成，但是有別的項目代替，爬蟲管理是重點。

重點

這一系列文章，剖析了整個框架，以及爬蟲如何編寫。

我們可以從實現方式，制定3種方案

1 全模擬全程使用驅動去完成爬取過程，優點：操作簡單缺點：速度慢，即便用了headless

2 半模擬在登入獲取cookie等值，或者中途去執行js，優點：中等難度，速度較快，易於迅速解決問題缺點：速度較慢

3 無模擬僅憑代碼實現各種參數生成，優點：速度很快，缺點：真的很煩，解析Js最煩

歡迎加羣313557283（剛創建），小白互相學習~，如果覺得這系列文章很好，想要打賞，可以加羣發紅包給管理員

前幾篇打好基礎，後面我們有機會用python實戰下

從頭學習爬蟲（二十五）重構篇----WebMagic框架分析之細節

線程池設計

代理池設計

選擇器

分佈式

webmagic-avalon

重點

歡迎加羣313557283（剛創建），小白互相學習~，如果覺得這系列文章很好，想要打賞，可以加羣發紅包給管理員

從頭學習爬蟲（二十五）重構篇----WebMagic框架分析之細節

從頭學習爬蟲（四十六）高階篇----selenium獲取network

從頭學習爬蟲（四十七）進階篇----探索網易雲音樂私信功能

從頭學習爬蟲（二十四）重構篇----WebMagic框架分析之scheduler

從頭學習爬蟲（二十三）重構篇----WebMagic框架分析之pipeline

Mac下配置sublime實現LaTeX

https://yachay.unat.edu.pe/blog/index.php?comment_area=format_blog&comment_component=blog&comment_co

linux以太網驅動總結