java selenium獲取Network response的內容

這兩天遇到一個棘手的異常，時不時頁面會彈出：“系統繁忙，請稍候再試！”，這時候我們去看網絡請求數據，結果狀態碼全部都是 200，沒有其它信息，這壓根沒法定位不了問題。

這就說明：網絡出現異常的時候，僅靠狀態碼是不夠的。我們最好能拿到 http 所有數據，包括：請求頭、響應頭、請求體、響應體。其中請求頭、響應頭，可以通過 PERFORMANCE_LOG 拿到，問題都不大。但是請求體與響應體，我們可以拿到麼？

這個問題困擾了我整整一天的時間，終於解決了。爲什麼這麼困難？

我們先來看 selenium，它爲什麼不直接支持這個功能呢？因爲開發人員覺得這不是他們目標：

we will not be adding this feature to the WebDriver API as it falls outside of our current scope (emulating user actions).

然後我繼續翻網絡，發現谷歌的devtools-protocol明確是支持的

那到底有沒有辦法解決這個問題？

有！且不止一種方法....

第一種方法：我們使用BrowserMob Proxy

首先獲取WebDriver及BrowserProxy

import net.lightbody.bmp.BrowserMobProxy;
import net.lightbody.bmp.BrowserMobProxyServer;
import net.lightbody.bmp.client.ClientUtil;
import net.lightbody.bmp.proxy.CaptureType;
import org.openqa.selenium.Proxy;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.logging.LogType;
import org.openqa.selenium.logging.LoggingPreferences;
import org.openqa.selenium.remote.CapabilityType;

import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;

/**
 * BrowserProxy 單例
 *
 * @author ant
 * @date 2019-11-21 10:38:20
 */
public class BrowserProxy {
    /**
     * BrowserMobProxy
     */
    private BrowserMobProxy proxy;

    /**
     * WebDriver
     */
    private WebDriver driver;


    public BrowserMobProxy getProxy() {
        return proxy;
    }

    private void setProxy(BrowserMobProxy proxy) {
        this.proxy = proxy;
    }

    public WebDriver getDriver() {
        return driver;
    }

    private void setDriver(WebDriver driver) {
        this.driver = driver;
    }

    /**
     * 無參構造 初始化參數
     * 私有構造 禁止外部new
     */
    private BrowserProxy() {
        String webDriverPath = System.getProperty("user.dir") + "/resources/chromedriver.exe";
        System.out.println("webDriverPath:"+webDriverPath);
        //String webDriverPath = "E:\\IDEA\\common\\src\\test\\java\\com\\autotest\\tools\\chromedriver.exe";
        System.setProperty("webdriver.chrome.driver", webDriverPath);

        ChromeOptions options = new ChromeOptions();
        // 禁用阻止彈出窗口
        options.addArguments("--disable-popup-blocking");
        // 啓動無沙盒模式運行
        options.addArguments("no-sandbox");
        // 禁用擴展
        options.addArguments("disable-extensions");
        // 默認瀏覽器檢查
        options.addArguments("no-default-browser-check");

        Map<String, Object> prefs = new HashMap();
        prefs.put("credentials_enable_service", false);
        prefs.put("profile.password_manager_enabled", false);
        // 禁用保存密碼提示框
        options.setExperimentalOption("prefs", prefs);

        // set performance logger
        // this sends Network.enable to chromedriver
        LoggingPreferences logPrefs = new LoggingPreferences();
        logPrefs.enable(LogType.PERFORMANCE, Level.ALL);
        options.setCapability(CapabilityType.LOGGING_PREFS, logPrefs);

        // start the proxy
        BrowserMobProxy proxy = new BrowserMobProxyServer();
        // 端口號
        proxy.start(4566);

        // get the Selenium proxy object
        Proxy seleniumProxy = ClientUtil.createSeleniumProxy(proxy);
        options.setCapability(CapabilityType.PROXY, seleniumProxy);
        // start the browser up
        WebDriver driver = new ChromeDriver(options);
        // 等待10秒
        driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS);

        // enable more detailed HAR capture, if desired (see CaptureType for the complete list)
        proxy.enableHarCaptureTypes(CaptureType.REQUEST_CONTENT, CaptureType.RESPONSE_CONTENT);
        // enable cookies  --> REQUEST_COOKIES, RESPONSE_COOKIES
        proxy.enableHarCaptureTypes(CaptureType.getCookieCaptureTypes());
        // enable headers --> REQUEST_HEADERS, RESPONSE_HEADERS
        //proxy.enableHarCaptureTypes(CaptureType.getHeaderCaptureTypes());
        // 向對象賦值
        this.setProxy(proxy);
        this.setDriver(driver);
    }

    /**
     * 類級的內部類，也就是靜態的成員式內部類，該內部類的實例與外部類的實例沒有綁定關係，而且只有被調用到纔會裝載，從而實現了延遲加載
     */
    private static class BrowserProxyHolder {
        private final static BrowserProxy instance = new BrowserProxy();
    }

    /**
     * 提供公有調用方法
     *
     * @return
     */
    public static BrowserProxy getInstance() {
        return BrowserProxyHolder.instance;
    }

}

在這裏需要注意的是

proxy.enableHarCaptureTypes(CaptureType.REQUEST_CONTENT, CaptureType.RESPONSE_CONTENT);

如果要獲取request content或者response content則必需加上這行參數,否則下面獲取不到數據。

那麼，如果是獲取cookie呢？

proxy.enableHarCaptureTypes(CaptureType.getCookieCaptureTypes());

換了一種寫法，其實和上面獲取content效果是一樣的，當然還有其他寫法，很靈活。

準備工作做好了，下面幹正事。

import com.example.cat.selenium.BrowserProxy;
import net.lightbody.bmp.BrowserMobProxy;
import net.lightbody.bmp.core.har.*;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import java.util.List;


public class BrowserMobProxyTest {

    public void tearDown(WebDriver driver,BrowserMobProxy proxy) {
        if(proxy.isStarted()){
            proxy.stop();
        }
        // 關閉當前窗口
        driver.close();
    }

    public static void main(String[] args) {
        BrowserMobProxyTest browserMobProxyTest =  new BrowserMobProxyTest();
        BrowserProxy browserProxy = BrowserProxy.getInstance();
        WebDriver driver = browserProxy.getDriver();
        BrowserMobProxy proxy = browserProxy.getProxy();
        browserMobProxyTest.testMethod(driver,proxy);
        //browserMobProxyTest.tearDown(driver,proxy);
    }
    public void testMethod(WebDriver driver,BrowserMobProxy proxy) {
        // 這裏必需在driver.get(url)之前
        proxy.newHar("test_har");

        driver.get("https://translate.google.cn/");

        String text = "selenium";

        driver.findElement(By.xpath("//*[@id=\"source\"]")).sendKeys(text);

        Har har = proxy.getHar();
        List<HarEntry> list =  har.getLog().getEntries();
        for (HarEntry harEntry : list){
            HarResponse harResponse = harEntry.getResponse();
            String responseBody = harResponse.getContent().getText();
            List<HarCookie> cookies = harResponse.getCookies();
            // network response
            System.out.println("responseBody:"+responseBody);
            // 可獲取httpOnly 類型的cookie
            System.out.println("cookies:"+cookies);
        }
    }
}

如此，想獲取network response的內容或是獲取httpOnly類型的cookie也都ok了。

但是有一個問題，仍未解決

AnnotatedConnectException: Connection timed out: no further information: accounts.google.com/172.217.160.109:443

爲什麼會連接connection 這個ip呢？有發現的小夥伴留言交流！

接下來第二種方法我們通過requtesId去實現

package com.example.cat.test.hacpai;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.example.cat.utils.Hclient;
import org.apache.http.HttpResponse;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeDriverService;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.logging.LogEntries;
import org.openqa.selenium.logging.LogEntry;
import org.openqa.selenium.logging.LogType;
import org.openqa.selenium.logging.Logs;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;


public class ChromeDriverProxy extends ChromeDriver {
    /**
     * method --> Network.responseReceived
     */
    public static final String NETWORK_RESPONSE_RECEIVED = "Network.responseReceived";

    /**
     * post 請求超時時間
     */
    private static final int COMMAND_TIMEOUT = 5000;

    /**
     * 必須固定端口，因爲ChromeDriver沒有實時獲取端口的接口；
     */
    private static final int CHROME_DRIVER_PORT = 9102;

    private static ChromeDriverService driverService = new ChromeDriverService.Builder().usingPort(CHROME_DRIVER_PORT).build();

    public ChromeDriverProxy(ChromeOptions options) {
        super(driverService, options);
    }


    // 根據請求ID獲取返回內容
    public String getResponseBody(String requestId) {
        try {
            // CHROME_DRIVER_PORT chromeDriver提供的端口
            String url = String.format("http://localhost:%s/session/%s/goog/cdp/execute",
                    CHROME_DRIVER_PORT, getSessionId());
            HttpPost httpPost = new HttpPost(url);
            JSONObject object = new JSONObject();
            JSONObject params = new JSONObject();
            params.put("requestId", requestId);
            object.put("cmd", "Network.getResponseBody");
            object.put("params", params);
            httpPost.setEntity(new StringEntity(object.toString()));

            RequestConfig requestConfig = RequestConfig
                    .custom()
                    .setSocketTimeout(COMMAND_TIMEOUT)
                    .setConnectTimeout(COMMAND_TIMEOUT).build();

            CloseableHttpClient httpClient = HttpClientBuilder.create()
                    .setDefaultRequestConfig(requestConfig).build();

            HttpResponse response = httpClient.execute(httpPost);
            String s = EntityUtils.toString(response.getEntity());
            return  this.getResponseValue(s);
        } catch (IOException e) {
            //logger.error("getResponseBody failed!", e);
        }
        return null;
    }



    // 根據請求ID獲取返回內容
    // https://github.com/bayandin/chromedriver/blob/master/server/http_handler.cc
    public JSONArray getCookies(String requestId) {
        try {
            // CHROME_DRIVER_PORT chromeDriver提供的端口
            String url = String.format("http://localhost:%s/session/%s/cookie",
                    CHROME_DRIVER_PORT, getSessionId());
            String cookieStr = Hclient.doGet(url, null);
           return JSONArray.parseArray(getResponseValue(cookieStr));
        } catch (Exception e) {
            //logger.error("getResponseBody failed!", e);
        }
        return null;
    }
    /**
     * 獲取響應結果{"sessionId": "","status": 0,"value": ""}的value
     *
     * @param data
     * @return
     */
    private String getResponseValue(String data){
        JSONObject json = JSONObject.parseObject(data);
        ResponseBodyVo responseBodyVo = JSONObject.toJavaObject(json,ResponseBodyVo.class);
        if(0 == responseBodyVo.getStatus()){
            return responseBodyVo.getValue();
        }else{
            System.out.println("status error:" + data);
            return "";
        }

    }

    public static void saveHttpTransferDataIfNecessary(ChromeDriverProxy driver) {
        Logs logs = driver.manage().logs();
        Set<String> availableLogTypes = logs.getAvailableLogTypes();

        if(availableLogTypes.contains(LogType.PERFORMANCE)) {
            LogEntries logEntries = logs.get(LogType.PERFORMANCE);
            List<ResponseReceivedEvent> responseReceivedEvents = new ArrayList<>();

            for(LogEntry entry : logEntries) {
                JSONObject jsonObj = JSON.parseObject(entry.getMessage()).getJSONObject("message");
                String method = jsonObj.getString("method");
                String params = jsonObj.getString("params");

                if (method.equals(NETWORK_RESPONSE_RECEIVED)) {
                    ResponseReceivedEvent response = JSON.parseObject(params, ResponseReceivedEvent.class);
                    responseReceivedEvents.add(response);
                }
            }
            doSaveHttpTransferDataIfNecessary(driver, responseReceivedEvents);
        }

    }

    // 保存網絡請求
    private static void doSaveHttpTransferDataIfNecessary(ChromeDriverProxy driver, List<ResponseReceivedEvent> responses) {
        for(ResponseReceivedEvent responseReceivedEvent : responses) {
            String url = JSONObject.parseObject(responseReceivedEvent.getResponse()).getString("url");
            boolean staticFiles = url.endsWith(".png")
                    || url.endsWith(".jpg")
                    || url.endsWith(".css")
                    || url.endsWith(".ico")
                    || url.endsWith(".js")
                    || url.endsWith(".gif");

            if(!staticFiles && url.startsWith("http")) {
                // 使用上面開發的接口獲取返回數據
                String  bodyx= driver.getResponseBody(responseReceivedEvent.getRequestId());

                System.out.println("url:"+url+" ,body-->"+bodyx);
                JSONArray cookies = driver.getCookies(responseReceivedEvent.getRequestId());
                System.out.println("url:"+url+" ,cookies-->"+cookies);
            }
        }
    }



}

附上幾個實體類

ResponseBodyVo

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

@Data
@AllArgsConstructor
@NoArgsConstructor
public class ResponseBodyVo {
    private String sessionId;
    private String value;
    private int status;
}

ResponseReceivedEvent

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;



@Data
@AllArgsConstructor
@NoArgsConstructor
public class ResponseReceivedEvent {
    private String frameId;
    private String requestId;
    private String response;
    private String loaderId;
    private String type;
    private String timestamp;
}

測試類

import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.logging.LogType;
import org.openqa.selenium.logging.LoggingPreferences;
import org.openqa.selenium.remote.CapabilityType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;


public class Test {
    private final static Logger logger = LoggerFactory.getLogger(Test.class);

    public static void main(String[] args) {
        String url = "https://xxxx";
        test(url);
    }

    private static void test(String url) {
        String webDriverPath = "E:\\IDEA\\common\\src\\test\\java\\com\\autotest\\tools\\chromedriver.exe";
        System.setProperty("webdriver.chrome.driver", webDriverPath);
        ChromeDriverProxy driver = null;
        try {
            ChromeOptions options = new ChromeOptions();
            options.addArguments("--disable-popup-blocking"); // 禁用阻止彈出窗口
            options.addArguments("no-sandbox"); // 啓動無沙盒模式運行
            options.addArguments("disable-extensions"); // 禁用擴展
            options.addArguments("no-default-browser-check"); // 默認瀏覽器檢查
            Map<String, Object> prefs = new HashMap();
            prefs.put("credentials_enable_service", false);
            prefs.put("profile.password_manager_enabled", false);
            options.setExperimentalOption("prefs", prefs);// 禁用保存密碼提示框

            // set performance logger
            // this sends Network.enable to chromedriver
            LoggingPreferences logPrefs = new LoggingPreferences();
            logPrefs.enable(LogType.PERFORMANCE, Level.ALL);

            options.setCapability(CapabilityType.LOGGING_PREFS, logPrefs);

            driver = new ChromeDriverProxy(options);
            // do something

            ChromeDriverProxy.saveHttpTransferDataIfNecessary(driver);
        } finally {
            driver.close();
        }
    }
}


同樣 需要注意的是logPrefs.enable(LogType.PERFORMANCE, Level.ALL);

以上兩種方法基本獲取network response的內容了。

大家有好的方法可以一起留言交流。

參考：

java+selenium+chromedriver 有什麼辦法能拿到 network 返回的數據嗎，求大佬指教

selenium 獲取請求返回內容的解決方案

煙光凝而

發佈了18 篇原創文章 · 獲贊 10 · 訪問量 1萬+

私信關注

java selenium獲取Network response的內容

eclipse安裝SVN插件（2020最新，親測可用）

java selenium獲取Network response的內容

使用Navicat導出數據字典

'shaHex(java.lang.String)' is deprecated

org.java_websocket.exceptions.WebsocketNotConnectedException: null

https://yachay.unat.edu.pe/blog/index.php?comment_area=format_blog&comment_component=blog&comment_co

linux以太網驅動總結