cloudsolrclient的add方法調用的是父類solrclient的add方法
public UpdateResponse add(String collection, Collection<SolrInputDocument> docs, int commitWithinMs) throws SolrServerException, IOException {
UpdateRequest req = new UpdateRequest();//創建了一個request
req.add(docs);//把文檔複製進去
req.setCommitWithin(commitWithinMs);
return req.process(this, collection);//進行真正的上傳操作
}
下面來看看 req.process()方法
public final T process(SolrClient client, String collection) throws SolrServerException, IOException {
long startTime = TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS);
T res = createResponse(client);
//這裏面的client.request進行add操作
res.setResponse(client.request(this, collection));
long endTime = TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS);
res.setElapsedTime(endTime - startTime);
return res;
}
下面來看看client.reques
@Override
public NamedList<Object> request(SolrRequest request, String collection) throws SolrServerException, IOException {
SolrParams reqParams = request.getParams();
if (collection == null)
collection = (reqParams != null) ? reqParams.get("collection", getDefaultCollection()) : getDefaultCollection();
return requestWithRetryOnStaleState(request, 0, collection);//可見這個方法進行了具體操作
}
下面來看看requestWithRetryOnStaleState
protected NamedList<Object> requestWithRetryOnStaleState(SolrRequest request, int retryCount, String collection)
throws SolrServerException, IOException {
connect(); // important to call this before you start working with the ZkStateReader
// build up a _stateVer_ param to pass to the server containing all of the
// external collection state versions involved in this request, which allows
// the server to notify us that our cached state for one or more of the external
// collections is stale and needs to be refreshed ... this code has no impact on internal collections
String stateVerParam = null;
List<DocCollection> requestedCollections = null;
if (collection != null && !request.getPath().startsWith("/admin")) { // don't do _stateVer_ checking for admin requests
Set<String> requestedCollectionNames = getCollectionNames(getZkStateReader().getClusterState(), collection);
StringBuilder stateVerParamBuilder = null;
//這個for循環主要是查詢當前的機羣狀態,放入zk節點的version到請求參數裏面
for (String requestedCollection : requestedCollectionNames) {
// track the version of state we're using on the client side using the _stateVer_ param
DocCollection coll = getDocCollection(getZkStateReader().getClusterState(), requestedCollection,null);
int collVer = coll.getZNodeVersion();
if (coll.getStateFormat()>1) {
if(requestedCollections == null) requestedCollections = new ArrayList<>(requestedCollectionNames.size());
requestedCollections.add(coll);
if (stateVerParamBuilder == null) {
stateVerParamBuilder = new StringBuilder();
} else {
stateVerParamBuilder.append("|"); // hopefully pipe is not an allowed char in a collection name
}
stateVerParamBuilder.append(coll.getName()).append(":").append(collVer);
}
}
if (stateVerParamBuilder != null) {
stateVerParam = stateVerParamBuilder.toString();
}
}
if (request.getParams() instanceof ModifiableSolrParams) {
ModifiableSolrParams params = (ModifiableSolrParams) request.getParams();
if (stateVerParam != null) {
params.set(STATE_VERSION, stateVerParam);
} else {
params.remove(STATE_VERSION);
}
} // else: ??? how to set this ???
NamedList<Object> resp = null;
try {
//又調用了這個方法,調用得真深,坑
resp = sendRequest(request, collection);
//to avoid an O(n) operation we always add STATE_VERSION to the last and try to read it from there
Object o = resp.get(STATE_VERSION, resp.size()-1);
if(o != null && o instanceof Map) {
//remove this because no one else needs this and tests would fail if they are comparing responses
resp.remove(resp.size()-1);
Map invalidStates = (Map) o;
for (Object invalidEntries : invalidStates.entrySet()) {
Map.Entry e = (Map.Entry) invalidEntries;
getDocCollection(getZkStateReader().getClusterState(),(String)e.getKey(), (Integer)e.getValue());
}
}
} catch (Exception exc) {
}
return resp;
}
resp = sendRequest(request, collection);
protected NamedList<Object> sendRequest(SolrRequest request, String collection)
throws SolrServerException, IOException {
connect();
ClusterState clusterState = zkStateReader.getClusterState();
boolean sendToLeaders = false;
List<String> replicas = null;
if (request instanceof IsUpdateRequest) {
if (request instanceof UpdateRequest) {
//主要看這就可以了
NamedList<Object> response = directUpdate((AbstractUpdateRequest) request, collection, clusterState);
if (response != null) {
return response;
}
}
.....
}
下面分析directUpdate
private NamedList<Object> directUpdate(AbstractUpdateRequest request, String collection, ClusterState clusterState) throws SolrServerException {
UpdateRequest updateRequest = (UpdateRequest) request;
ModifiableSolrParams params = (ModifiableSolrParams) request.getParams();
ModifiableSolrParams routableParams = new ModifiableSolrParams();
ModifiableSolrParams nonRoutableParams = new ModifiableSolrParams();
if(params != null) {
nonRoutableParams.add(params);
routableParams.add(params);
for(String param : NON_ROUTABLE_PARAMS) {
routableParams.remove(param);
}
}
if (collection == null) {
throw new SolrServerException("No collection param specified on request and no default collection has been set.");
}
//Check to see if the collection is an alias.
Aliases aliases = zkStateReader.getAliases();
if(aliases != null) {
Map<String, String> collectionAliases = aliases.getCollectionAliasMap();
if(collectionAliases != null && collectionAliases.containsKey(collection)) {
collection = collectionAliases.get(collection);
}
}
DocCollection col = getDocCollection(clusterState, collection,null);
DocRouter router = col.getRouter();
if (router instanceof ImplicitDocRouter) {
// short circuit as optimization
return null;
}
//Create the URL map, which is keyed on slice name.
//The value is a list of URLs for each replica in the slice.
//The first value in the list is the leader for the slice.
Map<String,List<String>> urlMap = buildUrlMap(col);
if (urlMap == null) {
// we could not find a leader yet - use unoptimized general path
return null;
}
NamedList<Throwable> exceptions = new NamedList<>();
NamedList<NamedList> shardResponses = new NamedList<>();
//這裏是獲取路由策略
Map<String, LBHttpSolrClient.Req> routes = updateRequest.getRoutes(router, col, urlMap, routableParams, this.idField);
if (routes == null) {
return null;
}
long start = System.nanoTime();
//並行更新索引,值默認爲true
if (parallelUpdates) {
final Map<String, Future<NamedList<?>>> responseFutures = new HashMap<>(routes.size());
for (final Map.Entry<String, LBHttpSolrClient.Req> entry : routes.entrySet()) {
final String url = entry.getKey();
final LBHttpSolrClient.Req lbRequest = entry.getValue();
try {
MDC.put("CloudSolrClient.url", url);
//主要就是這裏操作
responseFutures.put(url, threadPool.submit(new Callable<NamedList<?>>() {
@Override
public NamedList<?> call() throws Exception {
return lbClient.request(lbRequest).getResponse();
}
}));
} finally {
MDC.remove("CloudSolrClient.url");
}
}
for (final Map.Entry<String, Future<NamedList<?>>> entry: responseFutures.entrySet()) {
final String url = entry.getKey();
final Future<NamedList<?>> responseFuture = entry.getValue();
try {
shardResponses.add(url, responseFuture.get());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException(e);
} catch (ExecutionException e) {
exceptions.add(url, e.getCause());
}
}
if (exceptions.size() > 0) {
Throwable firstException = exceptions.getVal(0);
if(firstException instanceof SolrException) {
SolrException e = (SolrException) firstException;
throw new RouteException(ErrorCode.getErrorCode(e.code()), exceptions, routes);
} else {
throw new RouteException(ErrorCode.SERVER_ERROR, exceptions, routes);
}
}
} else {
for (Map.Entry<String, LBHttpSolrClient.Req> entry : routes.entrySet()) {
String url = entry.getKey();
LBHttpSolrClient.Req lbRequest = entry.getValue();
try {
NamedList<Object> rsp = lbClient.request(lbRequest).getResponse();
shardResponses.add(url, rsp);
} catch (Exception e) {
if(e instanceof SolrException) {
throw (SolrException) e;
} else {
throw new SolrServerException(e);
}
}
}
}
//後面是處理delete的操作,可以不看了
UpdateRequest nonRoutableRequest = null;
List<String> deleteQuery = updateRequest.getDeleteQuery();
if (deleteQuery != null && deleteQuery.size() > 0) {
UpdateRequest deleteQueryRequest = new UpdateRequest();
deleteQueryRequest.setDeleteQuery(deleteQuery);
nonRoutableRequest = deleteQueryRequest;
}
Set<String> paramNames = nonRoutableParams.getParameterNames();
Set<String> intersection = new HashSet<>(paramNames);
intersection.retainAll(NON_ROUTABLE_PARAMS);
if (nonRoutableRequest != null || intersection.size() > 0) {
if (nonRoutableRequest == null) {
nonRoutableRequest = new UpdateRequest();
}
nonRoutableRequest.setParams(nonRoutableParams);
List<String> urlList = new ArrayList<>();
urlList.addAll(routes.keySet());
Collections.shuffle(urlList, rand);
LBHttpSolrClient.Req req = new LBHttpSolrClient.Req(nonRoutableRequest, urlList);
try {
LBHttpSolrClient.Rsp rsp = lbClient.request(req);
shardResponses.add(urlList.get(0), rsp.getResponse());
} catch (Exception e) {
throw new SolrException(ErrorCode.SERVER_ERROR, urlList.get(0), e);
}
}
long end = System.nanoTime();
RouteResponse rr = condenseResponse(shardResponses, (long)((end - start)/1000000));
rr.setRouteResponses(shardResponses);
rr.setRoutes(routes);
return rr;
}
public Rsp request(Req req) throws SolrServerException, IOException {
Rsp rsp = new Rsp();
Exception ex = null;
boolean isUpdate = req.request instanceof IsUpdateRequest;
List<ServerWrapper> skipped = null;
long timeAllowedNano = getTimeAllowedInNanos(req.getRequest());
long timeOutTime = System.nanoTime() + timeAllowedNano;
for (String serverStr : req.getServers()) {
if(isTimeExceeded(timeAllowedNano, timeOutTime)) {
break;
}
serverStr = normalize(serverStr);
// if the server is currently a zombie, just skip to the next one
ServerWrapper wrapper = zombieServers.get(serverStr);
if (wrapper != null) {
// System.out.println("ZOMBIE SERVER QUERIED: " + serverStr);
final int numDeadServersToTry = req.getNumDeadServersToTry();
if (numDeadServersToTry > 0) {
if (skipped == null) {
skipped = new ArrayList<>(numDeadServersToTry);
skipped.add(wrapper);
}
else if (skipped.size() < numDeadServersToTry) {
skipped.add(wrapper);
}
}
continue;
}
rsp.server = serverStr;
try {
MDC.put("LBHttpSolrClient.url", serverStr);
HttpSolrClient client = makeSolrClient(serverStr);
//更新的邏輯在這
ex = doRequest(client, req, rsp, isUpdate, false, null);
if (ex == null) {
return rsp; // SUCCESS
}
} finally {
MDC.remove("LBHttpSolrClient.url");
}
}
// try the servers we previously skipped
if (skipped != null) {
for (ServerWrapper wrapper : skipped) {
if(isTimeExceeded(timeAllowedNano, timeOutTime)) {
break;
}
ex = doRequest(wrapper.client, req, rsp, isUpdate, true, wrapper.getKey());
if (ex == null) {
return rsp; // SUCCESS
}
}
}
if (ex == null) {
throw new SolrServerException("No live SolrServers available to handle this request");
} else {
throw new SolrServerException("No live SolrServers available to handle this request:" + zombieServers.keySet(), ex);
}
}
doRequest(client, req, rsp, isUpdate, false, null);調用的是
try {
//繼續看這
rsp.rsp = client.request(req.getRequest(), (String) null);
if (isZombie) {
zombieServers.remove(zombieKey);
}
//上面那段的源碼在這,這個方法在httpsolrclient類中,由上面可見該cloudsolrclient底層實現就是利用多線程有路由的httpsolrclient
public NamedList<Object> request(final SolrRequest request, String collection)
throws SolrServerException, IOException {
ResponseParser responseParser = request.getResponseParser();
if (responseParser == null) {
responseParser = parser;
}
return request(request, responseParser, collection);
}
繼續分析可以發現httpsolrclient會發送http請求到各個solr節點
發送的urlpath爲
public UpdateRequest() {
super(METHOD.POST, "/update");
}
下面繼續分析各個機器接收到 請求怎麼做