larbin中sequencer()及其相關函數閱讀

<!-- @page { margin: 2cm } P { margin-bottom: 0.21cm } -->

函數調用關係：

sequencer()

---canGetUrl()

------get()

------tryget()

------ putPriorityUrlWait (url *u) { putGenericUrl(u, 1, true); }

------ putPriorityUrl (url *u) { putGenericUrl(u, 5, true); }

------ putUrlWait (url *u) { putGenericUrl(u, 10, false); }

------ putUrl (url *u) { putGenericUrl(u, 15, false); }

/*start the sequencer*/ void sequencer () { bool testPriority = true; if (space == 0) { space = global::inter->putAll();//能夠裝載的urls數量 } int still = space; if (still > maxPerCall) still = maxPerCall;//限定每次處理的最大值 while (still) {//還沒處理完 if (canGetUrl(&testPriority)) { space--; still--;//要處理的urls減一，能裝載的urls減一 } else { still = 0; }//end if (canGetUrl(&testPriority)) }//end while (still) } /*putPriorityUrlWait()，putPriorityUrl()，putUrlWait()，putUrl() *對於每個站點，將相應的URL放進去,最終調用的都是putGenericUrl*/ /*uint hostHashCode ();返回url的主機的哈西值*/ /*get();獲得第一個對象*/ /*tryget();獲得第一個對象,如果沒有則返回NULL*/ /* Get the next url * here is defined how priorities are handled*/ static bool canGetUrl (bool *testPriority) { url *u; if (global::readPriorityWait) {//readPriorityWait是URLsPriorityWait中url的數量 global::readPriorityWait--; u = global::URLsPriorityWait->get(); global::namedSiteList[u->hostHashCode()].putPriorityUrlWait(u); return true; } else if (*testPriority && (u=global::URLsPriority->tryGet()) != NULL) { // We've got one url (priority) global::namedSiteList[u->hostHashCode()].putPriorityUrl(u); return true; } else { *testPriority = false; // Try to get an ordinary url if (global::readWait) {//readWait是URLsDiskWait中的url數量 global::readWait--; u = global::URLsDiskWait->get(); global::namedSiteList[u->hostHashCode()].putUrlWait(u); return true; } else { u = global::URLsDisk->tryGet();//獲得第一個對象，如果沒有返回空 if (u != NULL) { global::namedSiteList[u->hostHashCode()].putUrl(u); return true; } else { return false; }//end if (u != NULL) }//end if (global::readWait) }//end if (global::readPriorityWait) } /* 如果已經有足夠多的url在內存裏，執行if中的代碼， * strcmp(name,u->getHost()) 是判斷這個主機是不是已經就進行過dns方面的判斷，也就是說對於一個站點，只做一次dns解析的判斷，以後就按這個結果進行處理， * dnsState有noDns，noConnDns，還有robots.txt不允許的情況，如果沒有問題，就把它放到URLsDisk中。*/ /* Put an url in the fifo if their are not too many */ void NamedSite::putGenericUrl(url *u, int limit, bool prio) { if (nburls > maxUrlsBySite-limit) {//nburls爲內存中的url數量 ，Already enough Urls in memory for this Site if (!strcmp(name, u->getHost())) {// first check if it can already be forgotten if (dnsState == errorDns) {//dns錯誤 nburls++; forgetUrl(u, noDNS); return; } if (dnsState == noConnDns) {//連接錯誤 nburls++; forgetUrl(u, noConnection); return; } if (u->getPort() == port && dnsState == doneDns && !testRobots(u->getFile())) {//不允許訪問的內容 nburls++; forgetUrl(u, forbiddenRobots); return; } }//end if(!strcmp(name, u->getHost())) /*其他情況放到URLsDisk隊列中*/ refUrl();// missUrl++ ; Number of urls reput in ram global::inter->getOne(); if (prio) {//優先性 global::URLsPriorityWait->put(u); } else { global::URLsDiskWait->put(u); } } /*如果需要判斷dns能不能解析，就將它放到dnsSites裏，這個會在fetchDns中判斷。 * 或是如果還能放到內存裏，並且又是doneDns，表示可以解析，就調用transfer()；將url放入到IPSiteList的相應ipHash中*/ else { nburls++; if (dnsState == waitDns || strcmp(name, u->getHost()) || port != u->getPort() || global::now > dnsTimeout) { // dns not done or other site putInFifo(u); addNamedUrl(); // Put Site in fifo if not yet in if (!isInFifo) { isInFifo = true; global::dnsSites->put(this); } } else switch (dnsState) { case doneDns: transfer(u); /*將url放入到IPSiteList的相應ipHash中*/ break; case errorDns: forgetUrl(u, noDNS); break; default: // noConnDns forgetUrl(u, noConnection); } }//end if (nburls > maxUrlsBySite-limit) }