hadoop源碼之DataNode

DataNode存放hdfs中的數據，接受客戶端與其他DataNode的請求。

DataNode服務由如下線程構成，按DataNode啓動各線程的先後出場順序列出：

1、DataXceiveServer：數據接收監聽守護線程，當監聽有請求時，開啓一個DataXceiver線程；

        public void run() {
            try {
                while (shouldListen) {
                    Socket s = ss.accept();
                    //s.setSoTimeout(READ_TIMEOUT);
                    new Daemon(new DataXceiver(s)).start();
                }
                ss.close();
            } catch (IOException ie) {
                LOG.info("Exiting DataXceiveServer due to " + ie.toString());
            }
        }

2、DataXceiver：數據接收線程，從DataXceiveServer讀數據，或是寫數據到DataXceiveServer；

3、DataNode：DataNode自身還有一個線程，一直運行offerService()，直到服務器被shutdown爲止；

offerService一直循環做以下幾件事情：

1）每隔HEARTBEAT_INTERVAL ，默認 3 * 1000=3秒鐘，向NameNode發送一次心跳信息；

2）每隔blockReportIntervalBasis - new Random().nextInt((int)(blockReportIntervalBasis/10))，默認blockReportIntervalBasis=60 * 60 * 1000=1小時，向NameNode發送最新的數據塊信息，並且返回被刪除的數據塊列表，以便被GC進行垃圾回收；

3）只要接收到數據庫列表大於0，就向NameNode發送這些數據塊ID；

4）在DataNode啓動後，延後DATANODE_STARTUP_PERIOD，默認是2 * 60 * 1000=2分鐘，從NameNode獲取數據塊相關指令，對每個數據塊，新啓動一個DataTransfer線程來處理；

while (shouldRun) {
    long now = System.currentTimeMillis();
    // Every so often, send heartbeat or block-report
    synchronized (receivedBlockList) {
	  if (now - lastHeartbeat > HEARTBEAT_INTERVAL) {
	    namenode.sendHeartbeat(localName, data.getCapacity(), data.getRemaining());
	    //LOG.info("Just sent heartbeat, with name " + localName);
	    lastHeartbeat = now;
	}
	if (now - lastBlockReport > blockReportInterval) {
	    // Send latest blockinfo report if timer has expired.
	    // Get back a list of local block(s) that are obsolete
	    // and can be safely GC'ed.
	    Block toDelete[] = namenode.blockReport(localName, data.getBlockReport());
	    data.invalidate(toDelete);
	    lastBlockReport = now;
	    continue;
	}
	if (receivedBlockList.size() > 0) {
	    // Send newly-received blockids to namenode
	    Block blockArray[] = (Block[]) receivedBlockList.toArray(new Block[receivedBlockList.size()]);
	    receivedBlockList.removeAllElements();
	    namenode.blockReceived(localName, blockArray);
	}

	if (now - sendStart > datanodeStartupPeriod) {
	    // Check to see if there are any block-instructions from the
	    // namenode that this datanode should perform.
	    BlockCommand cmd = namenode.getBlockwork(localName, xmitsInProgress);
	    if (cmd != null && cmd.transferBlocks()) {
		//
		// Send a copy of a block to another datanode
		//
		Block blocks[] = cmd.getBlocks();
		DatanodeInfo xferTargets[][] = cmd.getTargets();
		
		for (int i = 0; i < blocks.length; i++) {
		    if (!data.isValidBlock(blocks[i])) {
			String errStr = "Can't send invalid block " + blocks[i];
			LOG.info(errStr);
			namenode.errorReport(localName, errStr);
			break;
		    } else {
			if (xferTargets[i].length > 0) {
			    LOG.info("Starting thread to transfer block " + blocks[i] + " to " + xferTargets[i]);
			    new Daemon(new DataTransfer(xferTargets[i], blocks[i])).start();
			}
		    }
		}
	    } else if (cmd != null && cmd.invalidateBlocks()) {
		data.invalidate(cmd.getBlocks());
	    }
	}

	// There is no work to do;  sleep until hearbeat timer elapses, 
	// or work arrives, and then iterate again.
	long waitTime = HEARTBEAT_INTERVAL - (now - lastHeartbeat);
	if (waitTime > 0 && receivedBlockList.size() == 0) {
	    try {
		receivedBlockList.wait(waitTime);
	    } catch (InterruptedException ie) {
	    }
	}
    }
}

4、DataTransfer：數據塊傳輸線程，將指定的數據塊傳到指定的DataNode；

Socket s = new Socket();
s.connect(curTarget, READ_TIMEOUT);
s.setSoTimeout(READ_TIMEOUT);
DataOutputStream out = new DataOutputStream(new BufferedOutputStream(s.getOutputStream()));
try {
    long filelen = data.getLength(b);
    DataInputStream in = new DataInputStream(new BufferedInputStream(data.getBlockData(b)));
    try {
	//
	// Header info
	//
	out.write(OP_WRITE_BLOCK);
	out.writeBoolean(true);
	b.write(out);
	out.writeInt(targets.length);
	for (int i = 0; i < targets.length; i++) {
	    targets[i].write(out);
	}
	out.write(RUNLENGTH_ENCODING);
	out.writeLong(filelen);

	//
	// Write the data
	//
	while (filelen > 0) {
	    int bytesRead = in.read(buf, 0, (int) Math.min(filelen, buf.length));
	    out.write(buf, 0, bytesRead);
	    filelen -= bytesRead;
	}
    } finally {
	in.close();
    }
} finally {
    out.close();
}