多生產者多消費者的Blocking Queue
/**
* A blocking queue which wraps {@code ArrayBlockingQueue}, and with the
* following features.
* <li>Use null object as end of queue.
* <li>Supports muti-producers for the queue. Only if all the producers put a position-pill to
* the queue, the queue will reach end of life.
*
* @param <T>
* element type
*/
public class Queue<T> {
private ArrayBlockingQueue<Element<T>> queue = null;
private AtomicInteger lifeValue ;
private static int DEFAULT_LIFE_VALUE = 1;
/**
*
* @param capacity The queue size
* @param lifeValue It is set as the number of producer of the queue as a rule
*/
public Queue(int capacity, int lifeValue) {
this.queue = new ArrayBlockingQueue<Element<T>>(capacity);
this.lifeValue = new AtomicInteger(lifeValue);
}
public Queue(int capacity) {
this( capacity, DEFAULT_LIFE_VALUE );
}
/**
* Put an position-pill to the queue.
* @throws InterruptedException
*/
@SuppressWarnings("unchecked")
public void putEnd() throws InterruptedException{
int life = this.lifeValue.decrementAndGet();
if(life <= 0 ){
this.queue.put(Element.NULL);
}
}
/**
* Put an element to the queue
* @param element
* @throws InterruptedException
*/
public void put(T element ) throws InterruptedException{
try {
queue.put(new Element<T>(element));
} catch (InterruptedException e) {
this.queue.clear();
throw e;
}
}
/**
* Take an element from the head of queue.
* If queue is empty, the operation will be blocked.
*
* @return a non-nullable object if queue if not empty, or null if the queue is ended
* @throws InterruptedException
*/
public T take() throws InterruptedException{
Element<T> element = this.queue.take();
if( Element.NULL == element){
this.queue.put(Element.NULL);
}
return element.getObject();
}
/**
* Take elements in batch from the queue. The operation will be blocked for a non-ended queue.
* The fetched list size should be equal to count unless the queue is ended
* @param count the desired element size to fetch
* @return a List which contains the batch fetched element, or null if the queue is ended and empty
* @throws InterruptedException
*/
public List<T> batchTake(int count) throws InterruptedException{
List<T> result = new ArrayList<T>(count);
int size = 0;
while(size < count){
T object = this.take();
if(object == null){
break;
}else{
result.add(object);
size ++;
}
}
if(result.isEmpty()){
return null;
}
return result;
}
/**
* Wrapper of object, which supports null
*
* @param <T>
*/
private static class Element<T> {
@SuppressWarnings({"unchecked","rawtypes"})
static Element NULL = new Element(null);
private T object;
Element(T o) {
this.object = o;
}
T getObject() {
return this.object;
}
}
}PipeLine調度框架
/**
*
* An executor that uses multiple threads to run a job which involve muti-steps pipeline like source > intermediate steps > output.
*
* <li> Steps can be executed concurrently, and they communicates with each other by {@code Queue}
* <li> A step can be executed with muti-thread concurrently, and the thread number can be configured
*
* @author yuliang
*
* @param <S> source type
* @param <O> output type
*/
@SuppressWarnings("rawtypes")
public class ParallelPipeLineExecutor<S, O> {
protected static final Logger LOGGER = Logger.getLogger(FourStepPiplineExecutor.class);
private String pipeLineName;
private ISource<S> source;
private Queue<S> sourceQueue;
private List<Queue> queues = new LinkedList<Queue>();
private List<PipeLineStep> processorSteps = new LinkedList<PipeLineStep>();
public ParallelPipeLineExecutor() {
}
public void setName(String pipeLineName) {
this.pipeLineName = pipeLineName;
}
public void setSource(ISource<S> source, int sourceQueueSize) {
this.source = source;
this.sourceQueue = new Queue<S>(sourceQueueSize);
}
private String generateStepProcessorName(String stepName, int stepSequence){
return this.pipeLineName+"."+stepName+"#"+stepSequence;
}
private <T1, T2> void addNextStep(String name, ProcessorProvider<T1, T2> provider, int threadNum, int outQueueSize) {
String stepName = (name == null) ? "Step" + (processorSteps.size()+1) : name;
PipeLineStep step = new PipeLineStep(stepName, provider, threadNum);
this.processorSteps.add(step);
if (outQueueSize > 0) {
Queue<T1> outQueue = new Queue<T1>(outQueueSize, threadNum);
queues.add(outQueue);
}
}
/**
* Add next step for the pipeline
*
* @param provider
* a {@code Processor} provider
* @param threadNum
* the thread number that run the step
* @param outQueueSize
* the size of queue where the step output to
*/
public <T1, T2> void addNextStep(ProcessorProvider<T1, T2> provider, int threadNum, int outQueueSize) {
Preconditions.checkArgument(threadNum > 0, "threadNum: " + threadNum + "should be positive");
Preconditions.checkArgument(outQueueSize >= threadNum, "outQueueSize: " + outQueueSize
+ "should be greater than threadNum:" + threadNum);
if (provider instanceof IdentityProcessor.Provider) {
LOGGER.info("Skip a step because it use Indentity.Provider");
} else {
this.addNextStep(null, provider, threadNum, outQueueSize);
}
}
/**
* Add sink step for the pipeline
*
* @param provider
* provider a {@code Processor} provider
* @param threadNum
* the thread number that run the step
*/
public void addSinks(ProcessorProvider<O, Void> provider, int threadNum) {
Preconditions.checkArgument(threadNum > 0, "threadNum: " + threadNum + "should be positive");
this.addNextStep("Output", provider, threadNum, 0);
}
/**
* Launch the execution of the pipeline and wait until it is finished or exception occurs
*/
public void launchAndWait() {
int workersCount = 0;
final ExecutorService executorService = Executors.newCachedThreadPool();
ExecutorCompletionService<Worker> workerCompletionService = new ExecutorCompletionService<Worker>(
executorService);
try {
//submit source worker
Worker sourceWorker = new SourceWorker<S>(this.source, this.sourceQueue);
sourceWorker.setName(this.generateStepProcessorName("sourceWorker", workersCount));
workerCompletionService.submit(sourceWorker);
workersCount++;
//submit processor workers including sink workers
Queue inQueue = sourceQueue;
Iterator<Queue> queuesIter = queues.iterator();
for (PipeLineStep step : processorSteps) {
final Queue outQueue = queuesIter.hasNext() ? queuesIter.next() : null;
int stepSequence = 0;
for (IProcessor processor : step.generateProcessors()) {
stepSequence ++;
ProcessWorker worker = new ProcessWorker(inQueue, processor, outQueue);
worker.setName(this.generateStepProcessorName(step.getName(), stepSequence));
workerCompletionService.submit(worker);
workersCount++;
}
inQueue = outQueue;
}
//wait for completion or exception
while (workersCount > 0) {
Worker w = workerCompletionService.take().get();
if (w.getException() != null) {
LOGGER.error(w + " finished with exception");
throw w.getException();
} else {
LOGGER.info(workersCount + " finished ");
}
workersCount--;
LOGGER.info(workersCount + " threads still running...");
}
} catch (Exception e) {
//Just throw exceptions so that job failure cause be be directly shown in DJS/HAM
throw new RuntimeException(e);
} finally {
executorService.shutdown();
}
}
/**
* A process step in the pipe line
*
*/
private static class PipeLineStep {
private String name;
private ProcessorProvider processorProvider;
private int concurrentThreadNum;
public PipeLineStep(String name, ProcessorProvider provider, int threadNum) {
this.name = name;
this.processorProvider = provider;
this.concurrentThreadNum = threadNum;
}
public String getName() {
return name;
}
public List<IProcessor> generateProcessors() throws Exception {
List<IProcessor> processors = new ArrayList<IProcessor>();
for (int i = 0; i < concurrentThreadNum; i++) {
processors.add(processorProvider.get());
}
return processors;
}
}
}Worker類
/**
* A {@code Worker} which can pull inputs from {@code inQueue}, process inputs
* and output the results to {@code outQueue} continuously
* @author yuliang
*
* @param <I> process input type
* @param <O> process output type
*/
public class ProcessWorker<I, O> extends Worker {
private Queue<I> inQueue;
private IProcessor<I, O> processor;
private Queue<O> outQueue;
private int batchSize = 1;
public ProcessWorker(Queue<I> inQueue, IProcessor<I, O> processor, Queue<O> outQueue) {
super();
Preconditions.checkArgument(batchSize > 0, "batchSize should be greater than 0");
this.inQueue = inQueue;
this.processor = processor;
this.outQueue = outQueue;
this.batchSize = processor.getBatchSize();
}
@Override
public void work() throws Exception {
try {
if (this.batchSize > 1 && this.processor.isBatchSupported()) {
this.processByBatch();
} else {
this.processOneByOne();
}
} finally {
if (outQueue != null) {
outQueue.putEnd();
}
}
}
/**
* Fetch inputs from {@code inQueue}, and process inputs one by one and put
* results to {@code outQueue}
*/
private void processOneByOne() throws Exception {
while (true) {
I in = inQueue.take();
if (in == null) {
break;
}
O o = processor.process(in);
if (o != null && outQueue != null) {
outQueue.put(o);
}
}
}
/**
* Fetch inputs from {@code inQueue}, and process inputs in batch and put
* results to {@code outQueue}
*/
private void processByBatch() throws Exception {
while (true) {
List<I> in = inQueue.batchTake(this.batchSize);
if (in == null) {
break;
}
List<O> outs = processor.batchProcess(in);
if (outs != null && outQueue != null) {
for (O o : outs) {
if (o != null) {
outQueue.put(o);
}
}
}
}
}
}
/**
*
* A source worker is a {@code Worker} which produce source elements
* @author yuliang
*
* @param <S> source element type
*/
public class SourceWorker <S> extends Worker{
private Queue<S> outQueue;
private ISource<S> source;
public SourceWorker(ISource<S> source, Queue<S> sourceQueue) {
super();
this.source = source;
this.outQueue = sourceQueue;
}
@Override
public void work() throws Exception {
try{
while(true){
S o = source.nextSource();
if(o == null){
break;
}
this.outQueue.put(o);
}
}finally{
this.outQueue.putEnd();
}
}
}