Flink源碼-4-task執行

執行

flink 作業的最小執行單元是task

示例

public class WorldCount {

    public static void main(String[] args) throws Exception {

        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        env.setParallelism(1);

        DataStream<Tuple2<String, Integer>> dataStream = env
                .socketTextStream("localhost", Integer.parseInt(args[0]))
                .flatMap(new Splitter())
                .keyBy(0)
                .timeWindow(Time.seconds(5))
                .sum(1);

        dataStream.print();


        env.getExecutionPlan();

        env.execute("Window WordCount");
    }

    public static class Splitter implements FlatMapFunction<String, Tuple2<String, Integer>> {
        @Override
        public void flatMap(String sentence, Collector<Tuple2<String, Integer>> out) throws Exception {
            for (String word : sentence.split(" ")) {
                out.collect(new Tuple2<String, Integer>(word, 1));
            }
        }
    }
}

入口類

org.apache.flink.runtime.taskexecutor.TaskExecutor
核心方法

public CompletableFuture<Acknowledge> submitTask(){

			// now load and instantiate the task's invokable code
			invokable = loadAndInstantiateInvokable(userCodeClassLoader, nameOfInvokableClass, env);
			// run the invokable
			invokable.invoke();
			
	}

source部分

這裏的invoable就是
org.apache.flink.streaming.runtime.tasks.SourceStreamTask 這個類，調用invoke方法

任務基類
org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable 所有的task都實現了這個類
所有的任務執行都是invoke

重點關注SourceStreamTask的初始化
主要通過反射拿到構造器，再調用instance方法初始化這個對象
Environment這個數據代表了這個任務的執行信息

private static AbstractInvokable loadAndInstantiateInvokable(
		ClassLoader classLoader,
		String className,
		Environment environment) throws Throwable {

		final Class<? extends AbstractInvokable> invokableClass;
		try {
			invokableClass = Class.forName(className, true, classLoader)
				.asSubclass(AbstractInvokable.class);
		} catch (Throwable t) {
			throw new Exception("Could not load the task's invokable class.", t);
		}

		Constructor<? extends AbstractInvokable> statelessCtor;

		try {
			statelessCtor = invokableClass.getConstructor(Environment.class);
		} catch (NoSuchMethodException ee) {
			throw new FlinkException("Task misses proper constructor", ee);
		}

		// instantiate the class
		try {
			//noinspection ConstantConditions  --> cannot happen
			//----------------核心方法------------------
			return statelessCtor.newInstance(environment);
		} catch (InvocationTargetException e) {
			// directly forward exceptions from the eager initialization
			throw e.getTargetException();
		} catch (Exception e) {
			throw new FlinkException("Could not instantiate the task's invokable class.", e);
		}
	}

回到invoke方法

org.apache.flink.streaming.runtime.tasks.StreamTask 這個類給出了
invoke的實現

@Override
	public final void invoke() throws Exception {

		boolean disposed = false;
		try {
			

			operatorChain = new OperatorChain<>(this, recordWriters);
			headOperator = operatorChain.getHeadOperator();

			// task specific initialization
			init();

			// save the work of reloading state, etc, if the task is already canceled
			if (canceled) {
				throw new CancelTaskException();
			}

			// -------- Invoke --------
			LOG.debug("Invoking {}", getName());

			// we need to make sure that any triggers scheduled in open() cannot be
			// executed before all operators are opened
			synchronized (lock) {

				// both the following operations are protected by the lock
				// so that we avoid race conditions in the case that initializeState()
				// registers a timer, that fires before the open() is called.

				initializeState();
				openAllOperators();
			}

			// final check to exit early before starting to run
			if (canceled) {
				throw new CancelTaskException();
			}

			//開始執行開個算子
			isRunning = true;
			//------------------------這是重要方法--------------------------
			run();

			// if this left the run() method cleanly despite the fact that this was canceled,
			// make sure the "clean shutdown" is not attempted
			if (canceled) {
				throw new CancelTaskException();
			}

			LOG.debug("Finished task {}", getName());

			// make sure no further checkpoint and notification actions happen.
			// we make sure that no other thread is currently in the locked scope before
			// we close the operators by trying to acquire the checkpoint scope lock
			// we also need to make sure that no triggers fire concurrently with the close logic
			// at the same time, this makes sure that during any "regular" exit where still
			synchronized (lock) {
				// this is part of the main logic, so if this fails, the task is considered failed
				closeAllOperators();

				// make sure no new timers can come
				timerService.quiesce();

				// only set the StreamTask to not running after all operators have been closed!
				// See FLINK-7430
				isRunning = false;
			}

			// make sure all timers finish
			timerService.awaitPendingAfterQuiesce();

			LOG.debug("Closed operators for task {}", getName());

			// make sure all buffered data is flushed
			operatorChain.flushOutputs();

			// make an attempt to dispose the operators such that failures in the dispose call
			// still let the computation fail
			tryDisposeAllOperators();
			disposed = true;
		}
	}

分析run方法

private void run() throws Exception {
			performDefaultAction(actionContext);
		}
	}

performDefaultAction 方法在org.apache.flink.streaming.runtime.tasks.SourceStreamTask 重寫了

protected void performDefaultAction(ActionContext context) throws Exception {
		
		sourceThread.start();


	}

啓動了sourceThread線程，再往下走

private class LegacySourceFunctionThread extends Thread {

		private Throwable sourceExecutionThrowable;

		LegacySourceFunctionThread() {
			this.sourceExecutionThrowable = null;
		}

        //headOperator 代表第一步算子
		@Override
		public void run() {
			try {
				headOperator.run(getCheckpointLock(), getStreamStatusMaintainer(), operatorChain);
			} catch (Throwable t) {
				sourceExecutionThrowable = t;
			} finally {
				mailbox.clearAndPut(SOURCE_POISON_LETTER);
			}
		}

		void checkThrowSourceExecutionException() throws Exception {
			if (sourceExecutionThrowable != null) {
				throw new Exception(sourceExecutionThrowable);
			}
		}
	}

最後走到了
org.apache.flink.streaming.api.functions.source.SocketTextStreamFunction
這就是業務代碼了

public void run(SourceContext<String> ctx) throws Exception {
		final StringBuilder buffer = new StringBuilder();
		long attempt = 0;

		while (isRunning) {

			try (Socket socket = new Socket()) {
				currentSocket = socket;

				LOG.info("Connecting to server socket " + hostname + ':' + port);
				socket.connect(new InetSocketAddress(hostname, port), CONNECTION_TIMEOUT_TIME);
				try (BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream()))) {

					char[] cbuf = new char[8192];
					int bytesRead;
					while (isRunning && (bytesRead = reader.read(cbuf)) != -1) {
						buffer.append(cbuf, 0, bytesRead);
						int delimPos;
						while (buffer.length() >= delimiter.length() && (delimPos = buffer.indexOf(delimiter)) != -1) {
							String record = buffer.substring(0, delimPos);
							// truncate trailing carriage return
							if (delimiter.equals("\n") && record.endsWith("\r")) {
								record = record.substring(0, record.length() - 1);
							}
							ctx.collect(record);
							buffer.delete(0, delimPos + delimiter.length());
						}
					}
				}
			}

			// if we dropped out of this loop due to an EOF, sleep and retry
			if (isRunning) {
				attempt++;
				if (maxNumRetries == -1 || attempt < maxNumRetries) {
					LOG.warn("Lost connection to server socket. Retrying in " + delayBetweenRetries + " msecs...");
					Thread.sleep(delayBetweenRetries);
				}
				else {
					// this should probably be here, but some examples expect simple exists of the stream source
					// throw new EOFException("Reached end of stream and reconnects are not enabled.");
					break;
				}
			}
		}

		// collect trailing data
		if (buffer.length() > 0) {
			ctx.collect(buffer.toString());
		}
	}

窗口函數

上面主要分析了窗口函數這部分
這部分的task類
org.apache.flink.streaming.runtime.tasks.OneInputStreamTask
直接從run分析
performDefaultAction 這個方法和sourcetask走的不同

/**
	 * Runs the stream-tasks main processing loop.
	 */
	private void run() throws Exception {
		final ActionContext actionContext = new ActionContext();
		while (true) {
			if (mailbox.hasMail()) {
				Optional<Runnable> maybeLetter;
				while ((maybeLetter = mailbox.tryTakeMail()).isPresent()) {
					Runnable letter = maybeLetter.get();
					if (letter == POISON_LETTER) {
						return;
					}
					letter.run();
				}
			}

			performDefaultAction(actionContext);
		}
	}

走到這裏

protected void performDefaultAction(ActionContext context) throws Exception {
		if (!inputProcessor.processInput()) {
			context.allActionsCompleted();
		}
	}

繼續走到這裏
org.apache.flink.streaming.runtime.io.StreamOneInputProcessor

@Override
	public boolean processInput() throws Exception {
		initializeNumRecordsIn();

		StreamElement recordOrMark = input.pollNextNullable();
		if (recordOrMark == null) {
			input.isAvailable().get();
			return !checkFinished();
		}
		int channel = input.getLastChannel();
		checkState(channel != StreamTaskInput.UNSPECIFIED);

		processElement(recordOrMark, channel);
		return true;
	}

然後這裏，感覺要到頭了

private void processElement(StreamElement recordOrMark, int channel) throws Exception {
		if (recordOrMark.isRecord()) {
			// now we can do the actual processing
			StreamRecord<IN> record = recordOrMark.asRecord();
			synchronized (lock) {
				numRecordsIn.inc();
				streamOperator.setKeyContextElement1(record);
				streamOperator.processElement(record);
			}
		}
		else if (recordOrMark.isWatermark()) {
			// handle watermark
			statusWatermarkValve.inputWatermark(recordOrMark.asWatermark(), channel);
		} else if (recordOrMark.isStreamStatus()) {
			// handle stream status
			statusWatermarkValve.inputStreamStatus(recordOrMark.asStreamStatus(), channel);
		} else if (recordOrMark.isLatencyMarker()) {
			// handle latency marker
			synchronized (lock) {
				streamOperator.processLatencyMarker(recordOrMark.asLatencyMarker());
			}
		} else {
			throw new UnsupportedOperationException("Unknown type of StreamElement");
		}
	}

接下爲是這個類
org.apache.flink.streaming.api.operators.StreamFlatMap
最後調用flink-core裏的userFunction.flatmap 方法

@Override
	public void processElement(StreamRecord<IN> element) throws Exception {
		collector.setTimestamp(element);
		userFunction.flatMap(element.getValue(), collector);
	}