執行
flink 作業的最小執行單元是task
示例
public class WorldCount {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
DataStream<Tuple2<String, Integer>> dataStream = env
.socketTextStream("localhost", Integer.parseInt(args[0]))
.flatMap(new Splitter())
.keyBy(0)
.timeWindow(Time.seconds(5))
.sum(1);
dataStream.print();
env.getExecutionPlan();
env.execute("Window WordCount");
}
public static class Splitter implements FlatMapFunction<String, Tuple2<String, Integer>> {
@Override
public void flatMap(String sentence, Collector<Tuple2<String, Integer>> out) throws Exception {
for (String word : sentence.split(" ")) {
out.collect(new Tuple2<String, Integer>(word, 1));
}
}
}
}
入口類
org.apache.flink.runtime.taskexecutor.TaskExecutor
核心方法
public CompletableFuture<Acknowledge> submitTask(){
// now load and instantiate the task's invokable code
invokable = loadAndInstantiateInvokable(userCodeClassLoader, nameOfInvokableClass, env);
// run the invokable
invokable.invoke();
}
source部分
這裏的invoable就是
org.apache.flink.streaming.runtime.tasks.SourceStreamTask 這個類，調用invoke方法
任務基類
org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable 所有的task都實現了這個類
所有的任務執行都是invoke
重點關注SourceStreamTask的初始化
主要通過反射拿到構造器，再調用instance方法初始化這個對象
Environment這個數據代表了這個任務的執行信息
private static AbstractInvokable loadAndInstantiateInvokable(
ClassLoader classLoader,
String className,
Environment environment) throws Throwable {
final Class<? extends AbstractInvokable> invokableClass;
try {
invokableClass = Class.forName(className, true, classLoader)
.asSubclass(AbstractInvokable.class);
} catch (Throwable t) {
throw new Exception("Could not load the task's invokable class.", t);
}
Constructor<? extends AbstractInvokable> statelessCtor;
try {
statelessCtor = invokableClass.getConstructor(Environment.class);
} catch (NoSuchMethodException ee) {
throw new FlinkException("Task misses proper constructor", ee);
}
// instantiate the class
try {
//noinspection ConstantConditions --> cannot happen
//----------------核心方法------------------
return statelessCtor.newInstance(environment);
} catch (InvocationTargetException e) {
// directly forward exceptions from the eager initialization
throw e.getTargetException();
} catch (Exception e) {
throw new FlinkException("Could not instantiate the task's invokable class.", e);
}
}
回到invoke方法
org.apache.flink.streaming.runtime.tasks.StreamTask 這個類給出了
invoke的實現
@Override
public final void invoke() throws Exception {
boolean disposed = false;
try {
operatorChain = new OperatorChain<>(this, recordWriters);
headOperator = operatorChain.getHeadOperator();
// task specific initialization
init();
// save the work of reloading state, etc, if the task is already canceled
if (canceled) {
throw new CancelTaskException();
}
// -------- Invoke --------
LOG.debug("Invoking {}", getName());
// we need to make sure that any triggers scheduled in open() cannot be
// executed before all operators are opened
synchronized (lock) {
// both the following operations are protected by the lock
// so that we avoid race conditions in the case that initializeState()
// registers a timer, that fires before the open() is called.
initializeState();
openAllOperators();
}
// final check to exit early before starting to run
if (canceled) {
throw new CancelTaskException();
}
//開始執行開個算子
isRunning = true;
//------------------------這是重要方法--------------------------
run();
// if this left the run() method cleanly despite the fact that this was canceled,
// make sure the "clean shutdown" is not attempted
if (canceled) {
throw new CancelTaskException();
}
LOG.debug("Finished task {}", getName());
// make sure no further checkpoint and notification actions happen.
// we make sure that no other thread is currently in the locked scope before
// we close the operators by trying to acquire the checkpoint scope lock
// we also need to make sure that no triggers fire concurrently with the close logic
// at the same time, this makes sure that during any "regular" exit where still
synchronized (lock) {
// this is part of the main logic, so if this fails, the task is considered failed
closeAllOperators();
// make sure no new timers can come
timerService.quiesce();
// only set the StreamTask to not running after all operators have been closed!
// See FLINK-7430
isRunning = false;
}
// make sure all timers finish
timerService.awaitPendingAfterQuiesce();
LOG.debug("Closed operators for task {}", getName());
// make sure all buffered data is flushed
operatorChain.flushOutputs();
// make an attempt to dispose the operators such that failures in the dispose call
// still let the computation fail
tryDisposeAllOperators();
disposed = true;
}
}
performDefaultAction 方法在org.apache.flink.streaming.runtime.tasks.SourceStreamTask 重寫了
protected void performDefaultAction(ActionContext context) throws Exception {
sourceThread.start();
}
啓動了sourceThread線程，再往下走
private class LegacySourceFunctionThread extends Thread {
private Throwable sourceExecutionThrowable;
LegacySourceFunctionThread() {
this.sourceExecutionThrowable = null;
}
//headOperator 代表第一步算子
@Override
public void run() {
try {
headOperator.run(getCheckpointLock(), getStreamStatusMaintainer(), operatorChain);
} catch (Throwable t) {
sourceExecutionThrowable = t;
} finally {
mailbox.clearAndPut(SOURCE_POISON_LETTER);
}
}
void checkThrowSourceExecutionException() throws Exception {
if (sourceExecutionThrowable != null) {
throw new Exception(sourceExecutionThrowable);
}
}
}
最後走到了
org.apache.flink.streaming.api.functions.source.SocketTextStreamFunction
這就是業務代碼了
public void run(SourceContext<String> ctx) throws Exception {
final StringBuilder buffer = new StringBuilder();
long attempt = 0;
while (isRunning) {
try (Socket socket = new Socket()) {
currentSocket = socket;
LOG.info("Connecting to server socket " + hostname + ':' + port);
socket.connect(new InetSocketAddress(hostname, port), CONNECTION_TIMEOUT_TIME);
try (BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream()))) {
char[] cbuf = new char[8192];
int bytesRead;
while (isRunning && (bytesRead = reader.read(cbuf)) != -1) {
buffer.append(cbuf, 0, bytesRead);
int delimPos;
while (buffer.length() >= delimiter.length() && (delimPos = buffer.indexOf(delimiter)) != -1) {
String record = buffer.substring(0, delimPos);
// truncate trailing carriage return
if (delimiter.equals("\n") && record.endsWith("\r")) {
record = record.substring(0, record.length() - 1);
}
ctx.collect(record);
buffer.delete(0, delimPos + delimiter.length());
}
}
}
}
// if we dropped out of this loop due to an EOF, sleep and retry
if (isRunning) {
attempt++;
if (maxNumRetries == -1 || attempt < maxNumRetries) {
LOG.warn("Lost connection to server socket. Retrying in " + delayBetweenRetries + " msecs...");
Thread.sleep(delayBetweenRetries);
}
else {
// this should probably be here, but some examples expect simple exists of the stream source
// throw new EOFException("Reached end of stream and reconnects are not enabled.");
break;
}
}
}
// collect trailing data
if (buffer.length() > 0) {
ctx.collect(buffer.toString());
}
}
窗口函數
上面主要分析了窗口函數這部分
這部分的task類
org.apache.flink.streaming.runtime.tasks.OneInputStreamTask
直接從run分析
performDefaultAction 這個方法和sourcetask走的不同
/**
* Runs the stream-tasks main processing loop.
*/
private void run() throws Exception {
final ActionContext actionContext = new ActionContext();
while (true) {
if (mailbox.hasMail()) {
Optional<Runnable> maybeLetter;
while ((maybeLetter = mailbox.tryTakeMail()).isPresent()) {
Runnable letter = maybeLetter.get();
if (letter == POISON_LETTER) {
return;
}
letter.run();
}
}
performDefaultAction(actionContext);
}
}
走到這裏
protected void performDefaultAction(ActionContext context) throws Exception {
if (!inputProcessor.processInput()) {
context.allActionsCompleted();
}
}
1
2
3
4
5
繼續走到這裏
org.apache.flink.streaming.runtime.io.StreamOneInputProcessor
@Override
public boolean processInput() throws Exception {
initializeNumRecordsIn();
StreamElement recordOrMark = input.pollNextNullable();
if (recordOrMark == null) {
input.isAvailable().get();
return !checkFinished();
}
int channel = input.getLastChannel();
checkState(channel != StreamTaskInput.UNSPECIFIED);
processElement(recordOrMark, channel);
return true;
}
然後這裏，感覺要到頭了
private void processElement(StreamElement recordOrMark, int channel) throws Exception {
if (recordOrMark.isRecord()) {
// now we can do the actual processing
StreamRecord<IN> record = recordOrMark.asRecord();
synchronized (lock) {
numRecordsIn.inc();
streamOperator.setKeyContextElement1(record);
streamOperator.processElement(record);
}
}
else if (recordOrMark.isWatermark()) {
// handle watermark
statusWatermarkValve.inputWatermark(recordOrMark.asWatermark(), channel);
} else if (recordOrMark.isStreamStatus()) {
// handle stream status
statusWatermarkValve.inputStreamStatus(recordOrMark.asStreamStatus(), channel);
} else if (recordOrMark.isLatencyMarker()) {
// handle latency marker
synchronized (lock) {
streamOperator.processLatencyMarker(recordOrMark.asLatencyMarker());
}
} else {
throw new UnsupportedOperationException("Unknown type of StreamElement");
}
}
接下爲是這個類
org.apache.flink.streaming.api.operators.StreamFlatMap
最後調用flink-core裏的userFunction.flatmap 方法
@Override
public void processElement(StreamRecord<IN> element) throws Exception {
collector.setTimestamp(element);
userFunction.flatMap(element.getValue(), collector);
}