public interface CheckpointedFunction {
// Checkpoint觸發時會調用這個方法,我們要實現具體的snapshot邏輯,比如將哪些本地狀態持久化
void snapshotState(FunctionSnapshotContext context) throws Exception;
// 第一次初始化時會調用這個方法,或者從之前的檢查點恢復時也會調用這個方法
void initializeState(FunctionInitializationContext context) throws Exception;
}
可以自定義把哪些變量如何保存到檢查點,以及如何初始化、如何從檢查點恢復
public class MyFunction<T> implements MapFunction<T, T>, CheckpointedFunction {
private ReducingState<Long> countPerKey;
private ListState<Long> countPerPartition;
private long localCount;
public void initializeState(FunctionInitializationContext context) throws Exception {
// get the state data structure for the per-key state
countPerKey = context.getKeyedStateStore().getReducingState(
new ReducingStateDescriptor<>("perKeyCount", new AddFunction<>(), Long.class));
// get the state data structure for the per-partition state
countPerPartition = context.getOperatorStateStore().getOperatorState(
new ListStateDescriptor<>("perPartitionCount", Long.class));
// initialize the "local count variable" based on the operator state
for (Long l : countPerPartition.get()) {
localCount += l;
}
}
public void snapshotState(FunctionSnapshotContext context) throws Exception {
// the keyed state is always up to date anyways
// just bring the per-partition state in shape
countPerPartition.clear();
countPerPartition.add(localCount);
}
public T map(T value) throws Exception {
// update the states
countPerKey.add(1L);
localCount++;
return value;
}
}
flink 1.9.2, java1.8