Flink源碼-8-JobVertex生成JobGraph

JobGraph

通常是由DataStream生成的，然後轉換生成的JobGraph
我們可以直接JobVertex--------->JobGraph 來深入理解這個過程

JobVertex

       //生成一個頂點
		final JobVertex sender = new JobVertex("Sender");
		sender.setParallelism(parallelism);
		//設置反射類
		sender.setInvokableClass(TestingAbstractInvokables.Sender.class);


		//生成一個頂點
		final JobVertex receiver = new JobVertex("Receiver");
		receiver.setParallelism(parallelism);
		receiver.setInvokableClass(TestingAbstractInvokables.Receiver.class);

		//生成一個邊
		receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

		//生成一個jobgraph圖
		final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);

		return jobGraph;

構造具體的Job

Sender類
Receiver類

public class TestingInvokables {

	private TestingInvokables() {
		throw new UnsupportedOperationException(getClass().getSimpleName() + " should not be instantiated.");
	}



	public static class Sender extends AbstractInvokable {

		public Sender(Environment environment) {
			super(environment);
		}

		@Override
		public void invoke() throws Exception {
			//寫數據
			final RecordWriter<IntValue> writer = new RecordWriterBuilder<IntValue>().build(getEnvironment().getWriter(0));
			try {
				writer.emit(new IntValue(42));
				writer.emit(new IntValue(1337));
				writer.flushAll();
			} finally {
				writer.clearBuffers();
			}
		}
	}


	public static class Receiver extends AbstractInvokable {

		public Receiver(Environment environment) {
			super(environment);
		}

		@Override
		public void invoke() throws Exception {
			//讀數據
			final RecordReader<IntValue> reader = new RecordReader<>(
				getEnvironment().getInputGate(0),
				IntValue.class,
				getEnvironment().getTaskManagerInfo().getTmpDirectories());
			final IntValue i1 = reader.next();
			System.out.println("輸出" + i1);
			final IntValue i2 = reader.next();
			System.out.println("輸出" + i2);
			final IntValue i3 = reader.next();
			System.out.println("輸出" + i3);
			if (i1.getValue() != 42 || i2.getValue() != 1337 || i3 != null) {
				throw new Exception("Wrong data received.");
			}
		}
	}
}

測試類

@Test
	public void testCoLocationConstraintJobExecution() throws Exception {
		final int numSlotsPerTaskExecutor = 1;
		final int numTaskExecutors = 1;
		final int parallelism = numTaskExecutors * numSlotsPerTaskExecutor;
		final JobGraph jobGraph = createNewJobGraph(parallelism);

		final TestingMiniClusterConfiguration miniClusterConfiguration = new TestingMiniClusterConfiguration.Builder()
			.setNumSlotsPerTaskManager(numSlotsPerTaskExecutor)
			.setNumTaskManagers(numTaskExecutors)
			.setLocalCommunication(true)
			.build();


       // 測試任務提交
		try (MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration)) {
			miniCluster.start();
			miniCluster.submitJob(jobGraph).get();
			final CompletableFuture<JobResult> jobResultFuture = miniCluster.requestJobResult(jobGraph.getJobID());
			assertThat(jobResultFuture.get().isSuccess(), is(true));
		}
	}