Flink 廣播變量 實例

版本： flink1.9.2,java1.8

廣播變量用於DataSet：

package BroadCast;

import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.configuration.Configuration;

import java.util.ArrayList;
import java.util.List;

/**
 * @Author you guess
 * @Date 2020/6/24 21:02
 * @Version 1.0
 * @Desc
 */
public class BroadCastSetTest1 {

    public static void main(String[] args) throws Exception {

        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

        // 1. 準備等待廣播的DataSet數據
        DataSet<Integer> broadCastSet = env.fromElements(300, 400);

        DataSet<String> data = env.fromElements("a", "b");

        data.map(new RichMapFunction<String, String>() {

            //需要定義在map方法裏面，不然報錯。是匿名內部類
            List<Integer> broadcastList = new ArrayList<>();

            @Override
            public void open(Configuration parameters) throws Exception {
                // 3. 獲取廣播的DataSet數據 作爲一個Collection
                broadcastList = getRuntimeContext().getBroadcastVariable("name1");
            }

            @Override
            public String map(String value) throws Exception {
                return value + broadcastList.get(value.charAt(0) - "a".charAt(0));
            }
        }).withBroadcastSet(broadCastSet, "name1").print(); // 2. 廣播DataSet

        //處理數據集 不必寫execute，不然有異常
        //env.execute("Flink  BroadCastStreamTest1 ");
    }
}
/*
輸出：
a300
b400
*/

源碼：實際用的HashMap存儲的

@Public
public abstract class SingleInputUdfOperator<IN, OUT, O extends SingleInputUdfOperator<IN, OUT, O>>
	extends SingleInputOperator<IN, OUT, O> implements UdfOperator<O> {
......

	@Override
	public O withBroadcastSet(DataSet<?> data, String name) {
		if (data == null) {
			throw new IllegalArgumentException("Broadcast variable data must not be null.");
		}
		if (name == null) {
			throw new IllegalArgumentException("Broadcast variable name must not be null.");
		}

		if (this.broadcastVariables == null) {
			this.broadcastVariables = new HashMap<String, DataSet<?>>();
		}

		this.broadcastVariables.put(name, data);

		@SuppressWarnings("unchecked")
		O returnType = (O) this;
		return returnType;
	}
......
}

 

廣播變量用於DataStream：

      DataStream<Long> num = source1.broadcast().map(new MapFunction<Long, Long>() {
            @Override
            public Long map(Long value) throws Exception {
                long id = Thread.currentThread().getId();
                System.out.println("線程id：" + id + ",接收到數據：" + value);
                return value;
            }
        });