Flink,coGroup雙流，coGroup實例，java版本

flink版本1.9.2,java版本1.8

package CoGroup;

import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.common.functions.JoinFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.api.windowing.assigners.TumblingProcessingTimeWindows;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.util.Collector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Date;
import java.util.Random;
import java.util.concurrent.TimeUnit;

/**
 * @Author you guess
 * @Date 2020/6/16 10:27
 * @Version 1.0
 * @Desc
 */
public class CoGroupTest {
    private static final Logger LOG = LoggerFactory.getLogger(CoGroupTest.class);
    private static final String[] TYPE = {"a", "b","c","d"};

    public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        //添加自定義數據源,每秒發出一筆訂單信息{商品名稱,商品數量}
        DataStreamSource<Tuple2<String, Integer>> orderSource1 = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {
            private volatile boolean isRunning = true;
            private final Random random = new Random();

            @Override
            public void run(SourceContext<Tuple2<String, Integer>> ctx) throws Exception {
                while (isRunning) {
                    TimeUnit.SECONDS.sleep(1);
                    Tuple2<String, Integer> tuple2 = Tuple2.of(TYPE[random.nextInt(TYPE.length)], random.nextInt(10));
                    System.out.println(new Date() + ",orderSource1提交元素:" + tuple2);
                    ctx.collect(tuple2);
                }
            }

            @Override
            public void cancel() {
                isRunning = false;
            }

        }, "orderSource1");

        DataStreamSource<Tuple2<String, Integer>> orderSource2 = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {
            private volatile boolean isRunning = true;
            private final Random random = new Random();

            @Override
            public void run(SourceContext<Tuple2<String, Integer>> ctx) throws Exception {
                while (isRunning) {
                    TimeUnit.SECONDS.sleep(1);
                    Tuple2<String, Integer> tuple2 = Tuple2.of(TYPE[random.nextInt(TYPE.length)], random.nextInt(10));
                    System.out.println(new Date() + ",orderSource2提交元素:" + tuple2);
                    ctx.collect(tuple2);
                }
            }

            @Override
            public void cancel() {
                isRunning = false;
            }

        }, "orderSource2");

        orderSource1.coGroup(orderSource2)
                .where(new KeySelector<Tuple2<String, Integer>, String>() {//指定第一個輸入的分區字段
            @Override
            public String getKey(Tuple2<String, Integer> value) throws Exception {
                return value.f0;
            }
        }).equalTo(new KeySelector<Tuple2<String, Integer>, String>() {//指定第二個輸入的分區字段。這倆分區字段要相同。
            @Override
            public String getKey(Tuple2<String, Integer> value) throws Exception {
                return value.f0;
            }
        })
                .window(TumblingProcessingTimeWindows.of(Time.seconds(4))).apply(new CoGroupFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>>() {
            @Override
            public void coGroup(Iterable<Tuple2<String, Integer>> first, Iterable<Tuple2<String, Integer>> second, Collector<Tuple2<String, Integer>> out) throws Exception {
                for (Tuple2<String, Integer> firstElement : first) {//第一個輸入
                    out.collect(firstElement);
                }
                for (Tuple2<String, Integer> secondElement : second) {//第二個輸入
                    out.collect(secondElement);
                }
            }
        }).print();
        env.execute("Flink Streaming Java API Skeleton");
    }
}

 

示例1:

Tue Jun 16 22:32:08 CST 2020,orderSource1提交元素:(d,3)
Tue Jun 16 22:32:08 CST 2020,orderSource2提交元素:(c,0)
Tue Jun 16 22:32:09 CST 2020,orderSource2提交元素:(b,7)
Tue Jun 16 22:32:09 CST 2020,orderSource1提交元素:(a,2)
Tue Jun 16 22:32:10 CST 2020,orderSource1提交元素:(b,9)
Tue Jun 16 22:32:10 CST 2020,orderSource2提交元素:(d,3)
Tue Jun 16 22:32:11 CST 2020,orderSource2提交元素:(a,8)
Tue Jun 16 22:32:11 CST 2020,orderSource1提交元素:(c,8)
3> (b,9)
8> (a,2)
7> (d,3)
7> (d,3)
8> (a,8)
3> (b,7)
6> (c,8)
6> (c,0)

整理後 =》

Tue Jun 16 22:32:08 CST 2020,orderSource1提交元素:(d,3)
Tue Jun 16 22:32:09 CST 2020,orderSource1提交元素:(a,2)
Tue Jun 16 22:32:10 CST 2020,orderSource1提交元素:(b,9)
Tue Jun 16 22:32:11 CST 2020,orderSource1提交元素:(c,8)

Tue Jun 16 22:32:08 CST 2020,orderSource2提交元素:(c,0)
Tue Jun 16 22:32:09 CST 2020,orderSource2提交元素:(b,7)
Tue Jun 16 22:32:10 CST 2020,orderSource2提交元素:(d,3)
Tue Jun 16 22:32:11 CST 2020,orderSource2提交元素:(a,8)

3> (b,9)
3> (b,7)
8> (a,2)
8> (a,8)
7> (d,3)
7> (d,3)
6> (c,8)
6> (c,0)

---------------

示例2:整理後：

Tue Jun 16 22:27:12 CST 2020,orderSource1提交元素:(c,6)
Tue Jun 16 22:27:13 CST 2020,orderSource1提交元素:(c,9)
Tue Jun 16 22:27:14 CST 2020,orderSource1提交元素:(b,3)
Tue Jun 16 22:27:15 CST 2020,orderSource1提交元素:(b,9)

Tue Jun 16 22:27:12 CST 2020,orderSource2提交元素:(c,1)
Tue Jun 16 22:27:13 CST 2020,orderSource2提交元素:(d,9)
Tue Jun 16 22:27:14 CST 2020,orderSource2提交元素:(d,8)
Tue Jun 16 22:27:15 CST 2020,orderSource2提交元素:(d,1)

6> (c,6)
6> (c,9)
6> (c,1)
3> (b,3)
3> (b,9)
7> (d,9)
7> (d,8)
7> (d,1)

只有c能匹配上，b和d沒有匹配上也會輸出。

6>中的6應該是分區標識，表示(c,6)、(c,9)、(c,1)都被分到6這個分區裏。

(b,3)、(b,9)都被分到3這個分區裏。

(d,9)、 (d,8)、(d,1)都被分到7這個分區裏。