一.項目概述
基本需求
- 用戶在短時間內頻繁登錄失敗，有程序惡意攻擊的可能
- 同一用戶（可以是不同IP）在2秒內連續兩次登錄失敗，需要報警
解決思路
- 將用戶的登錄失敗行爲存入 ListState，設定定時器2秒後觸發，查看 ListState 中有幾次失敗登錄
- 更加精確的檢測，可以使用 CEP 庫實現事件流的模式匹配
二.代碼
2.1 pom文件配置
pom文件配置如下:
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>1.10.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.11</artifactId>
<version>1.10.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka_2.11</artifactId>
<version>1.10.1</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>1.10.1</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_2.11</artifactId>
<version>1.10.1</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-redis_2.11</artifactId>
<version>1.1.5</version>
</dependency>
<!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java -->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.19</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-statebackend-rocksdb_2.11</artifactId>
<version>1.10.1</version>
</dependency>
<!-- Table API 和 Flink SQL -->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-planner-blink_2.11</artifactId>
<version>1.10.1</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-planner_2.11</artifactId>
<version>1.10.1</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-api-java-bridge_2.11</artifactId>
<version>1.10.1</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-scala_2.11</artifactId>
<version>1.10.1</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-common</artifactId>
<version>1.10.1</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-csv</artifactId>
<version>1.10.1</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-cep_2.11</artifactId>
<version>1.10.1</version>
</dependency>
2.2 POJO類
LoginEvent
private Long userId;
private String ip;
private String loginState;
private Long timestamp;
LoginFailWarning
private Long userId;
private Long firstFailTime;
private Long lastFailTime;
private String warningMsg;
2.3 惡意登陸監控 - KeyedProcessFunction
代碼:
package com.zqs.flink.project.loginfail_detect;
/**
* @author 只是甲
* @date 2021-10-20
* @remark 登陸失敗監控
*/
import com.zqs.flink.project.loginfail_detect.beans.LoginFailWarning;
import com.zqs.flink.project.loginfail_detect.beans.LoginEvent;
import org.apache.flink.api.common.state.ListState;
import org.apache.flink.api.common.state.ListStateDescriptor;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.shaded.guava18.com.google.common.collect.Lists;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.util.Collector;
import java.net.URL;
import java.util.ArrayList;
import java.util.Iterator;
public class LoginFail {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
// 1. 從文件中讀取
URL resource = LoginFail.class.getResource("/LoginLog.csv");
DataStream<LoginEvent> loginEventStream= env.readTextFile(resource.getPath())
.map(line -> {
String[] fields = line.split(",");
return new LoginEvent(new Long(fields[0]), fields[1], fields[2], new Long(fields[3]));
})
.assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<LoginEvent>(Time.seconds(3)) {
@Override
public long extractTimestamp(LoginEvent element) {
return element.getTimestamp()*1000L;
}
});
// 自定義處理函數檢測連續登錄失敗事件
SingleOutputStreamOperator<LoginFailWarning> warningStream = loginEventStream
.keyBy(LoginEvent::getUserId)
.process(new LoginFailDetectWarning(2));
warningStream.print();
env.execute("login fail detect job");
}
// 實現自定義KeyedProcessFunction
public static class LoginFailDetectWarning0 extends KeyedProcessFunction<Long, LoginEvent, LoginFailWarning>{
// 定義屬性，最大連續登錄失敗次數
private Integer maxFailTimes;
public LoginFailDetectWarning0(Integer maxFailTimes) {
this.maxFailTimes = maxFailTimes;
}
// 定義狀態: 保存2秒內所有登陸失敗事件
ListState<LoginEvent> loginFailEventListState;
// 定義狀態: 保存註冊的定時器時間戳
ValueState<Long> timerTsState;
@Override
public void open(Configuration parameters) throws Exception {
loginFailEventListState = getRuntimeContext().getListState(new ListStateDescriptor<LoginEvent>("login-fail-list", LoginEvent.class));
timerTsState = getRuntimeContext().getState(new ValueStateDescriptor<Long>("timer-ts", Long.class));
}
@Override
public void processElement(LoginEvent value, Context ctx, Collector<LoginFailWarning> out) throws Exception {
// 判斷當前登陸事件
if ( "fail".equals(value.getLoginState()) ){
// 1. 如果是失敗事件, 添加到列表狀態中
loginFailEventListState.add(value);
// 如果沒有定時器,註冊一個2秒之後的定時器
if(timerTsState.value() == null ){
Long ts = (value.getTimestamp() + 2) * 1000L;
ctx.timerService().registerEventTimeTimer(ts);
timerTsState.update(ts);
}
} else {
// 2. 如果是登陸成功, 刪除定時器，清空狀態，重新開始
if( timerTsState.value() != null )
ctx.timerService().deleteEventTimeTimer(timerTsState.value());
loginFailEventListState.clear();
timerTsState.clear();
}
}
@Override
public void onTimer(long timestamp, OnTimerContext ctx, Collector<LoginFailWarning> out) throws Exception {
// 定時器觸發, 說明2秒內沒有登陸成功來， 判斷ListState中的失敗個數
ArrayList<LoginEvent> loginFailEvents = Lists.newArrayList(loginFailEventListState.get());
Integer failTimes = loginFailEvents.size();
if( failTimes >= maxFailTimes ){
// 如果超出設定的最大失敗次數，輸出報警
out.collect( new LoginFailWarning(ctx.getCurrentKey(),
loginFailEvents.get(0).getTimestamp(),
loginFailEvents.get(failTimes -1).getTimestamp(),
"login fail in 2s for " + failTimes + " times"
));
}
// 清空狀態
loginFailEventListState.clear();
timerTsState.clear();
}
}
// 實現自定義KeyedProcessFunction
public static class LoginFailDetectWarning extends KeyedProcessFunction<Long, LoginEvent, LoginFailWarning>{
// 定義屬性, 最大連續登陸失敗次數
private Integer maxFailTimes;
public LoginFailDetectWarning(Integer maxFailTimes) {
this.maxFailTimes = maxFailTimes;
}
// 定義狀態: 保存2秒內所有的登陸失敗事件
ListState<LoginEvent> loginEventListState;
@Override
public void open(Configuration parameters) throws Exception {
loginEventListState = getRuntimeContext().getListState(new ListStateDescriptor<LoginEvent>("login-fail-list", LoginEvent.class));
}
@Override
public void processElement(LoginEvent value, Context ctx, Collector<LoginFailWarning> out) throws Exception {
// 判斷當前事件登陸狀態
if ( "fail".equals(value.getLoginState()) ){
// 1. 如果是登陸失敗, 獲取狀態中之前的登陸失敗事件， 繼續判斷是否已有失敗事件
Iterator<LoginEvent> iterator = loginEventListState.get().iterator();
if( iterator.hasNext() ){
// 1.1 如果已經有登陸失敗事件, 繼續判斷是否已有失敗事件
// 獲取已有的登陸失敗事件
LoginEvent firstFailEvent = iterator.next();
if (value.getTimestamp() - firstFailEvent.getTimestamp() <= 2){
// 1.1.1 如果2秒之內， 輸出報警
out.collect( new LoginFailWarning(value.getUserId(), firstFailEvent.getTimestamp(), value.getTimestamp(), "login fail 2 times in 2s"));
}
// 不管報不報警， 這次都已處理完畢，直接更新狀態
loginEventListState.clear();
loginEventListState.add(value);
} else {
// 1.2 如果沒有登陸失敗，直接將當前事件存入ListState
loginEventListState.add(value);
}
} else {
// 2. 如果是登陸成功，直接清空狀態
loginEventListState.clear();
}
}
}
}
2.4 惡意登陸監控 - CEP
代碼:
package com.zqs.flink.project.loginfail_detect;
/**
* @author 只是甲
* @date 2021-10-20
* @remark 登陸失敗監控 - CEP
*/
import com.zqs.flink.project.loginfail_detect.beans.LoginEvent;
import com.zqs.flink.project.loginfail_detect.beans.LoginFailWarning;
import org.apache.flink.cep.CEP;
import org.apache.flink.cep.PatternSelectFunction;
import org.apache.flink.cep.PatternStream;
import org.apache.flink.cep.pattern.Pattern;
import org.apache.flink.cep.pattern.conditions.SimpleCondition;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor;
import org.apache.flink.streaming.api.windowing.time.Time;
import sun.rmi.runtime.Log;
import java.net.URL;
import java.util.List;
import java.util.Map;
public class LoginFailWithCep {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
// 1. 從文件中讀取數據
URL resource = LoginFailWithCep.class.getResource("/LoginLog.csv");
DataStream<LoginEvent> loginEventStream = env.readTextFile(resource.getPath())
.map(line -> {
String[] fields = line.split(",");
return new LoginEvent(new Long(fields[0]), fields[1], fields[2], new Long(fields[3]));
})
.assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<LoginEvent>(Time.seconds(3)) {
@Override
public long extractTimestamp(LoginEvent element) {
return element.getTimestamp() * 1000L;
}
});
// 1. 定義一個匹配模式
// firstFail -> secondFail, within 2s
Pattern<LoginEvent, LoginEvent> loginFailPattern0 = Pattern
.<LoginEvent>begin("firstFail").where(new SimpleCondition<LoginEvent>() {
@Override
public boolean filter(LoginEvent value) throws Exception {
return "fail".equals(value.getLoginState());
}
})
.next("secondFail").where(new SimpleCondition<LoginEvent>() {
@Override
public boolean filter(LoginEvent value) throws Exception {
return "fail".equals(value.getLoginState());
}
})
.next("thirdFail").where(new SimpleCondition<LoginEvent>() {
@Override
public boolean filter(LoginEvent value) throws Exception {
return "fail".equals(value.getLoginState());
}
})
.within(Time.seconds(3));
Pattern<LoginEvent, LoginEvent> loginFailPattern = Pattern
.<LoginEvent>begin("failEvents").where(new SimpleCondition<LoginEvent>() {
@Override
public boolean filter(LoginEvent value) throws Exception {
return "fail".equals(value.getLoginState());
}
}).times(3).consecutive()
.within(Time.seconds(5));
// 2. 將匹配模式應用到數據流上，得到一個pattern stream
PatternStream<LoginEvent> patternStream = CEP.pattern(loginEventStream.keyBy(LoginEvent::getUserId), loginFailPattern);
// 3. 檢出符合匹配條件的複雜事件，進行轉換處理，得到報警信息
SingleOutputStreamOperator<LoginFailWarning> warningStream = patternStream.select(new LoginFailMatchDetectWarning());
warningStream.print();
env.execute("login fail detect with cep job");
}
// 實現自定義的PatternSelectFunction
public static class LoginFailMatchDetectWarning implements PatternSelectFunction<LoginEvent, LoginFailWarning>{
@Override
public LoginFailWarning select(Map<String, List<LoginEvent>> pattern) throws Exception {
LoginEvent firstFailEvent = pattern.get("failEvents").get(0);
LoginEvent lastFailEvent = pattern.get("failEvents").get(pattern.get("failEvents").size() - 1);
return new LoginFailWarning(firstFailEvent.getUserId(), firstFailEvent.getTimestamp(), lastFailEvent.getTimestamp(), "login fail " + pattern.get("failEvents").size() + " times");
}
}
}
測試記錄:
