緩存擊穿之布隆過濾器bloom Filter實現方式

一、什麼是緩存擊穿

查詢一個在緩存內必然不存在的數據，導致每次請求都要去存儲層去查詢，這樣緩存就失去了意義。如果在大流量下數據庫可能掛掉。緩存擊穿是黑客攻擊系統的常用手段。

二、怎麼解決緩存擊穿問題？

採用布隆過濾器來實現。

什麼是布隆過濾器？

它是一種空間效率極高的概率型算法和數據結構，用於判斷一個元素是否在集合中(類似Hashset)。它的核心是一個很長的二進制向量和一系列的hash函數。

java中如何使用布隆過濾器？

使用谷歌的guava實現布隆過濾器。

bloom Filter布隆過濾器優劣勢？

優勢:

1)全量存儲但不存儲元素本身，在某些保密要求非常嚴格的場合有優勢

2)空間效率高

3)插入/查詢時間都是常數，遠遠超過一般算法

劣勢:

1）存在誤算率，隨着存入的元素數量增加，誤算率也隨着增加

2)一般情況下不能從布隆過濾器刪除元素

3)數組長度以及hash函數個數確定過程複雜

布隆過濾器的使用場景？

1)垃圾郵件地址過濾(地址數量很龐大)

2)爬蟲URL地址去重

3)解決緩存擊穿問題

4)瀏覽器安全瀏覽網址提醒

5)google 分佈式數據庫Bigtable以及Hbase使用布隆過濾器來查找不存在的行或列，以減少磁盤查找的IO次數

6)文檔存儲檢索系統也可以採用布隆過濾器來檢測先前存儲的數據

三、java中使用guava中的bloom Filter解決緩存擊穿問題

工程目錄結構圖:

模擬100個線程同時查詢緩存中必不存在的情況。

package com.ocean.cache;

import java.util.List;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;

import javax.annotation.PostConstruct;
import javax.annotation.Resource;

import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.ValueOperations;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import com.google.common.base.Charsets;
import com.google.common.hash.BloomFilter;
import com.google.common.hash.Funnels;
import com.ocean.dao.UserDao;
import com.ocean.dto.UserDto;

/**
 * 緩存擊穿
 * @author 
 *
 */
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations = {"classpath:config/spring/spring-dao.xml",
		"classpath:config/spring/spring-bean.xml",
		"classpath:config/spring/spring-redis.xml"})
public class CacheBreakDownTest {
	private static final Logger logger = LoggerFactory.getLogger(CacheBreakDownTest.class);
	
	private static final int THREAD_NUM = 100;//線程數量
	
	@Resource
	private UserDao UserDao;
	
	@Resource
	private RedisTemplate redisTemplate;
	
	private int count = 0;
	
	//初始化一個計數器
	private CountDownLatch countDownLatch = new CountDownLatch(THREAD_NUM);
	
	private BloomFilter<String> bf;
	
	List<UserDto> allUsers;
	
	@PostConstruct
	public void init(){
		//將數據從數據庫導入到本地
		allUsers = UserDao.getAllUser();
		if(allUsers == null || allUsers.size()==0){
			return;
		}
		//創建布隆過濾器(默認3%誤差)
		bf = BloomFilter.create(Funnels.stringFunnel(Charsets.UTF_8), allUsers.size());
		//將數據存入布隆過濾器
		for(UserDto userDto : allUsers){
			bf.put(userDto.getUserName());
		}
	}
	
	@Test
	public void cacheBreakDownTest(){
		for(int i=0;i<THREAD_NUM;i++){
			new Thread(new MyThread()).start();
			//計數器減一
			countDownLatch.countDown();
		}
		try {
			Thread.currentThread().join();
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
	}
	
	class MyThread implements Runnable{

		@Override
		public void run() {
			try {
				//所有子線程等待，當子線程全部創建完成再一起併發執行後面的代碼
				countDownLatch.await();
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
			//隨機產生一個字符串
			String randomUser = UUID.randomUUID().toString();
//			String randomUser = allUsers.get(new Random().nextInt(allUsers.size())).getUserName();
			String key = "Key:"+randomUser;
			
			//如果布隆過濾器中不存在這個用戶直接返回，將流量擋掉
			if(!bf.mightContain(randomUser)){
				System.out.println("bloom filter don't has this user");
				return;
			}
			//查詢緩存，如果緩存中存在直接返回緩存數據
			ValueOperations<String,String> operation = (ValueOperations<String, String>) redisTemplate.opsForValue();
			synchronized (countDownLatch) {
				Object cacheUser = operation.get(key);
				if(cacheUser!=null){
					System.out.println("return user from redis");
					return;
				}
				//如果緩存不存在查詢數據庫
				List<UserDto> user = UserDao.getUserByUserName(randomUser);
				if(user == null || user.size() == 0){
					return;
				}
				//將mysql數據庫查詢到的數據寫入到redis中
				System.out.println("write to redis");
				operation.set("Key:"+user.get(0).getUserName(), user.get(0).getUserName());
			}
		}
		
	}
}

maven配置文件pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"  
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">  
     <modelVersion>4.0.0</modelVersion>
  <groupId>manager</groupId>
  <artifactId>manager</artifactId>
  <packaging>war</packaging>
  <version>0.0.1-SNAPSHOT</version>
  <name>manager Maven Webapp</name>
  <url>http://maven.apache.org</url>
  <properties>
		<guava.version>21.0</guava.version>
	</properties>
    <dependencies>  
        <dependency>  
            <groupId>junit</groupId>  
            <artifactId>junit</artifactId>  
            <version>4.9</version>  
        </dependency> 
        
        <dependency>
			<groupId>org.mockito</groupId>
			<artifactId>mockito-all</artifactId>
			<version>1.10.19</version>
			<scope>test</scope>
		</dependency> 
        <!-- spring-mvc包 -->  
        <dependency>  
            <groupId>org.springframework</groupId>  
            <artifactId>spring-webmvc</artifactId>  
            <version>4.2.0.RELEASE</version>  
        </dependency>  
        <!-- jsp顯示數據需要的jar包 -->  
        <dependency>  
            <groupId>jstl</groupId>  
            <artifactId>jstl</artifactId>  
            <version>1.2</version>  
        </dependency>  
        <!--日誌 -->  
        <dependency>  
            <groupId>log4j</groupId>  
            <artifactId>log4j</artifactId>  
            <version>1.2.17</version>  
        </dependency>  
        <dependency>  
            <groupId>org.apache.logging.log4j</groupId>  
            <artifactId>log4j-core</artifactId>  
            <version>2.0</version>  
        </dependency>  
        <dependency>  
            <groupId>org.apache.logging.log4j</groupId>  
            <artifactId>log4j-api</artifactId>  
            <version>2.0</version>  
        </dependency>  
        <!--spring4.0以上json和java對象的轉換jackson包 spring4.0以上需要jackson2.6版本 -->  
        <dependency>  
            <groupId>com.fasterxml.jackson.core</groupId>  
            <artifactId>jackson-core</artifactId>  
            <version>2.6.1</version>  
        </dependency>  
        <dependency>  
            <groupId>com.fasterxml.jackson.core</groupId>  
            <artifactId>jackson-databind</artifactId>  
            <version>2.6.1</version>  
        </dependency>  
        <dependency>  
            <groupId>com.fasterxml.jackson.core</groupId>  
            <artifactId>jackson-annotations</artifactId>  
            <version>2.6.1</version>  
        </dependency>  
        <!--mybatis的包 -->  
        <dependency>  
            <groupId>org.mybatis</groupId>  
            <artifactId>mybatis</artifactId>  
            <version>3.2.8</version>  
        </dependency>  
        <!--mybatis和spring整合包 -->  
        <dependency>  
            <groupId>org.mybatis</groupId>  
            <artifactId>mybatis-spring</artifactId>  
            <version>1.2.2</version>  
        </dependency>  
        <!-- 我使用的連接數據的jar包 -->  
        <dependency>  
            <groupId>commons-dbcp</groupId>  
            <artifactId>commons-dbcp</artifactId>  
            <version>1.4</version>  
        </dependency>  
        <dependency>  
            <groupId>org.springframework</groupId>  
            <artifactId>spring-jdbc</artifactId>  
            <version>4.2.0.RELEASE</version>  
        </dependency>  
        <!--spring事務tx包的引用 -->  
        <dependency>  
            <groupId>org.springframework</groupId>  
            <artifactId>spring-tx</artifactId>  
            <version>4.2.0.RELEASE</version>  
        </dependency>  
  
        <!--數據庫jar包 -->  
        <dependency>
			<groupId>mysql</groupId>
			<artifactId>mysql-connector-java</artifactId>
			<version>5.1.39</version>
		</dependency> 
        <dependency>  
            <groupId>commons-io</groupId>  
            <artifactId>commons-io</artifactId>  
            <version>2.2</version>  
        </dependency>  
        
        <!-- https://mvnrepository.com/artifact/dom4j/dom4j -->
		<dependency>
    		<groupId>dom4j</groupId>
    		<artifactId>dom4j</artifactId>
    		<version>1.6.1</version>
		</dependency>
		 <dependency>  
    		<groupId>jaxen</groupId>  
    		<artifactId>jaxen</artifactId>  
    		<version>1.1.6</version>  
		</dependency>
		<!-- google guava jar -->
		<dependency>
			<groupId>com.google.guava</groupId>
			<artifactId>guava</artifactId>
			<version>${guava.version}</version>
		</dependency>
		
		<dependency>
			<groupId>org.slf4j</groupId>
			<artifactId>slf4j-api</artifactId>
			<version>1.7.21</version>
		</dependency>
		
		<!-- https://mvnrepository.com/artifact/org.springframework/spring-test -->
	<dependency>
		<groupId>org.springframework</groupId>
		<artifactId>spring-test</artifactId>
		<version>4.2.9.RELEASE</version>
		<scope>test</scope>
	</dependency>
	
	<dependency> 
	    	<groupId>redis.clients</groupId> 
	    	<artifactId>jedis</artifactId>
	    	<version>2.6.2</version>
	</dependency>
	
	<!-- https://mvnrepository.com/artifact/org.springframework.data/spring-data-redis -->
	<dependency>
    	<groupId>org.springframework.data</groupId>
    	<artifactId>spring-data-redis</artifactId>
   	 	<version>1.2.0.RELEASE</version>
	</dependency>
        
    </dependencies>  
    <build>  
        <finalName>manager</finalName>  
    </build>  
</project>

spring-bean.xml文件

<beans xmlns="http://www.springframework.org/schema/beans"  
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:mvc="http://www.springframework.org/schema/mvc"  
    xmlns:context="http://www.springframework.org/schema/context"  
    xmlns:aop="http://www.springframework.org/schema/aop" xmlns:tx="http://www.springframework.org/schema/tx"  
    xsi:schemaLocation="http://www.springframework.org/schema/beans   
        http://www.springframework.org/schema/beans/spring-beans-3.2.xsd   
        http://www.springframework.org/schema/mvc   
        http://www.springframework.org/schema/mvc/spring-mvc-3.2.xsd   
        http://www.springframework.org/schema/context   
        http://www.springframework.org/schema/context/spring-context-3.2.xsd   
        http://www.springframework.org/schema/aop   
        http://www.springframework.org/schema/aop/spring-aop-3.2.xsd   
        http://www.springframework.org/schema/tx   
        http://www.springframework.org/schema/tx/spring-tx-3.2.xsd ">  
    
    <bean id="orderDao" class="com.ocean.spring.bean.OrderDao">
	</bean>
	
	<bean id="orderService" class="com.ocean.spring.bean.OrderService">
		<!-- 需要注入的屬性 -->
		<property name="orderDao" ref="orderDao">
		</property>
	</bean>  
</beans>

spring-dao.xml文件

<beans xmlns="http://www.springframework.org/schema/beans"  
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:mvc="http://www.springframework.org/schema/mvc"  
    xmlns:context="http://www.springframework.org/schema/context"  
    xmlns:aop="http://www.springframework.org/schema/aop" xmlns:tx="http://www.springframework.org/schema/tx"  
    xsi:schemaLocation="http://www.springframework.org/schema/beans   
        http://www.springframework.org/schema/beans/spring-beans-3.2.xsd   
        http://www.springframework.org/schema/mvc   
        http://www.springframework.org/schema/mvc/spring-mvc-3.2.xsd   
        http://www.springframework.org/schema/context   
        http://www.springframework.org/schema/context/spring-context-3.2.xsd   
        http://www.springframework.org/schema/aop   
        http://www.springframework.org/schema/aop/spring-aop-3.2.xsd   
        http://www.springframework.org/schema/tx   
        http://www.springframework.org/schema/tx/spring-tx-3.2.xsd ">  
    <!--配置數據庫連接屬性-->  
    <!--配置數據庫連接屬性加載db.properties的數據庫連接信息文件 這裏配置有很多學問，需要我以後學習-->  
    <context:property-placeholder location="classpath:db.properties"></context:property-placeholder>  
    <bean id="dataSource" class="org.apache.commons.dbcp.BasicDataSource">  
        <property name="driverClassName" value="${jdbc.driver}"/>  
        <property name="url" value="${jdbc.url}"/>  
        <property name="username" value="${jdbc.username}"/>  
        <property name="password" value="${jdbc.password}"/>  
        <property name="maxActive" value="30"/>  
        <property name="maxIdle" value="5"/>  
    </bean>  
  
    <!-- 讓spring管理sqlsessionfactory 使用mybatis和spring整合包中的 -->  
    <bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">  
        <!--注入數據庫連接對象-->  
        <property name="dataSource" ref="dataSource"/>  
        <!--加載mybatis配置文件 這裏mybatis配置文件的工作都在spring中配置了所以mybatis只是配置別名就可以-->  
        <property name="configLocation" value="classpath:/config/mybatis/mybatisconfig.xml"/>  
    </bean>  
      
    <!-- mapper掃描器 -->  
   <!--  <bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">  
        掃描的包，如果要掃描多個，中間用,隔開  
        <property name="basePackage" value="com.ys.mybatis.mapper"></property>  
        <property name="sqlSessionFactoryBeanName" value="sqlSessionFactory" />  
    </bean> -->
    
    <bean id="userDao" class="org.mybatis.spring.mapper.MapperFactoryBean">
		<property name="sqlSessionFactory" ref="sqlSessionFactory" />
		<property name="mapperInterface" value="com.ocean.dao.UserDao" />
	</bean> 
  
</beans>

spring-redis.xml文件


<beans xmlns="http://www.springframework.org/schema/beans"  
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:mvc="http://www.springframework.org/schema/mvc"  
    xmlns:context="http://www.springframework.org/schema/context"  
    xmlns:aop="http://www.springframework.org/schema/aop" xmlns:tx="http://www.springframework.org/schema/tx"  
    xsi:schemaLocation="http://www.springframework.org/schema/beans   
        http://www.springframework.org/schema/beans/spring-beans-3.2.xsd   
        http://www.springframework.org/schema/mvc   
        http://www.springframework.org/schema/mvc/spring-mvc-3.2.xsd   
        http://www.springframework.org/schema/context   
        http://www.springframework.org/schema/context/spring-context-3.2.xsd   
        http://www.springframework.org/schema/aop   
        http://www.springframework.org/schema/aop/spring-aop-3.2.xsd   
        http://www.springframework.org/schema/tx   
        http://www.springframework.org/schema/tx/spring-tx-3.2.xsd ">  
    
	<!-- redis 連接池-->
	<bean id="jedisPoolConfig" class="redis.clients.jedis.JedisPoolConfig">
	</bean>
 
 <!-- redis 數據源 -->
	<bean id="jedisConnectionFactory"
    class="org.springframework.data.redis.connection.jedis.JedisConnectionFactory">
    
    <!-- 連接地址 -->    
    <property name="hostName" value="127.0.0.1" />
    <!-- 連接端口號 -->
    <property name="port" value="6379" />
    <!-- 連接超時時間 -->
    <!-- <property name="timeout" value="3000" /> -->
    <!-- 連接連接池 -->
    <property name="poolConfig" ref="jedisPoolConfig" />
   
    <property name="usePool" value="true" />
	</bean>
 
 
	<bean id="redisTemplate" class="org.springframework.data.redis.core.RedisTemplate">
    	<property name="connectionFactory" ref="jedisConnectionFactory" />
    	<!-- 解決redis序列化問題 -->
    	<property name="keySerializer">  
        	<bean class="org.springframework.data.redis.serializer.StringRedisSerializer" />  
    	</property>  
    	<property name="valueSerializer">  
        	<bean class="org.springframework.data.redis.serializer.StringRedisSerializer" />  
    	</property>  
    	<property name="hashKeySerializer">  
        	<bean class="org.springframework.data.redis.serializer.StringRedisSerializer" />  
    	</property>  
    	<property name="hashValueSerializer">  
        	<bean class="org.springframework.data.redis.serializer.StringRedisSerializer" />  
   		 </property> 
	</bean> 
</beans>

mybatisconfig.xml文件

<?xml version="1.0" encoding="UTF-8" ?>  
<!DOCTYPE configuration  
PUBLIC "-//mybatis.org//DTD Config 3.0//EN"  
"http://mybatis.org/dtd/mybatis-3-config.dtd">  
<configuration>  
    <typeAliases>  
        <!-- 掃描別名 -->  
       <typeAlias type="com.ocean.dto.UserDto" alias="userDto" />
    </typeAliases>  
<!-- 加載我們的Sql語句的xml文件 -->  
  <mappers>
		<mapper resource="config/mybatis/user.xml" />
  </mappers>		
<!-- 使用自動掃描器時，mapper.xml文件如果和mapper.java接口在一個目錄則此處不用定義mappers -->  
</configuration>

user.xml文件

<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC 
	"-//mybatis.org//DTD Mapper 3.0//EN" 
	" http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.ocean.dao.UserDao">
	<select id="getUser" resultType="userDto" parameterType="int">
		select account userName,password passWord from market.`store_account` where id = #{id}
	</select>
	
	<select id="getAllUser" resultType="userDto">
		select account userName,password passWord from market.`store_account`
	</select>
	
	<select id="getUserByUserName" resultType="userDto" parameterType="String">
		select account userName,password passWord from market.`store_account` where account = #{userName}
	</select>
</mapper>

緩存擊穿之布隆過濾器bloom Filter實現方式

java web日誌記錄之spring aop實現方式

Java線程(二):Lock

spring ioc中實例化bean以及依賴注入bean和基本類型屬性簡單實現方案

java多線程之fork/join

POI實現導出Excel數據工具類

Mac下配置sublime實現LaTeX

https://yachay.unat.edu.pe/blog/index.php?comment_area=format_blog&comment_component=blog&comment_co

linux以太網驅動總結