JStorm與springboot(cloud)集成

集成spring boot

本地模式

使用jstorm本地模式topology時，可以是jstorm包含springboot，也可以是springboot 包含jstorm，這不影響jar和topology的運行。所以本地模式下，jstorm與springboot或springcloud如何進行集成，都沒有影響，但是如果將topology提交到集羣去運行時，代碼的工程結構就得是jstorm包含springboot了。

集羣模式

首先是pom.xml配置調整，解決在集羣上運行時衝突：
1､log相關的類包衝突

java.lang.NoSuchMethodError: ch.qos.logback.classic.LoggerContext.removeObject(Ljava/lang/String;)V

2､ 多個defaults.yaml資源衝突

Invalid configuration defaults.yaml:Found multiple defaults.yaml resources. You're probably bundling the Storm jars with your topology jar.

在topology真正開始運行時啓動springboot，解決獲取不到Application以及發佈到spring裏面的bean的問題。

pom.xml配置

• 排除spring包中有關log4j-to-slf4j和logback-classic的依賴
• 將jstorm-core設置爲provided範圍

...

<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter</artifactId>
    <exclusions>
        <exclusion>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-to-slf4j2</artifactId>
        </exclusion>
        <exclusion>
            <groupId>ch.qos.logback</groupId>
            <artifactId>logback-classic</artifactId>
        </exclusion>
    </exclusions>
</dependency>

...

<dependency>
    <groupId>com.alibaba.jstorm</groupId>
    <artifactId>jstorm-core</artifactId>
    <version>2.2.1</version>
    <scope>provided</scope>
</dependency>

....

springboot啓動

由於topology在jstorm集羣中運行時，是會將topology序列化後傳遞到worker上，所以springboot只在nimbus上提交時啓動，意義是不大的，需要在topology運行的worker上也啓動springboot。
主要有以下幾處調整：

1. 調整SpringApplication類
   SpringApplication

   import com.ctrip.framework.apollo.spring.annotation.EnableApolloConfig;
   import com.ctrip.framework.foundation.internals.provider.DefaultServerProvider;
   import lombok.extern.slf4j.Slf4j;
   import org.springframework.boot.WebApplicationType;
   import org.springframework.boot.autoconfigure.SpringBootApplication;
   import org.springframework.boot.autoconfigure.gson.GsonAutoConfiguration;
   import org.springframework.boot.builder.SpringApplicationBuilder;
   import org.springframework.boot.logging.LoggingSystem;
   import org.springframework.context.ConfigurableApplicationContext;
   import org.springframework.core.env.Environment;
   import java.net.InetAddress;
   import java.net.UnknownHostException;
   import java.util.Objects;
   import java.util.concurrent.atomic.AtomicBoolean;
   
   @Slf4j
   @EnableApolloConfig
   @SpringBootApplication(exclude = {GsonAutoConfiguration.class,
           org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration.class})
   public class SpringApplication {
       private static AtomicBoolean springStarted = new AtomicBoolean(false);
   
   
       public synchronized static void runSpring() {
           runSpring(new String[0]);
       }
   
       public synchronized static void runSpring(String[] args) {
           if(!Objects.isNull(SpringUtils.getApplicationContext())){
               return;
           }
           try{
               run(args);
           }catch (UnknownHostException e){
               log.error("啓動springboot時出現異常", e);
           }
       }
   
   
       public synchronized static void run(String[] args) throws UnknownHostException {
           log.info("apollo env is {}", getApolloEnv());
           ConfigurableApplicationContext applicationContext = new SpringApplicationBuilder()
                   .sources(SpringApplication.class)
                   .web(WebApplicationType.NONE)
                   .run(args);
           SpringUtils.setApplicationContext(applicationContext);
           Environment env = applicationContext.getEnvironment();
           log.info(
                   "\n----------------------------------------------------------\n\t"
                           + "Application '{}' is running! Access URLs:\n\t"
                           + "Local: \t\thttp://127.0.0.1:{}\n\t"
                           + "External: \thttp://{}:{}\n----------------------------------------------------------",
                   env.getProperty("spring.application.name"),
                   env.getProperty("server.port"),
                   InetAddress.getLocalHost().getHostAddress(),
                   env.getProperty("server.port"));
   
           String configServerStatus = env.getProperty("configserver.status");
           log.info(
                   "\n----------------------------------------------------------\n\t"
                           + "Config Server: \t{}\n----------------------------------------------------------",
                   configServerStatus == null
                           ? "Not found or not setup for this application"
                           : configServerStatus);
       }
   
   
       private static String getApolloEnv(){
           DefaultServerProvider defaultServerProvider = new DefaultServerProvider();
           defaultServerProvider.initialize();
           return defaultServerProvider.getEnvType();
       }
   }
   

   SpringUtils

   import org.springframework.context.ApplicationContext;
   
   public class SpringUtils {
       static ApplicationContext applicationContext;
   
   
       public static ApplicationContext getApplicationContext() {
           return applicationContext;
       }
   
       public static void setApplicationContext(ApplicationContext applicationContext) {
           SpringUtils.applicationContext = applicationContext;
       }
   
       /**
        * 通過name獲取 Bean
        * @param name
        * @return
        */
       public static Object getBean(String name){
           return getApplicationContext().getBean(name);
   
       }
   
   
       /**
        * 通過class獲取Bean
        * @param clazz
        * @param <T>
        * @return
        */
       public static <T> T getBean(Class<T> clazz){
           return getApplicationContext().getBean(clazz);
       }
   
       /**
        * 通過name,以及Clazz返回指定的Bean
        * @param name
        * @param clazz
        * @param <T>
        * @return
        */
       public static <T> T getBean(String name,Class<T> clazz){
           return getApplicationContext().getBean(name, clazz);
       }
   }
   

2. 調整Jstorm Main-Class的main方法

   public class CalBootstrap {
   
       public static void main(String[] args) throws Exception {
       		//啓動springboot
           SpringApplication.runSpring(args);
           // topologyBuilder 封裝了jstorm topology的創建和提交邏輯
           TopologyBuilder topologyBuilder = SpringUtils.getBean(TopologyBuilder.class);
           topologyBuilder();
           Runtime.getRuntime().addShutdownHook(new Thread(() -> topologyBuilder.destory()));
       }
   }
   

3. 調整Jstorm IRichSpout或IRichBolt的開始和準備方法
   在IRichSpout#open()方法中加入啓動springboot的邏輯

	import backtype.storm.spout.SpoutOutputCollector;
	import backtype.storm.task.TopologyContext;
	import backtype.storm.topology.OutputFieldsDeclarer;
	import backtype.storm.topology.base.BaseRichSpout;
	import lombok.extern.slf4j.Slf4j;

	
	@Slf4j
	public class SourceSpout extends BaseRichSpout {
	
	    private static final long serialVersionUID = 2664953801711903206L;
	
	    private SpoutOutputCollector spoutOutputCollector;
	
	    @Override
	    public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
	    	 	//啓動springboot
	        SpringApplication.runSpring();
	
	        SpoutProperties properties = SpringUtils.getBean(SpoutProperties.class);
			   
	
	        log.info("starting....................{}", kafkaProperties);
			  .......
	    }
	    ...... 
	}

或者在IRichBolt#prepare()中加入啓動springboot的邏輯

	import backtype.storm.task.OutputCollector;
	import backtype.storm.task.TopologyContext;
	import backtype.storm.topology.IRichBolt;
	import backtype.storm.topology.OutputFieldsDeclarer;
	import backtype.storm.tuple.Fields;
	import backtype.storm.tuple.Tuple;
	import backtype.storm.tuple.Values;	
	import java.util.Map;
	import java.util.Objects;
	import java.util.concurrent.TimeUnit;
	
	
	public class CalBolt implements IRichBolt {
	
	    private static final long serialVersionUID = -8234268118517500220L;

	    private OutputCollector outputCollector;
	
	
	    @Override
	    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
	        //啓動springboot
	        SpringApplication.runSpring();
				....
	    }
	    ......
	}

---

以上就基本完成了jstorm和springboot的集成，可以發現，jstorm與springboot的集成的衝突主要在log4j-to-slf4j和logback-classic的依賴衝突，在springboot中排除掉就可以了，這也是網上找到的主流解決方案；但其實，還有另一種更優雅，也更簡單的解決方法，請求下面的<集成spring cloud>。

集成spring cloud

jstorm與spring cloud集成出現的衝突與springboot一樣，也是因爲log相關。原因是spring-cloud-context包中的spring.factories裏面的org.springframework.cloud.bootstrap.LoggingSystemShutdownListener；在啓動時，運行到LoggingSystemShutdownListener.shutdownLogging()方法時會報java.lang.NoSuchMethodError: ch.qos.logback.classic.LoggerContext.removeObject(Ljava/lang/String;)V

解決方法是在啓動springboot前，在系統屬性中設置"org.springframework.boot.logging.LoggingSystem"=“none”，表示springboot不再處理log的邏輯。

public class SpringApplication {
    .....
	
	
    public synchronized static void run(String[] args) throws UnknownHostException {
    		//設置"org.springframework.boot.logging.LoggingSystem"="none"
        System.setProperty(LoggingSystem.SYSTEM_PROPERTY, LoggingSystem.NONE);
        log.info("apollo env is {}", getApolloEnv());
        ConfigurableApplicationContext applicationContext = new SpringApplicationBuilder()
                .sources(SpringApplication.class)
                .web(WebApplicationType.NONE)
                .run(args);
        SpringUtils.setApplicationContext(applicationContext);
        Environment env = applicationContext.getEnvironment();
        log.info(
                "\n----------------------------------------------------------\n\t"
                        + "Application '{}' is running! Access URLs:\n\t"
                        + "Local: \t\thttp://127.0.0.1:{}\n\t"
                        + "External: \thttp://{}:{}\n----------------------------------------------------------",
                env.getProperty("spring.application.name"),
                env.getProperty("server.port"),
                InetAddress.getLocalHost().getHostAddress(),
                env.getProperty("server.port"));
	
        String configServerStatus = env.getProperty("configserver.status");
        log.info(
                "\n----------------------------------------------------------\n\t"
                        + "Config Server: \t{}\n----------------------------------------------------------",
                configServerStatus == null
                        ? "Not found or not setup for this application"
                        : configServerStatus);
    }
}

集成apollo

在pom.xml中添加apollo客戶端依賴

<dependency>
    <groupId>com.ctrip.framework.apollo</groupId>
    <artifactId>apollo-client</artifactId>
    <version>1.2.0</version>
</dependency>

在工程的resources目錄中添加apollo-env.properties文件

dev.meta=http://***.***.**.*:****
fat.meta=http://***.***.**.*:****
uat.meta=http://***.***.**.*:****
pro.meta=http://***.***.**.*:****

在SpringApplication類中添加@EnableApolloConfig註解

@Slf4j
@EnableApolloConfig
@SpringBootApplication(exclude = {GsonAutoConfiguration.class,
	        org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration.class})
public class SpringApplication {
	.......
}

調試時添加-Denv=dev參數
在服務器上運行時，創建文件 /opt/settings/server.properties

env=pro