電商推薦系統項目總結

項目架構

項目過程中的數據轉換

注意：圖中RDD後跟的不是元素類型，而是其中元素的含義，如：userId即爲用戶Id，score和rating均爲用戶對商品的打分

離線階段數據轉化流程

實時階段數據轉化流程

部分代碼

商品推薦優先級計算（實時推薦階段）

def computeProductScores(
	simProducts:scala.collection.Map[Int,scala.collection.immutable.Map[Int,Doub
	le]],userRecentlyRatings:Array[(Int,Double)],topSimProducts: Array[Int]): 
	Array[(Int,Double)] ={

  //用於保存每一個待選商品和最近評分的每一個商品的權重得分
  val score = scala.collection.mutable.ArrayBuffer[(Int,Double)]()

  //用於保存每一個商品的增強因子數
  val increMap = scala.collection.mutable.HashMap[Int,Int]()

  //用於保存每一個商品的減弱因子數
  val decreMap = scala.collection.mutable.HashMap[Int,Int]()

  for (topSimProduct <- topSimProducts; userRecentlyRating <- userRecentlyRatings){
    val simScore = getProductsSimScore(simProducts,userRecentlyRating._1,topSimProduct)
    if(simScore > 0.6){
      score += ((topSimProduct, simScore * userRecentlyRating._2 ))
      if(userRecentlyRating._2 > 3){
        increMap(topSimProduct) = increMap.getOrDefault(topSimProduct,0) + 1
      }else{
        decreMap(topSimProduct) = decreMap.getOrDefault(topSimProduct,0) + 1
      }
    }
  }

  score.groupBy(_._1).map{case (productId,sims) =>
    (productId,sims.map(_._2).sum / sims.length + log(increMap.getOrDefault(productId, 1)) - log(decreMap.getOrDefault(productId, 1)))
  }.toArray.sortWith(_._2>_._2)

}

自定義攔截器

package flume.interceptor;

import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;

import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;

public class LogETLInterceptor implements Interceptor {

    @Override
    public void initialize() {

    }

    @Override
    public Event intercept(Event event) {

        // 1 獲取數據
        byte[] body = event.getBody();
        String log = new String(body, Charset.forName("UTF-8"));

        if (log.contains("PRODUCT_RATING_PREFIX:")) {
       		newBody = log.split(":")[1].trim()
	        event.setBody(newBody.getBytes())
	        return event;   
        }else {
        	return null;
        }
    }

    @Override
    public List<Event> intercept(List<Event> events) {

        ArrayList<Event> interceptors = new ArrayList<>();

        for (Event event : events) {
            Event intercept1 = intercept(event);

            if (intercept1 != null){
                interceptors.add(intercept1);
            }
        }

        return interceptors;
    }

    @Override
    public void close() {

    }

    public static class Builder implements Interceptor.Builder{

        @Override
        public Interceptor build() {
            return new LogETLInterceptor();
        }

        @Override
        public void configure(Context context) {

        }
    }
}