Elasticsearch流程設計

一、控制器層的更新、添加、刪除

class AddKnowledgeAction extends CAction {
    //add and  update
    public function actionPost() { 
      if ($_POST) { //如果是post操作
          $res = array('code'=>0,'message'=>'');
          $kid = Yii::app()->request->getPost('kid');   //這裏是知識主鍵id
          $cid = Yii::app()->request->getPost('cid');
          $title = Yii::app()->request->getPost('title');
          $content = Yii::app()->request->getPost('content');
          $auth_group = Yii::app()->request->getPost('auth_group');
          $end_time = Yii::app()->request->getPost('end_time');
          $keywords = Yii::app()->request->getPost('keywords');
 
          if (empty($kid))  {  //$kid不存在則說明是走add操作，否則是update
             if ($kid = CallCenterService::addKnowledge($cid, $title, $content, $auth_group, $end_time, $keywords)) {
                //這裏表示添加成功
             }
          } else {
              if (CallCenterService::updateKnowledge($kid, $cid, $title, $content, $auth_group, $end_time, $keywords)) {
                //這裏表示修改成功
              }
          }
      }      
    }
    
    //delete
    public function actionDelete (){
        $kid = Yii::app()->request->getQuery('kid');
        $action = Yii::app()->request->getQuery('action'); //action => ['delete' => '物理刪除', 'Invalid' => '邏輯刪除']if ($kid && $action) {
            if (CallCenterService::delKnowledge($kid, $action)) {
                //表示刪除成功
            }
        } 
    }
}

二、服務層的更新、添加、刪除

//服務層
class CallCenterService {

    private static $instance;

    public static $auth_group = null;

    public static function getInstance() {
       if (empty(self::$instance)) {
            self::$instance = new CallCenterService();
        }
        return self::$instance;
    }

     /**
     * 添加知識
     */
    public static function addKnowledge($cid, $title, $content, $auth_group, $end_time, $keywords) {
        $model = new Knowledgenew;
        $operator = Yii::app()->user->id;
        $created = date('Y-m-d H:i:s');
        $model->attributes = array(
            'cid' => $cid,
            'title' => $title,
            'content' => $content,
            'operator' => $operator,
            'created' => $created,
            'auth_group' => $auth_group,
            'end_time' => $end_time,
            'keywords' => $keywords,
            'updated' => $created
        );


        if ($model->save()) {
            $id = $model->id;
            //異步添加索引到es
            Knowledgenew::onKnowledgeChanged('add', array('id' => $id));
            return $id;
        } else {
        }
        return false;
    }

    /**
     * 編輯知識
     */
    public static function updateKnowledge($kid, $cid, $title, $content , $auth_group, $end_time, $keywords) {

        $knowledge = Knowledgenew::getKnowledge($kid);
        if ($knowledge) {
            $model = new Knowledgenew;
            $model->updateByPk($kid,
                array(
                    'cid' => $cid,
                    'title' => $title,
                    'content' => $content,
                    'auth_group' => $auth_group,
                    'end_time' => isset($end_time) && !empty($end_time) ? $end_time : null,
                    'keywords' => $keywords,
                    'updated' => date('Y-m-d H:i:s')
                )
            );
            Knowledgenew::onKnowledgeChanged('update', array('id' => $kid));
            return true;
        }
        return false;
    }


    /**刪除一條知識
     * @param $kid
     * @param string $action  Invalid => 邏輯刪除 ,delete =>物理刪除
     * @return bool
     */
    public static function delKnowledge($kid, $action = 'invalid') {
        $knowledge = Knowledgenew::getKnowledge($kid);

        if ($knowledge) {
            $model = new Knowledgenew;
            if ($action == 'delete') {
                $model->deleteByPk($kid);
            } else {
                $model->updateByPk($kid,array('status'=>Knowledgenew::STATUS_DEL));
            }
            //更新es
            Knowledgenew::onKnowledgeChanged('delete', array('id' => $kid));
            //刪除收藏夾中的相關知識
            KnowledgenewCollection::model()->deleteAll("kid = $kid");
            return true;
        }
        return false;
    }

}

三、Model層的更新點擊瀏覽次數場景及異步任務更新ES信息

//model層
class Knowledgenew extends CActiveRecord
{
    const STATUS_NORMAL = 1;
    const STATUS_DEL = 2; //Invalid


  public function tableName()
  {
    return '{{knowledgenew}}';
  }


  public static function model($className=__CLASS__)
  {
    return parent::model($className);
  }


    /**
     * 增加瀏覽數
     */
    public static function addClickNum($kid) {
        $model = self::model();
        $model->updateCounters(array('click_num'=>1),'id=:id',array(':id'=>$kid));
        Knowledgenew::onKnowledgeChanged('update', array('id' => $kid));
        return true;
    }


    //更新es信息
    public static function onKnowledgeChanged($action, $param){
       //echo '更新知識庫索引action='.$action.PHP_EOL;
        EdjLog::info('更新知識庫索引action='.$action);
        $base_param = array('es_source' => 'Knowledgenew', 'es_action' => $action);
        Queue::model()->putin( //異步
            array(
                'method'=>'synchronize_elasticsearch',
                'params'=>array_merge($base_param, $param)
            ),
            'synchronize_elasticsearch'
        );
    }
}

四、異步Job隊列生產

<?php
/**
 * 基於redis的queue隊列
 */
class Queue {
    private static $_models;

    public $queue_max_length = array(
    );

    public static function model($className=__CLASS__) {
        $model=null;
        if (isset(self::$_models[$className]))
            $model=self::$_models[$className];
        else {
            $model=self::$_models[$className]=new $className(null);
        }
        return $model;
    }

    //確定redis
    private function select_redis($type) {
        return QueuePool::model()->get_zone($type);
    }

    private function trim($queue_name) {

        $type = str_replace("queue_", "", $queue_name);
        $max = 0;
        if (isset($this->queue_max_length[$type])) {
            $max = intval($this->queue_max_length[$type]);
        }
        if ($max>0) {
            $zone = $this->select_redis($type);
            if($zone) {
                $zone['redis']->lTrim($queue_name, 0, $max-1);
            }
            else {
                EdjLog::error("can not find zone, queue name: " . $type);
                return;
            }
        }
    }

    /**
     * 放入隊列，統一隊列對外暴露方法,增加類型默認放task隊列，指定了就放對應的隊列，同時如果不在指定類型內的，也放默認隊列
     *
     * @param unknown_type $params
     * @param unknown_type $type
     * @return mixed
     */
    public function putin($params=null, $type){
        $type = empty($type) ? 'error' : strtolower($type);

                $base_qname = QNameManagerService::model()->get_base_qname($type);

        if(!empty($base_qname)) {
            $this->queue_name = 'queue_'.$base_qname;
        }else{
            $this->queue_name = 'queue_error';
        }

        if ($params===null) {
            return $this->get();
        } else {
            return $this->add($params);  //如果add替換爲processTask方法，則同步
        }
    }

    /**
     * 取一條隊列數據,封裝多個隊列，統一調用方法
     * @param string $type
     * @return array
     */
    public function getit($type='default')
    {
        $base_qname = QNameManagerService::model()->get_base_qname($type);

        if(!empty($base_qname)) {
            $this->queue_name = 'queue_'.$base_qname;
        }else{
            return array();
        }

        $zone = $this->select_redis($type);
        if($zone) {
            if($zone['brpop']) {
                $json = '';
                $result = $zone['redis']->brPop($this->queue_name, $zone['brpop']);
                if(!empty($result) && isset($result[1])) {
                    $json = $result[1];
                }
            }
            else {
                $json = $zone['redis']->rPop($this->queue_name);
            }
        }
        else {
            EdjLog::error("can not find zone, queue name: " . $type);
            return array();
        }

        return json_decode($json, true);
    }

    /**
     * 返回隊列接收的類型列表
     * @return array
     */
    public function getQueueTypeList()
    {
        $list = QNameManager::model()->findall();
        if($list) {
            return $list;
        }

        EdjLog::error("Error: get queue list from database");
        return array();
    }

    /**
     * 設置或者讀取位置隊列
     * @param array $params
     * @return mixed
     */
    public function position($params=null) {
        $this->queue_name='queue_position';

        if ($params===null) {
            return $this->get();
        } else {
            return $this->add($params);
        }
    }

    /**
     * 心跳隊列
     * @param string $params
     * @return mixed
     */
    public function heartbeat($params=null) {
        $this->queue_name='queue_heartbeat';

        if ($params===null) {
            return $this->get();
        } else {
            return $this->add($params);
        }
    }

    /**
     * 最高優先級隊列
     * @param string $params
     * @return mixed
     */
    public function task($params=null) {
        $this->queue_name='queue_task';

        if ($params===null) {
            return $this->get();
        } else {
            return $this->add($params);
        }
    }

    /**
     * 保存日誌到數據庫
     * @param string $params
     * @return mixed
     */
    public function dumplog($params=null) {
        $this->queue_name='queue_dumplog';

        if ($params===null) {
            return $this->get();
        } else {
            return $this->add($params);
        }
    }

    /**
     * 返回各個隊列中的任務總數
     */
    public function length() {

        $queue = $this->getQueueTypeList();

        $queue_length=array();
        $reg = "/P[0-9]+$/";
        foreach($queue as $item) {
            $base_qname = $item->base_qname;
            $zone = $this->select_redis($base_qname);
            $key = 'queue_'.$base_qname;
            if($zone) {
                $len = $zone['redis']->lLen($key);
                if(isset($item->max) && $len > $item->max) {
                    $key = '!'.$key;
                }

                $pkey = '';
                if(preg_match($reg, $zone['name'])) {
                    $pkey = $key.'@'.$zone['name'];
                }
                else {
                    $pkey = $key.'@'.$zone['name']."_P".$item->level;
                }

                $queue_length[$pkey] = $len;
            }
            else {
                EdjLog::error("can not find zone, queue name: " . $key);
            }
        }

        return $queue_length;
    }

    private function get() {
        $type = str_replace("queue_", "", $this->queue_name);
        $zone = $this->select_redis($type);
        if($zone) {
            if($zone['brpop']) {
                $json = '';
                $result = $zone['redis']->brPop($this->queue_name, $zone['brpop']);
                if(!empty($result) && isset($result[1])) {
                    $json = $result[1];
                }
            }
            else {
                $json = $zone['redis']->rPop($this->queue_name);
            }
        }
        else {
            EdjLog::error("can not find zone, queue name: " . $type);
            return array();
        }
        return json_decode($json, true);
    }

    private function add($params) {
        $json=json_encode($params);
        $type = str_replace("queue_", "", $this->queue_name);
        $zone = $this->select_redis($type);
        $return = 0;
        if($zone) {
            try {
                $return = $zone['redis']->lPush($this->queue_name, $json);
            } catch (Exception $e) {
                EdjLog::error("write redis error,msg:".$e->getMessage());
                //echo $e->getMessage();
            }
        }
        else {
            EdjLog::error("can not find zone, queue name: " . $type);
        }

        return $return;
    }
 
    //如果add 替換爲此方法，則同步
    public function processTask($task) {
        if(!isset($task['method'], $task['params'])) {
            $task_content = json_encode($task);
            EdjLog::error("can not run task due to no 'method' or 'params' specified, task is $task_content");
            return;
        }

        $method=$task['method'];
        $params=$task['params'];
        $class = isset($task['class']) ? $task['class'] : "QueueProcess";
        EdjLog::info("REDIS_QUEUE_OUT CLASS:$class METHOD:$method PARAMS:".json_encode($params));
        try {
            //throw new Exception("Value must be 1 or below");
            $queue_process=new $class();
            // check this method is exist, if not throw ReflectionException
            new ReflectionMethod($queue_process, $method);
            call_user_func_array(array($queue_process, $method), array($params));
        } catch(Exception $e) {
            $errmsg = $e->getMessage();
            EdjLog::error("execption queue_run method:$method err: $errmsg");
        }
    }

    public function getLengthByType($type){
        $type = empty($type) ? 'error' : strtolower($type);
        $base_qname = QNameManagerService::model()->get_base_qname($type);
        $zone = $this->select_redis($base_qname);
        $key = 'queue_'.$base_qname;
        $len = 0;
        if($zone) {
            $len = $zone['redis']->lLen($key);
        } else {
            EdjLog::error("can not find zone, queue name: " . $base_qname);
        }
        return $len;
    }
}

五、異步Job隊列消費

<?php
/**
 * 隊列處理
 */
Yii::import("application.ecenter.service.HttpUtils");
class QueueProcess {
    private static $_models;
    private $message;

    public static function model($className=__CLASS__) {
        $model=null;
        if (isset(self::$_models[$className]))
            $model=self::$_models[$className];
        else {
            $model=self::$_models[$className]=new $className(null);
        }
        return $model;
    }


    public function synchronize_elasticsearch($param)
    {
        if (empty($param) || !isset($param['es_source'], $param['es_action'])) {
            return false;
        }

        $class_name = $param['es_source'].'Synchronizer';
        $method_name = $param['es_action'];
        if (class_exists($class_name) && method_exists($class_name, $method_name)) {
            unset($param['es_source']);
            unset($param['es_action']);
            call_user_func(array($class_name, $method_name), $param);
        } else {
            EdjLog::error('synchronize method does not exist. class name '.$class_name.' method name '.$method_name);
        }
    }

}

六、ES信息處理操作服務層

<?php
/**
 * Created by PhpStorm.
 */

class KnowledgenewSynchronizer {

    static public $index = 'knowledge_index';
    static public $type = 'knowledge';
    static public $filed = ' id, keywords, title, content, auth_group, cid, operator, click_num, status, created, updated ';


    static public function add($param)
    {
        if (empty($param) || !isset($param['id'])) {
            return false;
        }
        $id = $param['id'];
        $sql = "select".self::$filed."from t_knowledgenew where id=:id";
        $doc = Yii::app()->db->CreateCommand($sql)->queryRow(true,array('id'=>$id));

        if (empty($doc)) {
            EdjLog::error('cannot find knowledge with id: '.$id);
            return false;
        }

        return ElasticsearchSynchronizer::addDocument(self::$index, self::$type, $id, $doc);
    }

    static public function delete($param)
    {
        if (empty($param) || !isset($param['id'])) {
            return false;
        }
        $id = $param['id'];
        return ElasticsearchSynchronizer::deleteDocument(self::$index, self::$type, $id);
    }

    static public function update($param)
    {
        if (empty($param) || !isset($param['id'])) {
            return false;
        }
        $id = $param['id'];

        $sql = "select".self::$filed."from t_knowledgenew where id=:id";
        $doc = Yii::app()->db->CreateCommand($sql)->queryRow(true,array('id'=>$id));

        if (empty($doc)) {
            EdjLog::error('cannot find knowledge with id: '.$id);
            return false;
        }

        return ElasticsearchSynchronizer::updateDocument(self::$index, self::$type, $id, $doc);
    }

}

七、ES信息處理操作Model層

<?php

use Elastica\Client;
use Elastica\Query\QueryString;
use Elastica\Query;
use Elastica\Document;

Class ElasticsearchSynchronizer
{//測試
    //const ES_HOST='search.n.edaijia.cn';
    //const ES_PORT=9200;

    static public function addDocument($index, $type, $id, $doc)
    {
        $client = new Client(array('host' => self::ES_HOST, self::ES_PORT));
        $type = $client->getIndex($index)->getType($type);
        try {
            $response = $type->addDocument(new Document($id, $doc));
            if ($response->isOk()) {
                EdjLog::info("add document $id succeeded");
                return true;
            } else {
                EdjLog::info("add document $id failed");
                return false;
            }
        } catch (Exception $e) {
            print_r($e);
            EdjLog::error("add document $id failed with exception ".$e->getMessage());
            return false;
        }
    }

    static public function updateDocument($index, $type, $id, $doc)
    {
        $client = new Client(array('host' => self::ES_HOST, 'port' => self::ES_PORT));
        try {
            $response = $client->getIndex($index)
                ->getType($type)
                ->updateDocument(new Document($id, $doc));

            if ($response->isOk()) {
                EdjLog::info("update document $id succeeded");
                return true;
            } else {
                EdjLog::info("update document $id failed");
                return false;
            }
        } catch (Exception $e) {
            EdjLog::error("update document $id failed with exception ".$e->getMessage());
            return false;
        }
    }

    static public function deleteDocument($index, $type, $id)
    {
        $client = new Client(array('host' => self::ES_HOST, 'port' => self::ES_PORT));
        try {
            $response = $client->getIndex($index)->getType($type)->deleteById($id);
            if ($response->isOk()) {
                EdjLog::info("delete document $id succeeded");
                return true;
            } else {
                EdjLog::info("delete document $id failed");
                return false;
            }
        } catch (Exception $e) {
            EdjLog::error("delete document $id failed with exception ".$e->getMessage());
            return false;
        }
    }

}

 八、查詢

 /**
     * @param $keyword
     * @param int $page
     * @param int $size
     * @param str $search_type
     * 搜索知識
     * 搜索標題和內容，多個關鍵詞是並且關係，空格分隔
     */
    public static function searchKnowledge($keyword, $page = 0, $size = 50, $search_type = 'default') {
        //對搜索關鍵詞空格隔開
//        $keywords = explode(' ',trim($keyword));
        $start = $page * $size;
        $client = new \Elastica\Client(array('host' => ElasticsearchSynchronizer::ES_HOST, 'port' => ElasticsearchSynchronizer::ES_PORT));//更改成線上主機和端口
        $search = new \Elastica\Search($client);
        $search ->addIndex(KnowledgenewSynchronizer::$index)->addType(KnowledgenewSynchronizer::$type);
//        $query = new \Elastica\Query\Bool();
        $query = new \Elastica\Query();
        //設置必要查詢
//        foreach($keywords as $word) {
//            if($word) {
//                $knowledge_query = new \Elastica\Query\QueryString();
//                $knowledge_query->setFields(array('title', 'content'));
//                $knowledge_query->setQuery('"' . $word . '"');
//                $query->addMust($knowledge_query);
//            }
//        }
        $MultiMatch_obj = new \Elastica\Query\MultiMatch();
        $MultiMatch_obj->setQuery($keyword);
        if ($search_type == 'default') {
            $MultiMatch_obj->setFields(array('keywords'));
        } else {
            $MultiMatch_obj->setTieBreaker(0.3);
            $MultiMatch_obj->setType('best_fields');
            $MultiMatch_obj->setFields(array('keywords^901209','content'));
            $MultiMatch_obj->setOperator('or');
            //這裏是字符串，在es的擴展目錄下 setMinimumShouldMatch方法把轉int去掉//
            //$this->setParam('minimum_should_match', (int)$minimumShouldMatch);
            $MultiMatch_obj->setMinimumShouldMatch('30%');

        }
        $query->setQuery($MultiMatch_obj); //命中全部紀錄
        $query = \Elastica\Query::create($query);
        $query->setSource(["id","cid","updated", "title", 'keywords',"content",'auth_group','status']);
//        $query->setSort([
//            'click_num' => ['order' => 'desc']
//        ]);


        //設置起始頁
        $query->setFrom($start);
        $query->setSize($size);
        //設置高亮顯示
        $query->setHighlight(array(
            'pre_tags' => array('<span style="color: red">'),
            'post_tags' => array('</span>'),
            'fields' => array(
                'title' => array(
                    'fragment_size' => 5,//包含高亮詞語，最多展示多少個
                    'number_of_fragments' => 1,//分幾段展示，這裏一段就可以
                ),
                'keywords' => array(
                    'fragment_size' => 10,//包含高亮詞語，最多展示多少個
                    'number_of_fragments' => 1,//分幾段展示，這裏一段就可以
                ),
                'content' => array(
                    'fragment_size' => 10,//包含高亮詞語，最多展示多少個
                    'number_of_fragments' => 1,//分幾段展示，這裏一段就可以
                ),
            ),
        ));

        $search->setQuery($query);

        $results = array();
        $totalResults = 0;
        try {
            $resultSet = $search->search();
            $results = $resultSet->getResults();
            $totalResults = $resultSet->getTotalHits();
        } catch (Exception $e) {
            EdjLog::error("query elasticsearch failed");
        }

        if ($totalResults <= 0) {

            return;
        }

        $poi_result = array();
        foreach ($results as $result) {
            $highlight = $result->getHighlights();
            $title_hightlight = isset($highlight['title'][0])?$highlight['title'][0]:'';
            $content_hightlight = isset($highlight['content'][0])?$highlight['content'][0]:'';
            $keywords_highlight = isset($highlight['keywords'][0])?$highlight['keywords'][0]:'';

            $poi_result[] = array(
                'id' => $result->id,
                'cid' => $result->cid,
                'title' => $result->title,
                'keywords' => $result->keywords,
                'content' => $result->content,
                'auth_group' => $result->auth_group,
                'title_highlight'=>$title_hightlight, //高亮展示標題搜索關鍵詞
                'keywords_highlight'=>$keywords_highlight, //高亮展示標題搜索關鍵詞
                'content_highlight'=>$content_hightlight,//高亮展示內容搜索關鍵詞
                'updated'=>$result->updated,
                'status' => $result->status,
            );
        }

        //這裏過濾了用戶非權限列表
        $poi_result = self::filterNoAuthKnowledge($poi_result);
        $totalResults = count($poi_result) ;
        $info = array('totalNum'=>$totalResults,'data'=>$poi_result);
        return json_encode($info);
    }

 九、源碼包

鏈接：https://pan.baidu.com/s/1lVcrb50HSLrJh3zvBOdH5g 
提取碼：9c9c