ElasticSearch學習筆記-Vector向量搜索記錄

在Elasticsearch 7.0中，ES引入了高維向量的字段類型：

dense_vector存儲稠密向量，value是單一的float數值，可以是0、負數或正數，dense_vector數組的最大長度不能超過1024，每個文檔的數組長度可以不同。

sparse_vector存儲稀疏向量，value是單一的float數值，可以是0、負數或正數，sparse_vector存儲的是個非嵌套類型的json對象，key是向量的位置，即integer類型的字符串，範圍[0,65535]。

ElasticSearch版本：elasticsearch-7.3.0

環境準備：

curl -H "Content-Type: application/json" -XPUT 'http://192.168.0.1:9200/article_v1/' -d '
{
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0
  },
  "mappings": {
    "dynamic": "strict",
    "properties": {
      "id": {
        "type": "keyword"
      },
      "title": {
        "analyzer": "ik_smart",
        "type": "text"
      },
      "title_dv": {
        "type": "dense_vector",
        "dims": 200
      },
      "title_sv": {
        "type": "sparse_vector"
      }
    }
  }
}
'

測試驗證代碼：

# -*- coding:utf-8 -*-

import os
import sys
import jieba
import logging
import pymongo
from elasticsearch import Elasticsearch
from elasticsearch.serializer import TextSerializer, JSONSerializer
from gensim.models.doc2vec import TaggedDocument, Doc2Vec

default_encoding = 'utf-8'
if sys.getdefaultencoding() != default_encoding:
    reload(sys)
    sys.setdefaultencoding(default_encoding)

logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO)

# 網上隨便爬取一些新聞存入數據庫
client = pymongo.MongoClient(host='192.168.0.1', port=27017)
db = client['news']

es = Elasticsearch([{'host': '192.168.0.1', 'port': 9200}], timeout=3600)

chinese_stop_words_file = os.path.abspath(os.getcwd() + os.sep + '..' + os.sep + 'static' + os.sep + 'dic' + os.sep + 'chinese_stop_words.txt')
chinese_stop_words = [line.strip() for line in open(chinese_stop_words_file, 'r').readlines()]

total_cut_word_count = 0


# 句子分割
def sentence_segment(sentence):
    global total_cut_word_count
    result = []
    cut_words = jieba.cut(sentence)
    for cut_word in cut_words:
        if cut_word not in chinese_stop_words:
            result.append(cut_word)
            total_cut_word_count += 1
    return result


# 準備語料庫
def prepare_doc_corpus():
    datas = db['netease_ent_news_detail'].find({"create_time": {"$ne": None}}).sort('create_time', pymongo.ASCENDING)
    print datas.count()
    for i, data in enumerate(datas):
        if data['title'] is not None and data['content'] is not None:
            title = str(data['title']).strip()
            yield TaggedDocument(sentence_segment(title), [data['_id']])


# 訓練模型
def train_doc_model():
    corpus = prepare_doc_corpus()
    doc2vec = Doc2Vec(vector_size=200, min_count=2, window=5, workers=4, epochs=20)
    doc2vec.build_vocab(corpus)
    doc2vec.train(corpus, total_examples=doc2vec.corpus_count, epochs=doc2vec.epochs)
    doc2vec.save('doc2vec.model')


def insert_data_to_es():
    datas = db['netease_ent_news_detail'].find({"create_time": {"$ne": None}}).sort('create_time', pymongo.ASCENDING)
    print datas.count()
    doc2vec = Doc2Vec.load('doc2vec.model')
    for data in datas:
        if data['title'] is not None and data['content'] is not None:
            sentence = str(data['title']).strip()
            title_dv = doc2vec.infer_vector(sentence_segment(sentence)).tolist()
            body = {"id": data['_id'], "title": data['title'], "title_dv": title_dv}
            es_result = es.create(index="article_v1", doc_type="_doc",
                id=data['_id'], body=body, ignore=[400, 409])
            print es_result


# cosineSimilarity函數計算給定文檔與索引庫裏文檔的dense_vector相似度
def search_es_dense_vertor_1(sentence):
    doc2vec = Doc2Vec.load('doc2vec.model')
    query_vector = doc2vec.infer_vector(sentence_segment(sentence)).tolist()
    body = {
        "query": {
            "script_score": {
                "query": {
                    "match_all": {}
                },
                "script": {
                    "source": "cosineSimilarity(params.queryVector, doc['title_dv']) + 1",
                    "params": {
                        "queryVector": query_vector
                    }
                }
            }
        },
        "from": 0,
        "size": 5
    }
    result = es.search(index="article_v1", body=body)
    hits = result['hits']['hits']
    for hit in hits:
        source = hit['_source']
        for key, value in source.items():
            print '%s %s' % (key, value)
        print '----------'


# dotProduct函數計算給定文檔與索引庫文檔點積的距離
def search_es_dense_vertor_2(sentence):
    doc2vec = Doc2Vec.load('doc2vec.model')
    query_vector = doc2vec.infer_vector(sentence_segment(sentence)).tolist()
    body = {
        "query": {
            "script_score": {
                "query": {
                    "match_all": {}
                },
                "script": {
                    "source": "dotProduct(params.queryVector, doc['title_dv']) + 1",
                    "params": {
                        "queryVector": query_vector
                    }
                }
            }
        },
        "from": 0,
        "size": 5
    }
    result = es.search(index="article_v1", body=body)
    hits = result['hits']['hits']
    for hit in hits:
        source = hit['_source']
        for key, value in source.items():
            print '%s %s' % (key, value)
        print '----------'