- package com.jadyer.lucene;
- import java.io.IOException;
- import java.io.StringReader;
- import org.apache.lucene.analysis.Analyzer;
- import org.apache.lucene.analysis.TokenStream;
- import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
- import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
- import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
- import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
- /**
- * 【Lucene3.6.2入門系列】第05節_自定義分詞器
- * @see -----------------------------------------------------------------------------------------------------------------------
- * @see Lucene3.5推薦的四大分詞器:SimpleAnalyzer,StopAnalyzer,WhitespaceAnalyzer,StandardAnalyzer
- * @see 這四大分詞器有一個共同的抽象父類,此類有個方法public final TokenStream tokenStream(),即分詞的一個流
- * @see 假設有這樣的文本"how are you thank you",實際它是以一個java.io.Reader傳進分詞器中
- * @see Lucene分詞器處理完畢後,會把整個分詞轉換爲TokenStream,這個TokenStream中就保存所有的分詞信息
- * @see TokenStream有兩個實現類,分別爲Tokenizer和TokenFilter
- * @see Tokenizer---->用於將一組數據劃分爲獨立的語彙單元(即一個一個的單詞)
- * @see TokenFilter-->過濾語彙單元
- * @see -----------------------------------------------------------------------------------------------------------------------
- * @see 分詞流程
- * @see 1)將一組數據流java.io.Reader交給Tokenizer,由其將數據轉換爲一個個的語彙單元
- * @see 2)通過大量的TokenFilter對已經分好詞的數據進行過濾操作,最後產生TokenStream
- * @see 3)通過TokenStream完成索引的存儲
- * @see -----------------------------------------------------------------------------------------------------------------------
- * @see Tokenizer的一些子類
- * @see KeywordTokenizer-----不分詞,傳什麼就索引什麼
- * @see StandardTokenizer----標準分詞,它有一些較智能的分詞操作,諸如將'[email protected]'中的'yeah.net'當作一個分詞流
- * @see CharTokenizer--------針對字符進行控制的,它還有兩個子類WhitespaceTokenizer和LetterTokenizer
- * @see WhitespaceTokenizer--使用空格進行分詞,諸如將'Thank you,I am jadyer'會被分爲4個詞
- * @see LetterTokenizer------基於文本單詞的分詞,它會根據標點符號來分詞,諸如將'Thank you,I am jadyer'會被分爲5個詞
- * @see LowerCaseTokenizer---它是LetterTokenizer的子類,它會將數據轉爲小寫並分詞
- * @see -----------------------------------------------------------------------------------------------------------------------
- * @see TokenFilter的一些子類
- * @see StopFilter--------它會停用一些語彙單元
- * @see LowerCaseFilter---將數據轉換爲小寫
- * @see StandardFilter----對標準輸出流做一些控制
- * @see PorterStemFilter--還原一些數據,比如將coming還原爲come,將countries還原爲country
- * @see -----------------------------------------------------------------------------------------------------------------------
- * @see eg:'how are you thank you'會被分詞爲'how','are','you','thank','you'合計5個語彙單元
- * @see 那麼應該保存什麼東西,才能使以後在需要還原數據時保證正確的還原呢???其實主要保存三個東西,如下所示
- * @see CharTermAttribute(Lucene3.5以前叫TermAttribute),OffsetAttribute,PositionIncrementAttribute
- * @see 1)CharTermAttribute-----------保存相應的詞彙,這裏保存的就是'how','are','you','thank','you'
- * @see 2)OffsetAttribute-------------保存各詞彙之間的偏移量(大致理解爲順序),比如'how'的首尾字母偏移量爲0和3,'are'爲4和7,'thank'爲12和17
- * @see 3)PositionIncrementAttribute--保存詞與詞之間的位置增量,比如'how'和'are'增量爲1,'are'和'you'之間的也是1,'you'和'thank'的也是1
- * @see 但假設'are'是停用詞(StopFilter的效果),那麼'how'和'you'之間的位置增量就變成了2
- * @see 當我們查找某一個元素時,Lucene會先通過位置增量來取這個元素,但如果兩個詞的位置增量相同,會發生什麼情況呢
- * @see 假設還有一個單詞'this',它的位置增量和'how'是相同的,那麼當我們在界面中搜索'this'時
- * @see 也會搜到'how are you thank you',這樣就可以有效的做同義詞了,目前非常流行的一個叫做WordNet的東西,就可以做同義詞的搜索
- * @see -----------------------------------------------------------------------------------------------------------------------
- * @create Aug 4, 2013 5:48:25 PM
- * @author 玄玉<http://blog.csdn.net/jadyer>
- */
- public class HelloCustomAnalyzer {
- /**
- * 查看分詞信息
- * @see TokenStream還有兩個屬性,分別爲FlagsAttribute和PayloadAttribute,都是開發時用的
- * @see FlagsAttribute----標註位屬性
- * @see PayloadAttribute--做負載的屬性,用來檢測是否已超過負載,超過則可以決定是否停止搜索等等
- * @param txt 待分詞的字符串
- * @param analyzer 所使用的分詞器
- * @param displayAll 是否顯示所有的分詞信息
- */
- public static void displayTokenInfo(String txt, Analyzer analyzer, boolean displayAll){
- //第一個參數沒有任何意義,可以隨便傳一個值,它只是爲了顯示分詞
- //這裏就是使用指定的分詞器將'txt'分詞,分詞後會產生一個TokenStream(可將分詞後的每個單詞理解爲一個Token)
- TokenStream stream = analyzer.tokenStream("此參數無意義", new StringReader(txt));
- //用於查看每一個語彙單元的信息,即分詞的每一個元素
- //這裏創建的屬性會被添加到TokenStream流中,並隨着TokenStream而增加(此屬性就是用來裝載每個Token的,即分詞後的每個單詞)
- //當調用TokenStream.incrementToken()時,就會指向到這個單詞流中的第一個單詞,即此屬性代表的就是分詞後的第一個單詞
- //可以形象的理解成一隻碗,用來盛放TokenStream中每個單詞的碗,每調用一次incrementToken()後,這個碗就會盛放流中的下一個單詞
- CharTermAttribute cta = stream.addAttribute(CharTermAttribute.class);
- //用於查看位置增量(指的是語彙單元之間的距離,可理解爲元素與元素之間的空格,即間隔的單元數)
- PositionIncrementAttribute pia = stream.addAttribute(PositionIncrementAttribute.class);
- //用於查看每個語彙單元的偏移量
- OffsetAttribute oa = stream.addAttribute(OffsetAttribute.class);
- //用於查看使用的分詞器的類型信息
- TypeAttribute ta = stream.addAttribute(TypeAttribute.class);
- try {
- if(displayAll){
- //等價於while(stream.incrementToken())
- for(; stream.incrementToken() ;){
- System.out.println(ta.type() + " " + pia.getPositionIncrement() + " ["+oa.startOffset()+"-"+oa.endOffset()+"] ["+cta+"]");
- }
- }else{
- System.out.println();
- while(stream.incrementToken()){
- System.out.print("[" + cta + "]");
- }
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
下面是自定義的停用詞分詞器MyStopAnalyzer.java
- package com.jadyer.analysis;
- import java.io.Reader;
- import java.util.Set;
- import org.apache.lucene.analysis.Analyzer;
- import org.apache.lucene.analysis.LetterTokenizer;
- import org.apache.lucene.analysis.LowerCaseFilter;
- import org.apache.lucene.analysis.StopAnalyzer;
- import org.apache.lucene.analysis.StopFilter;
- import org.apache.lucene.analysis.TokenStream;
- import org.apache.lucene.util.Version;
- /**
- * 自定義的停用詞分詞器
- * @see 它主要用來過濾指定的字符串(忽略大小寫)
- * @create Aug 5, 2013 1:55:15 PM
- * @author 玄玉<http://blog.csdn.net/jadyer>
- */
- public class MyStopAnalyzer extends Analyzer {
- private Set<Object> stopWords; //存放停用的分詞信息
- /**
- * 自定義的用於過濾指定字符串的分詞器
- * @param _stopWords 用於指定所要過濾的字符串(忽略大小寫)
- */
- public MyStopAnalyzer(String[] _stopWords){
- //會自動將字符串數組轉換爲Set
- stopWords = StopFilter.makeStopSet(Version.LUCENE_36, _stopWords, true);
- //將原有的停用詞加入到現在的停用詞中
- stopWords.addAll(StopAnalyzer.ENGLISH_STOP_WORDS_SET);
- }
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- //爲這個分詞器設定過濾器鏈和Tokenizer
- return new StopFilter(Version.LUCENE_36,
- //這裏就可以存放很多的TokenFilter
- new LowerCaseFilter(Version.LUCENE_36, new LetterTokenizer(Version.LUCENE_36, reader)),
- stopWords);
- }
- }
下面是自定義的同義詞分詞器MySynonymAnalyzer.java
- package com.jadyer.analysis;
- import java.io.IOException;
- import java.io.Reader;
- import java.util.HashMap;
- import java.util.Map;
- import java.util.Stack;
- import org.apache.lucene.analysis.Analyzer;
- import org.apache.lucene.analysis.TokenFilter;
- import org.apache.lucene.analysis.TokenStream;
- import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
- import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
- import org.apache.lucene.util.AttributeSource;
- import com.chenlb.mmseg4j.ComplexSeg;
- import com.chenlb.mmseg4j.Dictionary;
- import com.chenlb.mmseg4j.analysis.MMSegTokenizer;
- /**
- * 自定義的同義詞分詞器
- * @create Aug 5, 2013 5:11:46 PM
- * @author 玄玉<http://blog.csdn.net/jadyer>
- */
- public class MySynonymAnalyzer extends Analyzer {
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- //藉助MMSeg4j實現自定義分詞器,寫法參考MMSegAnalyzer類的tokenStream()方法
- //但爲了過濾並處理分詞後的各個語彙單元,以達到同義詞分詞器的功能,故自定義一個TokenFilter
- //實際執行流程就是字符串的Reader首先進入MMSegTokenizer,由其進行分詞,分詞完畢後進入自定義的MySynonymTokenFilter
- //然後在MySynonymTokenFilter中添加同義詞
- return new MySynonymTokenFilter(new MMSegTokenizer(new ComplexSeg(Dictionary.getInstance()), reader));
- }
- }
- /**
- * 自定義的TokenFilter
- * @create Aug 5, 2013 5:11:58 PM
- * @author 玄玉<http://blog.csdn.net/jadyer>
- */
- class MySynonymTokenFilter extends TokenFilter {
- private CharTermAttribute cta; //用於獲取TokenStream中的語彙單元
- private PositionIncrementAttribute pia; //用於獲取TokenStream中的位置增量
- private AttributeSource.State tokenState; //用於保存語彙單元的狀態
- private Stack<String> synonymStack; //用於保存同義詞
- protected MySynonymTokenFilter(TokenStream input) {
- super(input);
- this.cta = this.addAttribute(CharTermAttribute.class);
- this.pia = this.addAttribute(PositionIncrementAttribute.class);
- this.synonymStack = new Stack<String>();
- }
- /**
- * 判斷是否存在同義詞
- */
- private boolean isHaveSynonym(String name){
- //先定義同義詞的詞典
- Map<String, String[]> synonymMap = new HashMap<String, String[]>();
- synonymMap.put("我", new String[]{"咱", "俺"});
- synonymMap.put("中國", new String[]{"兲朝", "大陸"});
- if(synonymMap.containsKey(name)){
- for(String str : synonymMap.get(name)){
- this.synonymStack.push(str);
- }
- return true;
- }
- return false;
- }
- @Override
- public boolean incrementToken() throws IOException {
- while(this.synonymStack.size() > 0){
- restoreState(this.tokenState); //將狀態還原爲上一個元素的狀態
- cta.setEmpty();
- cta.append(this.synonymStack.pop()); //獲取並追加同義詞
- pia.setPositionIncrement(0); //設置位置增量爲0
- return true;
- }
- if(input.incrementToken()){
- //注意:當發現當前元素存在同義詞之後,不能立即追加同義詞,即不能在目標元素上直接處理
- if(this.isHaveSynonym(cta.toString())){
- this.tokenState = captureState(); //存在同義詞時,則捕獲並保存當前狀態
- }
- return true;
- }else {
- return false; //只要TokenStream中沒有元素,就返回false
- }
- }
- }
最後是JUnit4.x編寫的小測試
- package com.jadyer.test;
- import org.apache.lucene.analysis.StopAnalyzer;
- import org.apache.lucene.analysis.standard.StandardAnalyzer;
- import org.apache.lucene.document.Document;
- import org.apache.lucene.document.Field;
- import org.apache.lucene.index.IndexReader;
- import org.apache.lucene.index.IndexWriter;
- import org.apache.lucene.index.IndexWriterConfig;
- import org.apache.lucene.index.Term;
- import org.apache.lucene.search.IndexSearcher;
- import org.apache.lucene.search.ScoreDoc;
- import org.apache.lucene.search.TermQuery;
- import org.apache.lucene.search.TopDocs;
- import org.apache.lucene.store.Directory;
- import org.apache.lucene.store.RAMDirectory;
- import org.apache.lucene.util.Version;
- import org.junit.Test;
- import com.jadyer.analysis.MyStopAnalyzer;
- import com.jadyer.analysis.MySynonymAnalyzer;
- import com.jadyer.lucene.HelloCustomAnalyzer;
- public class HelloCustomAnalyzerTest {
- /**
- * 測試自定義的用於過濾指定字符串(忽略大小寫)的停用詞分詞器
- */
- @Test
- public void stopAnalyzer(){
- String txt = "This is my house, I`m come from Haerbin,My email is [email protected], My QQ is 517751422";
- HelloCustomAnalyzer.displayTokenInfo(txt, new StandardAnalyzer(Version.LUCENE_36), false);
- HelloCustomAnalyzer.displayTokenInfo(txt, new StopAnalyzer(Version.LUCENE_36), false);
- HelloCustomAnalyzer.displayTokenInfo(txt, new MyStopAnalyzer(new String[]{"I", "EMAIL", "you"}), false);
- }
- /**
- * 測試自定義的同義詞分詞器
- */
- @Test
- public void synonymAnalyzer(){
- String txt = "我來自中國黑龍江省哈爾濱市巴彥縣興隆鎮";
- IndexWriter writer = null;
- IndexSearcher searcher = null;
- Directory directory = new RAMDirectory();
- try {
- writer = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_36, new MySynonymAnalyzer()));
- Document doc = new Document();
- doc.add(new Field("content", txt, Field.Store.YES, Field.Index.ANALYZED));
- writer.addDocument(doc);
- writer.close(); //搜索前要確保IndexWriter已關閉,否則會報告異常org.apache.lucene.index.IndexNotFoundException: no segments* file found
- searcher = new IndexSearcher(IndexReader.open(directory));
- TopDocs tds = searcher.search(new TermQuery(new Term("content", "咱")), 10);
- for(ScoreDoc sd : tds.scoreDocs){
- System.out.println(searcher.doc(sd.doc).get("content"));
- }
- searcher.close();
- } catch (Exception e) {
- e.printStackTrace();
- }
- HelloCustomAnalyzer.displayTokenInfo(txt, new MySynonymAnalyzer(), true);
- }
- }