anlp在功能上的擴展主要體現在以下幾個方面:
•關鍵詞提取
•自動摘要
•短語提取
•拼音轉換
•簡繁轉換
•文本推薦
下面是 hanLP分詞器的代碼
注：使用maven依賴
com.hankcs
hanlp
portable-1.3.4
使用了java8進行處理
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import com.hankcs.hanlp.seg.Segment;
import com.hankcs.hanlp.seg.Dijkstra.DijkstraSegment;
import com.hankcs.hanlp.seg.NShort.NShortSegment;
import com.hankcs.hanlp.tokenizer.IndexTokenizer;
import com.hankcs.hanlp.tokenizer.NLPTokenizer;
import com.hankcs.hanlp.tokenizer.SpeedTokenizer;
import com.hankcs.hanlp.tokenizer.StandardTokenizer;
public class HanLPTokenizer {
private static final Segment N_SHORT_SEGMENT = new NShortSegment().enableCustomDictionary(false)
.enablePlaceRecognize(true).enableOrganizationRecognize(true);
private static final Segment DIJKSTRA_SEGMENT = new DijkstraSegment().enableCustomDictionary(false)
.enablePlaceRecognize(true).enableOrganizationRecognize(true);
/**
- 標準分詞
- @param text
- @return
*/
public static List standard(String text) {
List list = new ArrayList();
StandardTokenizer.segment(text).forEach(term -> {
if (StringUtils.isNotBlank(term.word)) {
list.add(term.word);
}
});
return list.stream().distinct().collect(Collectors.toList());
}
/**
- NLP分詞
- @param text
- @return
*/
public static List nlp(String text) {
List list = new ArrayList();
NLPTokenizer.segment(text).forEach(term -> {
if (StringUtils.isNotBlank(term.word)) {
list.add(term.word);
}
});
return list.stream().distinct().collect(Collectors.toList());
}
/**
- 索引分詞
- @param text
- @return
*/
public static List index(String text) {
List list = new ArrayList();
IndexTokenizer.segment(text).forEach(term -> {
if (StringUtils.isNotBlank(term.word)) {
list.add(term.word);
}
});
return list.stream().distinct().collect(Collectors.toList());
}
/**
- 極速詞典分詞
- @param text
- @return
*/
public static List speed(String text) {
List list = new ArrayList();
SpeedTokenizer.segment(text).forEach(term -> {
if (StringUtils.isNotBlank(term.word)) {
list.add(term.word);
}
});
return list;
}
/**
- N-最短路徑分詞
- @param text
- @return
*/
public static List nShort(String text) {
List list = new ArrayList();
N_SHORT_SEGMENT.seg(text).forEach(term -> {
if (StringUtils.isNotBlank(term.word)) {
list.add(term.word);
}
});
return list.stream().distinct().collect(Collectors.toList());
}
/**
- 最短路徑分詞
- @param text
- @return
*/
public static List shortest(String text) {
List list = new ArrayList();
DIJKSTRA_SEGMENT.seg(text).forEach(term -> {
if (StringUtils.isNotBlank(term.word)) {
list.add(term.word);
}
});
return list.stream().distinct().collect(Collectors.toList());
}
public static void main(String[] args) {
String text = "測試勿動12";
System.out.println("標準分詞:" + standard(text));
System.out.println("NLP分詞:" + nlp(text));
System.out.println("索引分詞:" + index(text));
System.out.println("N-最短路徑分詞:" + nShort(text));
System.out.println("最短路徑分詞分詞:" + shortest(text));
System.out.println("極速詞典分詞:" + speed(text));
}
}
文章來源於猴德華的博客