Python——n-gram实现
目标:给定文本,以及划分的长度n,将文本划分为将长度为n的子文本,列表输出。
例子:
输入:哈哈
切分长度:2
列表输出:['哈哈']
集合输出:{('哈', '哈')}
输入:哈哈哈哈
切分长度:3
列表输出:['哈哈哈', '哈哈哈']
集合输出:{('哈', '哈', '哈')}
输入:唧唧复唧唧
切分长度:3
列表输出:['唧唧复', '唧复唧', '复唧唧']
集合输出:{('唧', '唧', '复'), ('复', '唧', '唧'), ('唧', '复', '唧')}
输入:君不见黄河之水天上来
切分长度:3
列表输出:['君不见', '不见黄', '见黄河', '黄河之', '河之水', '之水天', '水天上', '天上来']
集合输出:{('河', '之', '水'), ('君', '不', '见'), ('不', '见', '黄'), ('黄', '河', '之'), ('见', '黄', '河'), ('天', '上', '来'), ('水', '天', '上'), ('之', '水', '天')}
代码:
#!/usr/bin/env python
# -*- coding:utf-8 -*-
"""
@Time :2019/12/7
@Name :Zhang Wei
@Contact :[email protected]
@File :demo.py
@Software :Pycharm
"""
# 读取csv文件
def load_csv(path, encoding='utf-8', sep="\t"):
with open(path, 'r', encoding=encoding) as file:
lines = file.readlines()
if sep is not None:
lines = [line.replace("\n", "").split(sep) for line in lines]
else:
lines = [line.replace("\n", "") for line in lines]
return lines
# n-gram-list
def create_ngram_list(input_list, ngram_num):
ngram_list = []
if len(input_list) <= ngram_num:
ngram_list.append(input_list)
else:
for tmp in zip(*[input_list[i:] for i in range(ngram_num)]):
tmp = "".join(tmp)
ngram_list.append(tmp)
return ngram_list
# n-gram-set
def create_ngram_set(input_list, ngram_num):
if len(input_list) <= ngram_num:
return {tuple(list(input_list))}
else:
return set(zip(*[input_list[i:] for i in range(ngram_num)]))
# 随机生成实体词,长度为n
def get_entity(words_list, n):
entity = ""
import random
for i in range(n):
c = words_list[random.randint(0, len(words_list) - 1)]
entity += c
return entity
if __name__ == "__main__":
while True:
# demo 1:n-gram切分
text = input("demo 1:\n输入:")
ngram_num = int(input("切分长度:"))
print("\n列表输出:{0}".format(create_ngram_list(text, ngram_num))) # 列表形式
print("集合输出:{0}\n".format(create_ngram_set(text, ngram_num))) # 集合形式
# demo 2:随机生成长度为n的文本
print("demo 2:{0}\n".format(get_entity(
words_list=["我", "是", "四", "川", "人", "我", "爱", "吃", "火", "锅"],
n=3)))
# 是川四
补充:函数create_ngram_set的输入还可以是列表,如:
print("集合输出:{0}\n".format(create_ngram_set([1, 4, 9, 4, 1, 4], 3))) # 集合形式
# 集合输出:{(4, 1, 4), (4, 9, 4), (1, 4, 9), (9, 4, 1)}
但是列表输出会报错,如果希望输出列表,需要对create_ngram_list进行修改。