# 移除文本噪声的示范代码
noise_list = ["is", "a", "this", "..."]
def _remove_noise(input_text):
words = input_text.split()
noise_free_words = [word for word in words if word not in
noise_list]
noise_free_text = " ".join(noise_free_words)
return noise_free_text_remove_noise("this is a sample text")
>>> "sample text"
# 移除固定模式噪声的示例代码
import re
def _remove_regex(input_text, regex_pattern):
urls = re.finditer(regex_pattern, input_text)
for i in urls:
input_text = re.sub(i.group().strip(), '', input_text)
return input_text
regex_pattern = "#[\w]*"
_remove_regex("remove this #hashtag from analytics vidhya", regex_pattern)
>>> "remove this from analytics vidhya"
lookup_dict = {'rt':'Retweet', 'dm':'direct message', "awsm" : "awesome", "luv" :"love", "..."}
def _lookup_words(input_text):
words = input_text.split()
new_words = []
for word in words:
if word.lower() in lookup_dict:
word = lookup_dict[word.lower()]
new_words.append(word)
new_text = " ".join(new_words)
return new_text
_lookup_words("RT this is a retweeted tweet by Shivam Bansal")
>> "Retweet this is a retweeted tweet by Shivam Bansal"
从属关系树(Dependency Trees) –
句子是单词的集合，句子中单词的相互关系依赖于基本的从属关系语法。从属关系语法专门处理（带标签的）两个词汇单元（单词）的非对称二元关系。每种关系都可以用三元表示法（relation, governor, dependent）关系显示 举个例子：考虑如下句子“
Bills on ports and immigration were submitted by Senator Brownback, Republican of Kansas.
” 该句单词的关系可以用如下的树状结构表示：
doc1 = "Sugar is bad to consume. My sister likes to have sugar, but not my father."
doc2 = "My father spends a lot of time driving my sister around to dance practice."
doc3 = "Doctors suggest that driving may cause increased stress and blood pressure."
doc_complete = [doc1, doc2, doc3]
doc_clean = [doc.split() for doc in doc_complete]
import gensim from gensim
import corpora
# 建立一个语料库词典，其中每个词组都进行了编号
dictionary = corpora.Dictionary(doc_clean)
# 使用前一部构建的词典，把语料库词典从列表转换为文献-检索词矩阵
(DTM)doc_term_matrix = [dictionary.doc2bow(doc) for doc in doc_clean]
# 使用gensim库构建LDA模型
Lda = gensim.models.ldamodel.LdaModel
# 使用DTM构建并训练LDA模型
ldamodel = Lda(doc_term_matrix, num_topics=3, id2word = dictionary,
passes=50)
print(ldamodel.print_topics())
from textblob.classifiers import NaiveBayesClassifier as NBC
from textblob import TextBlob
training_corpus = [
('I am exhausted of this work.', 'Class_B'),
("I can't cooperate with this", 'Class_B'),
('He is my badest enemy!', 'Class_B'),
('My management is poor.', 'Class_B'),
('I love this burger.', 'Class_A'),
('This is an brilliant place!', 'Class_A'),
('I feel very good about these dates.', 'Class_A'),
('This is my best work.', 'Class_A'),
("What an awesome view", 'Class_A'),
('I do not like this dish', 'Class_B')]
test_corpus = [
("I am not feeling well today.", 'Class_B'),
("I feel brilliant!", 'Class_A'),
('Gary is a friend of mine.', 'Class_A'),
("I can't believe I'm doing this.", 'Class_B'),
('The date was good.', 'Class_A'),
('I do not enjoy my job', 'Class_B')]
model = NBC(training_corpus)
print(model.classify("Their codes are amazing."))
>>> "Class_A"
print(model.classify("I don't like their computer."))
>>> "Class_B"
print(model.accuracy(test_corpus))
Scikit.Learn也提供了文本分类的工作流程：
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import classification_report
from sklearn import svm
# 为SVM准备数据 (和朴素贝叶斯分类器使用同样特征)
train_data = []
train_labels = []
for row in training_corpus:
train_data.append(row[0])
train_labels.append(row[1])
test_data = []
test_labels = []
for row in test_corpus:
test_data.append(row[0])
test_labels.append(row[1])
# 创建特征向量
vectorizer = TfidfVectorizer(min_df=4, max_df=0.9)
# 训练特征向量
train_vectors = vectorizer.fit_transform(train_data)
# 在测试集上应用模型
test_vectors = vectorizer.transform(test_data)
# 训练使用线性核的SVM模型
model = svm.SVC(kernel='linear')
model.fit(train_vectors, train_labels)
prediction = model.predict(test_vectors)
>>> ['Class_A' 'Class_A' 'Class_B' 'Class_B' 'Class_A' 'Class_A']
print (classification_report(test_labels, prediction))
文本分类器的效果和特征的质量与数量关系很大，使用任一机器学习模型输入更多的训练数据总是更好的。
4.2 文本匹配/相似度
NLP的另一重要领域是匹配文本对象来计算相似度。它可以应用在拼写纠正，数据去重和基因组分析等。
依据不同的需求，现在已有很多中匹配技术可供使用，这部分会详细介绍常用的几种。
A. Levenshtein 距离
– 两个字符串间的Levenshtein距离被定义为将一个字符串转化为另一字符串所需要变动的修正数的最小值。其中，修正指单个字符的插入、删除或替换。下列代码能使用一种很省内存的算法计算该距离.
def levenshtein(s1,s2):
if len(s1) > len(s2):
s1,s2 = s2,s1
distances = range(len(s1) + 1)
for index2,char2 in enumerate(s2):
newDistances = [index2+1]
for index1,char1 in enumerate(s1):
if char1 == char2:
newDistances.append(distances[index1])
else:
newDistances.append(1 + min((distances[index1], distances[index1+1], newDistances[-1])))
distances = newDistances
return distances[-1]
print(levenshtein("analyze","analyse"))
B. 语音匹配
– 语音匹配算法会把关键词（人名、地名等）作为输入，之后返回一个能大致代表输入类别的单词。在对大型语料库进行索引，纠正拼写错误和匹配相关姓名时很有作用。Soundex和Metaphone是这一领域的两大主要算法，Python中的Fuzzy库能实现Soundex算法，比如：
C. 柔性字符串匹配
– 一套完整的文本匹配系统会包含各种不同的算法以适应文本的多样性。正则表达式对此也能发挥很大的作用。而另一种常用技术则包括 - 精确字符串匹配，词形还原匹配和紧凑匹配等。
D. 余弦相似度
– 当文本以向量形式出现时，使用余弦定理就能衡量两个文本的相似度了。下列代码就把文本按照词频转换成向量并使用余弦值来衡量两个文本的相似度。
import math
from collections import Counter
def get_cosine(vec1, vec2):
common = set(vec1.keys()) & set(vec2.keys())
numerator = sum([vec1[x] * vec2[x] for x in common])
sum1 = sum([vec1[x]**2 for x in vec1.keys()])
sum2 = sum([vec2[x]**2 for x in vec2.keys()])
denominator = math.sqrt(sum1) * math.sqrt(sum2)
if not denominator:
return 0.0
else:
return float(numerator) / denominator
def text_to_vector(text):
words = text.split()
return Counter(words)
text1 = 'This is an article on analytics vidhya'
text2 = 'article on analytics vidhya is about natural language processing'
