我想出了我自己的跳躍向量化器的實現。它的靈感來自於this的帖子。爲了限制特徵空間,我還限制了不跳過句子的邊界(使用nltk.sent_tokenize
)。這裏是我的代碼:
import nltk
from itertools import combinations
from toolz import compose
from sklearn.feature_extraction.text import CountVectorizer
class SkipGramVectorizer(CountVectorizer):
def __init__(self, k=1, **kwds):
super(SkipGramVectorizer, self).__init__(**kwds)
self.k=k
def build_sent_analyzer(self, preprocess, stop_words, tokenize):
return lambda sent : self._word_skip_grams(
compose(tokenize, preprocess, self.decode)(sent),
stop_words)
def build_analyzer(self):
preprocess = self.build_preprocessor()
stop_words = self.get_stop_words()
tokenize = self.build_tokenizer()
sent_analyze = self.build_sent_analyzer(preprocess, stop_words, tokenize)
return lambda doc : self._sent_skip_grams(doc, sent_analyze)
def _sent_skip_grams(self, doc, sent_analyze):
skip_grams = []
for sent in nltk.sent_tokenize(doc):
skip_grams.extend(sent_analyze(sent))
return skip_grams
def _word_skip_grams(self, tokens, stop_words=None):
"""Turn tokens into a sequence of n-grams after stop words filtering"""
# handle stop words
if stop_words is not None:
tokens = [w for w in tokens if w not in stop_words]
# handle token n-grams
min_n, max_n = self.ngram_range
k = self.k
if max_n != 1:
original_tokens = tokens
if min_n == 1:
# no need to do any slicing for unigrams
# just iterate through the original tokens
tokens = list(original_tokens)
min_n += 1
else:
tokens = []
n_original_tokens = len(original_tokens)
# bind method outside of loop to reduce overhead
tokens_append = tokens.append
space_join = " ".join
for n in xrange(min_n,
min(max_n + 1, n_original_tokens + 1)):
for i in xrange(n_original_tokens - n + 1):
# k-skip-n-grams
head = [original_tokens[i]]
for skip_tail in combinations(original_tokens[i+1:i+n+k], n-1):
tokens_append(space_join(head + list(skip_tail)))
return tokens
def test(text, ngram_range, k):
vectorizer = SkipGramVectorizer(ngram_range=ngram_range, k=k)
vectorizer.fit_transform(text)
print(vectorizer.get_feature_names())
def main():
text = ['Insurgents killed in ongoing fighting.']
# 2-skip-bi-grams
test(text, (2,2), 2)
# 2-skip-tri-grams
test(text, (3,3), 2)
###############################################################################################
if __name__ == '__main__':
main()
這將產生以下功能名稱:
[u'in fighting', u'in ongoing', u'insurgents in', u'insurgents killed', u'insurgents ongoing', u'killed fighting', u'killed in', u'killed ongoing', u'ongoing fighting']
[u'in ongoing fighting', u'insurgents in fighting', u'insurgents in ongoing', u'insurgents killed fighting', u'insurgents killed in', u'insurgents killed ongoing', u'insurgents ongoing fighting', u'killed in fighting', u'killed in ongoing', u'killed ongoing fighting']
請注意,我基本上是從VectorizerMixin
類拿了_word_ngrams
功能,取代了線
tokens_append(space_join(original_tokens[i: i + n]))
與以下內容:
head = [original_tokens[i]]
for skip_tail in combinations(original_tokens[i+1:i+n+k], n-1):
tokens_append(space_join(head + list(skip_tail)))
感謝您的回覆,兄弟。我會很快嘗試並讓你知道它。 –