streamcorpus_pipeline/_kvlayer_keyword_search.py
def collect_words(self, si):
        '''Collect all of the words to be indexed from a stream item.

        This scans `si` for all of the configured tagger IDs.  It
        collects all of the token values (the
        :attr:`streamcorpus.Token.token`) and returns a
        :class:`collections.Counter` of them.

        :param si: stream item to scan
        :type si: :class:`streamcorpus.StreamItem`
        :return: counter of :class:`unicode` words to index
        :returntype: :class:`collections.Counter`

        '''
        counter = Counter()
        for tagger_id, sentences in si.body.sentences.iteritems():
            if ((self.keyword_tagger_ids is not None
                 and tagger_id not in self.keyword_tagger_ids)):
                continue
            for sentence in sentences:
                for token in sentence.tokens:
                    term = token.token  # always a UTF-8 byte string
                    term = term.decode('utf-8')
                    term = cleanse(term)
                    if ((self.keyword_size_limit is not None and
                         len(term) > self.keyword_size_limit)):
                        continue
                    if term not in self.stop_words:
                        counter[term] += 1
        return counter
def collect_words(self, si):
        '''Collect all of the words to be indexed from a stream item.

        This scans `si` for all of the configured tagger IDs.  It
        collects all of the token values (the
        :attr:`streamcorpus.Token.token`) and returns a
        :class:`collections.Counter` of them.

        :param si: stream item to scan
        :type si: :class:`streamcorpus.StreamItem`
        :return: counter of :class:`unicode` words to index
        :returntype: :class:`collections.Counter`

        '''
        counter = Counter()
        for tagger_id, sentences in si.body.sentences.iteritems():
            if ((self.keyword_tagger_ids is not None
                 and tagger_id not in self.keyword_tagger_ids)):
                continue
            for sentence in sentences:
                for token in sentence.tokens:
                    term = token.token  # always a UTF-8 byte string
                    term = term.decode('utf-8')
                    term = cleanse(term)
                    if ((self.keyword_size_limit is not None and
                         len(term) > self.keyword_size_limit)):
                        continue
                    if term not in self.stop_words:
                        counter[term] += 1
        return counter
Collect all of the words to be indexed from a stream item.

        This scans `si` for all of the configured tagger IDs.  It
        collects all of the token values (the
        :attr:`streamcorpus.Token.token`) and returns a
        :class:`collections.Counter` of them.

        :param si: stream item to scan
        :type si: :class:`streamcorpus.StreamItem`
        :return: counter of :class:`unicode` words to index
        :returntype: :class:`collections.Counter`