splearn/feature_extraction/text.py
def fit_transform(self, Z):
        """Learn the vocabulary dictionary and return term-document matrix.

        This is equivalent to fit followed by transform, but more efficiently
        implemented.

        Parameters
        ----------
        Z : iterable or DictRDD with column 'X'
            An iterable of raw_documents which yields either str, unicode or
            file objects; or a DictRDD with column 'X' containing such
            iterables.

        Returns
        -------
        X : array, [n_samples, n_features] or DictRDD
            Document-term matrix.
        """
        self._validate_vocabulary()

        # map analyzer and cache result
        analyze = self.build_analyzer()
        A = Z.transform(lambda X: list(map(analyze, X)), column='X').persist()

        # create vocabulary
        X = A[:, 'X'] if isinstance(A, DictRDD) else A
        self.vocabulary_ = self._init_vocab(X)

        # transform according to vocabulary
        mapper = self.broadcast(self._count_vocab, A.context)
        Z = A.transform(mapper, column='X', dtype=sp.spmatrix)


        if not self.fixed_vocabulary_:
            X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z

            max_df = self.max_df
            min_df = self.min_df
            max_features = self.max_features

            # limit features according to min_df, max_df parameters
            n_doc = X.shape[0]
            max_doc_count = (max_df
                             if isinstance(max_df, numbers.Integral)
                             else max_df * n_doc)
            min_doc_count = (min_df
                             if isinstance(min_df, numbers.Integral)
                             else min_df * n_doc)
            if max_doc_count < min_doc_count:
                raise ValueError(
                    "max_df corresponds to < documents than min_df")
            kept_indices, self.stop_words_ = self._limit_features(
                X, self.vocabulary_, max_doc_count, min_doc_count, max_features)

            # sort features
            map_index = self._sort_features(self.vocabulary_)

            # combined mask
            mask = kept_indices[map_index]

            Z = Z.transform(lambda x: x[:, mask], column='X', dtype=sp.spmatrix)
        A.unpersist()
        return Z
def fit_transform(self, Z):
        """Learn the vocabulary dictionary and return term-document matrix.

        This is equivalent to fit followed by transform, but more efficiently
        implemented.

        Parameters
        ----------
        Z : iterable or DictRDD with column 'X'
            An iterable of raw_documents which yields either str, unicode or
            file objects; or a DictRDD with column 'X' containing such
            iterables.

        Returns
        -------
        X : array, [n_samples, n_features] or DictRDD
            Document-term matrix.
        """
        self._validate_vocabulary()

        # map analyzer and cache result
        analyze = self.build_analyzer()
        A = Z.transform(lambda X: list(map(analyze, X)), column='X').persist()

        # create vocabulary
        X = A[:, 'X'] if isinstance(A, DictRDD) else A
        self.vocabulary_ = self._init_vocab(X)

        # transform according to vocabulary
        mapper = self.broadcast(self._count_vocab, A.context)
        Z = A.transform(mapper, column='X', dtype=sp.spmatrix)


        if not self.fixed_vocabulary_:
            X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z

            max_df = self.max_df
            min_df = self.min_df
            max_features = self.max_features

            # limit features according to min_df, max_df parameters
            n_doc = X.shape[0]
            max_doc_count = (max_df
                             if isinstance(max_df, numbers.Integral)
                             else max_df * n_doc)
            min_doc_count = (min_df
                             if isinstance(min_df, numbers.Integral)
                             else min_df * n_doc)
            if max_doc_count < min_doc_count:
                raise ValueError(
                    "max_df corresponds to < documents than min_df")
            kept_indices, self.stop_words_ = self._limit_features(
                X, self.vocabulary_, max_doc_count, min_doc_count, max_features)

            # sort features
            map_index = self._sort_features(self.vocabulary_)

            # combined mask
            mask = kept_indices[map_index]

            Z = Z.transform(lambda x: x[:, mask], column='X', dtype=sp.spmatrix)
        A.unpersist()
        return Z
Learn the vocabulary dictionary and return term-document matrix.

        This is equivalent to fit followed by transform, but more efficiently
        implemented.

        Parameters
        ----------
        Z : iterable or DictRDD with column 'X'
            An iterable of raw_documents which yields either str, unicode or
            file objects; or a DictRDD with column 'X' containing such
            iterables.

        Returns
        -------
        X : array, [n_samples, n_features] or DictRDD
            Document-term matrix.