ktext/preprocess.py
def fit_transform(self,
                      data: List[str]) -> List[List[int]]:
        """
        Apply cleaner and tokenzier to raw data, build vocabulary and return
        transfomred dataset that is a List[List[int]].  This will use
        process-based-threading on all available cores.

        ex:
        >>> data = [["The quick brown fox"], ["jumps over the lazy dog"]]
        >>> pp = preprocess(maxlen=5, no_below=0)
        >>> pp.fit_transform(data)
        # 0 padding is applied
        [[0, 2, 3, 4, 5], [6, 7, 2, 8, 9]]

        Parameters
        ----------
        data : List[str]
            These are raw documents, which are a list of strings. ex:
            [["The quick brown fox"], ["jumps over the lazy dog"]]

        Returns
        -------
        numpy.array with shape (number of documents, max_len)

        """
        tokenized_data = self.fit(data, return_tokenized_data=True)

        logging.warning(f'...fit is finished, beginning transform')
        now = get_time()
        indexed_data = self.indexer.tokenized_texts_to_sequences(tokenized_data)
        logging.warning(f'...padding data')
        final_data = self.pad(indexed_data)
        logging.warning(f'done. {time_diff(now)} sec')
        return final_data
def fit_transform(self,
                      data: List[str]) -> List[List[int]]:
        """
        Apply cleaner and tokenzier to raw data, build vocabulary and return
        transfomred dataset that is a List[List[int]].  This will use
        process-based-threading on all available cores.

        ex:
        >>> data = [["The quick brown fox"], ["jumps over the lazy dog"]]
        >>> pp = preprocess(maxlen=5, no_below=0)
        >>> pp.fit_transform(data)
        # 0 padding is applied
        [[0, 2, 3, 4, 5], [6, 7, 2, 8, 9]]

        Parameters
        ----------
        data : List[str]
            These are raw documents, which are a list of strings. ex:
            [["The quick brown fox"], ["jumps over the lazy dog"]]

        Returns
        -------
        numpy.array with shape (number of documents, max_len)

        """
        tokenized_data = self.fit(data, return_tokenized_data=True)

        logging.warning(f'...fit is finished, beginning transform')
        now = get_time()
        indexed_data = self.indexer.tokenized_texts_to_sequences(tokenized_data)
        logging.warning(f'...padding data')
        final_data = self.pad(indexed_data)
        logging.warning(f'done. {time_diff(now)} sec')
        return final_data
Apply cleaner and tokenzier to raw data, build vocabulary and return
        transfomred dataset that is a List[List[int]].  This will use
        process-based-threading on all available cores.

        ex:
        >>> data = [["The quick brown fox"], ["jumps over the lazy dog"]]
        >>> pp = preprocess(maxlen=5, no_below=0)
        >>> pp.fit_transform(data)
        # 0 padding is applied
        [[0, 2, 3, 4, 5], [6, 7, 2, 8, 9]]

        Parameters
        ----------
        data : List[str]
            These are raw documents, which are a list of strings. ex:
            [["The quick brown fox"], ["jumps over the lazy dog"]]

        Returns
        -------
        numpy.array with shape (number of documents, max_len)