ktext/preprocess.py
def fit(self,
            data: List[str],
            return_tokenized_data: bool = False) -> Union[None, List[List[str]]]:
        """
        TODO: update docs

        Apply cleaner and tokenzier to raw data and build vocabulary.

        Parameters
        ----------
        data : List[str]
            These are raw documents, which are a list of strings. ex:
            [["The quick brown fox"], ["jumps over the lazy dog"]]
        return_tokenized_data : bool
            Return the tokenized strings.  This is primarly used for debugging
            purposes.

        Returns
        -------
        None or List[List[str]]
            if return_tokenized_data=True then will return tokenized documents,
            otherwise will not return anything.
        """
        self.__clear_data()
        now = get_time()
        logging.warning(f'....tokenizing data')
        tokenized_data = self.parallel_process_text(data)

        if not self.padding_maxlen:
            # its not worth the overhead to parallelize document length counts
            length_counts = map(count_len, tokenized_data)
            self.document_length_histogram = Counter(length_counts)
            self.generate_doc_length_stats()

        # Learn corpus on single thread
        logging.warning(f'(1/2) done. {time_diff(now)} sec')
        logging.warning(f'....building corpus')
        now = get_time()
        self.indexer = custom_Indexer(num_words=self.keep_n)
        self.indexer.fit_on_tokenized_texts(tokenized_data)

        # Build Dictionary accounting For 0 padding, and reserve 1 for unknown and rare Words
        self.token2id = self.indexer.word_index
        self.id2token = {v: k for k, v in self.token2id.items()}
        self.n_tokens = max(self.indexer.word_index.values())

        # logging
        logging.warning(f'(2/2) done. {time_diff(now)} sec')
        logging.warning(f'Finished parsing {self.indexer.document_count:,} documents.')

        if return_tokenized_data:
            return tokenized_data
def fit(self,
            data: List[str],
            return_tokenized_data: bool = False) -> Union[None, List[List[str]]]:
        """
        TODO: update docs

        Apply cleaner and tokenzier to raw data and build vocabulary.

        Parameters
        ----------
        data : List[str]
            These are raw documents, which are a list of strings. ex:
            [["The quick brown fox"], ["jumps over the lazy dog"]]
        return_tokenized_data : bool
            Return the tokenized strings.  This is primarly used for debugging
            purposes.

        Returns
        -------
        None or List[List[str]]
            if return_tokenized_data=True then will return tokenized documents,
            otherwise will not return anything.
        """
        self.__clear_data()
        now = get_time()
        logging.warning(f'....tokenizing data')
        tokenized_data = self.parallel_process_text(data)

        if not self.padding_maxlen:
            # its not worth the overhead to parallelize document length counts
            length_counts = map(count_len, tokenized_data)
            self.document_length_histogram = Counter(length_counts)
            self.generate_doc_length_stats()

        # Learn corpus on single thread
        logging.warning(f'(1/2) done. {time_diff(now)} sec')
        logging.warning(f'....building corpus')
        now = get_time()
        self.indexer = custom_Indexer(num_words=self.keep_n)
        self.indexer.fit_on_tokenized_texts(tokenized_data)

        # Build Dictionary accounting For 0 padding, and reserve 1 for unknown and rare Words
        self.token2id = self.indexer.word_index
        self.id2token = {v: k for k, v in self.token2id.items()}
        self.n_tokens = max(self.indexer.word_index.values())

        # logging
        logging.warning(f'(2/2) done. {time_diff(now)} sec')
        logging.warning(f'Finished parsing {self.indexer.document_count:,} documents.')

        if return_tokenized_data:
            return tokenized_data
TODO: update docs

        Apply cleaner and tokenzier to raw data and build vocabulary.

        Parameters
        ----------
        data : List[str]
            These are raw documents, which are a list of strings. ex:
            [["The quick brown fox"], ["jumps over the lazy dog"]]
        return_tokenized_data : bool
            Return the tokenized strings.  This is primarly used for debugging
            purposes.

        Returns
        -------
        None or List[List[str]]
            if return_tokenized_data=True then will return tokenized documents,
            otherwise will not return anything.