ktext/preprocess.py
def process_text_constructor(cleaner: Callable,
                             tokenizer: Callable,
                             append_indicators: bool,
                             start_tok: str,
                             end_tok: str):
    """Generate a function that will clean and tokenize text."""
    def process_text(text):
        if append_indicators:
            return [[start_tok] + tokenizer(cleaner(doc)) + [end_tok] for doc in text]
        return [tokenizer(cleaner(doc)) for doc in text]

    return process_text
def process_text_constructor(cleaner: Callable,
                             tokenizer: Callable,
                             append_indicators: bool,
                             start_tok: str,
                             end_tok: str):
    """Generate a function that will clean and tokenize text."""
    def process_text(text):
        if append_indicators:
            return [[start_tok] + tokenizer(cleaner(doc)) + [end_tok] for doc in text]
        return [tokenizer(cleaner(doc)) for doc in text]

    return process_text
Generate a function that will clean and tokenize text.