ciseau/sentence_tokenizer.py
def sent_tokenize(text, keep_whitespace=False, normalize_ascii=True):
    """
    Perform sentence + word tokenization on the input text
    using regular expressions and english/french specific
    rules.

    Arguments:
    ----------
        text : str, input string to tokenize
        keep_whitespace : bool, whether to strip out spaces
            and newlines.
        normalize_ascii : bool, perform some replacements
            on rare characters so that they become
            easier to process in a ascii pipeline
            (canonicalize dashes, replace œ -> oe, etc..)
    Returns:
    --------
        list<list<str>> : sentences with their content held
            in a list of strings for each token.
    """
    sentences = detect_sentence_boundaries(
        tokenize(
            text,
            normalize_ascii
        )
    )
    if not keep_whitespace:
        sentences = remove_whitespace(sentences)
    return sentences
def sent_tokenize(text, keep_whitespace=False, normalize_ascii=True):
    """
    Perform sentence + word tokenization on the input text
    using regular expressions and english/french specific
    rules.

    Arguments:
    ----------
        text : str, input string to tokenize
        keep_whitespace : bool, whether to strip out spaces
            and newlines.
        normalize_ascii : bool, perform some replacements
            on rare characters so that they become
            easier to process in a ascii pipeline
            (canonicalize dashes, replace œ -> oe, etc..)
    Returns:
    --------
        list<list<str>> : sentences with their content held
            in a list of strings for each token.
    """
    sentences = detect_sentence_boundaries(
        tokenize(
            text,
            normalize_ascii
        )
    )
    if not keep_whitespace:
        sentences = remove_whitespace(sentences)
    return sentences
Perform sentence + word tokenization on the input text
    using regular expressions and english/french specific
    rules.

    Arguments:
    ----------
        text : str, input string to tokenize
        keep_whitespace : bool, whether to strip out spaces
            and newlines.
        normalize_ascii : bool, perform some replacements
            on rare characters so that they become
            easier to process in a ascii pipeline
            (canonicalize dashes, replace œ -> oe, etc..)
    Returns:
    --------
        list<list<str>> : sentences with their content held
            in a list of strings for each token.