ciseau/sentence_tokenizer.py
def detect_sentence_boundaries(tokens):
    """
    Subdivide an input list of strings (tokens)
    into multiple lists according to detected
    sentence boundaries.

    ```
    detect_sentence_boundaries(
        ["Cat ", "sat ", "mat", ". ", "Cat ", "'s ", "named ", "Cool", "."]
    )
    #=> [
        ["Cat ", "sat ", "mat", ". "],
        ["Cat ", "'s ", "named ", "Cool", "."]
    ]
    ```

    Arguments:
    ----------

        tokens : list<str>

    Returns:
    --------
        list<list<str>> : original list subdivided into multiple
            lists according to (detected) sentence boundaries.
    """
    tokenized = group_quoted_tokens(tokens)
    words = []
    sentences = []
    for i in range(len(tokenized)):
        # this is a parenthetical:
        end_sentence = False
        if isinstance(tokenized[i], list):
            if len(words) == 0:
                # end if a sentence finishes inside quoted section,
                # and no sentence was begun beforehand
                if is_end_symbol(tokenized[i][-2].rstrip()):
                    end_sentence = True
            else:
                # end if a sentence finishes inside quote marks
                if (tokenized[i][0][0] == '"' and
                    is_end_symbol(tokenized[i][-2].rstrip()) and
                    not tokenized[i][1][0].isupper()):
                    end_sentence = True
            words.extend(tokenized[i])
        else:
            stripped_tokenized = tokenized[i].rstrip()
            if is_end_symbol(stripped_tokenized):
                words.append(tokenized[i])
                not_last_word = i + 1 != len(tokenized)
                next_word_lowercase = (
                    not_last_word and
                    tokenized[i+1][0].islower()
                )
                next_word_continue_punct = (
                    not_last_word and
                    tokenized[i+1][0] in CONTINUE_PUNCT_SYMBOLS
                )
                end_sentence = not (
                    not_last_word and
                    (
                        next_word_lowercase or
                        next_word_continue_punct
                    )
                )
            else:
                words.append(tokenized[i])
        if end_sentence:
            sentences.append(words)
            words = []

    # add final sentence, if it wasn't added yet.
    if len(words) > 0:
        sentences.append(words)

    # If the final word ends in a period:
    if len(sentences) > 0 and sentences[-1][-1]:
        alpha_word_piece = word_with_alpha_and_period.match(sentences[-1][-1])
        if alpha_word_piece:
            sentences[-1][-1] = alpha_word_piece.group(1)
            sentences[-1].append(alpha_word_piece.group(2))
    return sentences
def detect_sentence_boundaries(tokens):
    """
    Subdivide an input list of strings (tokens)
    into multiple lists according to detected
    sentence boundaries.

    ```
    detect_sentence_boundaries(
        ["Cat ", "sat ", "mat", ". ", "Cat ", "'s ", "named ", "Cool", "."]
    )
    #=> [
        ["Cat ", "sat ", "mat", ". "],
        ["Cat ", "'s ", "named ", "Cool", "."]
    ]
    ```

    Arguments:
    ----------

        tokens : list<str>

    Returns:
    --------
        list<list<str>> : original list subdivided into multiple
            lists according to (detected) sentence boundaries.
    """
    tokenized = group_quoted_tokens(tokens)
    words = []
    sentences = []
    for i in range(len(tokenized)):
        # this is a parenthetical:
        end_sentence = False
        if isinstance(tokenized[i], list):
            if len(words) == 0:
                # end if a sentence finishes inside quoted section,
                # and no sentence was begun beforehand
                if is_end_symbol(tokenized[i][-2].rstrip()):
                    end_sentence = True
            else:
                # end if a sentence finishes inside quote marks
                if (tokenized[i][0][0] == '"' and
                    is_end_symbol(tokenized[i][-2].rstrip()) and
                    not tokenized[i][1][0].isupper()):
                    end_sentence = True
            words.extend(tokenized[i])
        else:
            stripped_tokenized = tokenized[i].rstrip()
            if is_end_symbol(stripped_tokenized):
                words.append(tokenized[i])
                not_last_word = i + 1 != len(tokenized)
                next_word_lowercase = (
                    not_last_word and
                    tokenized[i+1][0].islower()
                )
                next_word_continue_punct = (
                    not_last_word and
                    tokenized[i+1][0] in CONTINUE_PUNCT_SYMBOLS
                )
                end_sentence = not (
                    not_last_word and
                    (
                        next_word_lowercase or
                        next_word_continue_punct
                    )
                )
            else:
                words.append(tokenized[i])
        if end_sentence:
            sentences.append(words)
            words = []

    # add final sentence, if it wasn't added yet.
    if len(words) > 0:
        sentences.append(words)

    # If the final word ends in a period:
    if len(sentences) > 0 and sentences[-1][-1]:
        alpha_word_piece = word_with_alpha_and_period.match(sentences[-1][-1])
        if alpha_word_piece:
            sentences[-1][-1] = alpha_word_piece.group(1)
            sentences[-1].append(alpha_word_piece.group(2))
    return sentences
Subdivide an input list of strings (tokens)
    into multiple lists according to detected
    sentence boundaries.

    ```
    detect_sentence_boundaries(
        ["Cat ", "sat ", "mat", ". ", "Cat ", "'s ", "named ", "Cool", "."]
    )
    #=> [
        ["Cat ", "sat ", "mat", ". "],
        ["Cat ", "'s ", "named ", "Cool", "."]
    ]
    ```

    Arguments:
    ----------

        tokens : list<str>

    Returns:
    --------
        list<list<str>> : original list subdivided into multiple
            lists according to (detected) sentence boundaries.