ciseau/word_tokenizer.py
def tokenize(text, normalize_ascii=True):
    """
    Convert a single string into a list of substrings
    split along punctuation and word boundaries. Keep
    whitespace intact by always attaching it to the
    previous token.

    Arguments:
    ----------
        text : str
        normalize_ascii : bool, perform some replacements
            on non-ascii characters to canonicalize the
            string (defaults to True).

    Returns:
    --------
        list<str>, list of substring tokens.
    """
    # 1. If there's no punctuation, return immediately
    if no_punctuation.match(text):
        return [text]
    # 2. let's standardize the input text to ascii (if desired)
    # Note: this will no longer respect input-to-output character positions
    if normalize_ascii:
        # normalize these greco-roman characters to ascii:
        text = text.replace(u"œ", "oe").replace(u"æ", "ae")
        # normalize dashes:
        text = repeated_dash_converter.sub("-", text)
    # 3. let's construct an integer array of the possible split locations:
    split_locations = [UNDECIDED] * len(text)

    regexes = (
        pure_whitespace,
        left_quote_shifter,
        left_quote_converter,
        left_single_quote_converter,
        remaining_quote_converter,
        # regex can't fix this -> regex ca n't fix this
        english_nots,
        # you'll dig this -> you 'll dig this
        english_contractions,
        # the rhino's horns -> the rhino 's horns
        english_specific_appendages,
        # qu'a tu fais au rhino -> qu ' a tu fais au rhino,
        french_appendages
    )
    # 4. Mark end locations for specific regular expressions:
    for regex in regexes:
        mark_regex(regex, text, split_locations)

    begin_end_regexes = (
        multi_single_quote_finder,
        right_single_quote_converter,
        # use dashes as the breakpoint:
        # the rhino--truck -> the rhino -- truck
        simple_dash_finder if normalize_ascii else advanced_dash_finder,
        numerical_expression,
        url_file_finder,
        shifted_ellipses,
        # the #rhino! -> the # rhino ! ;
        # the rino[sic] -> the rino [ sic ]
        shifted_standard_punctuation
    )

    # 5. Mark begin and end locations for other regular expressions:
    for regex in begin_end_regexes:
        mark_begin_end_regex(regex, text, split_locations)

    # 6. Remove splitting on exceptional uses of periods:
    # I'm with Mr. -> I 'm with Mr. , I'm with Mister. -> I 'm with Mister .
    protect_shorthand(text, split_locations)

    if normalize_ascii:
        text = dash_converter.sub("-", text)
    # 7. Return the split string using the integer list:
    return list(split_with_locations(text, split_locations))
def tokenize(text, normalize_ascii=True):
    """
    Convert a single string into a list of substrings
    split along punctuation and word boundaries. Keep
    whitespace intact by always attaching it to the
    previous token.

    Arguments:
    ----------
        text : str
        normalize_ascii : bool, perform some replacements
            on non-ascii characters to canonicalize the
            string (defaults to True).

    Returns:
    --------
        list<str>, list of substring tokens.
    """
    # 1. If there's no punctuation, return immediately
    if no_punctuation.match(text):
        return [text]
    # 2. let's standardize the input text to ascii (if desired)
    # Note: this will no longer respect input-to-output character positions
    if normalize_ascii:
        # normalize these greco-roman characters to ascii:
        text = text.replace(u"œ", "oe").replace(u"æ", "ae")
        # normalize dashes:
        text = repeated_dash_converter.sub("-", text)
    # 3. let's construct an integer array of the possible split locations:
    split_locations = [UNDECIDED] * len(text)

    regexes = (
        pure_whitespace,
        left_quote_shifter,
        left_quote_converter,
        left_single_quote_converter,
        remaining_quote_converter,
        # regex can't fix this -> regex ca n't fix this
        english_nots,
        # you'll dig this -> you 'll dig this
        english_contractions,
        # the rhino's horns -> the rhino 's horns
        english_specific_appendages,
        # qu'a tu fais au rhino -> qu ' a tu fais au rhino,
        french_appendages
    )
    # 4. Mark end locations for specific regular expressions:
    for regex in regexes:
        mark_regex(regex, text, split_locations)

    begin_end_regexes = (
        multi_single_quote_finder,
        right_single_quote_converter,
        # use dashes as the breakpoint:
        # the rhino--truck -> the rhino -- truck
        simple_dash_finder if normalize_ascii else advanced_dash_finder,
        numerical_expression,
        url_file_finder,
        shifted_ellipses,
        # the #rhino! -> the # rhino ! ;
        # the rino[sic] -> the rino [ sic ]
        shifted_standard_punctuation
    )

    # 5. Mark begin and end locations for other regular expressions:
    for regex in begin_end_regexes:
        mark_begin_end_regex(regex, text, split_locations)

    # 6. Remove splitting on exceptional uses of periods:
    # I'm with Mr. -> I 'm with Mr. , I'm with Mister. -> I 'm with Mister .
    protect_shorthand(text, split_locations)

    if normalize_ascii:
        text = dash_converter.sub("-", text)
    # 7. Return the split string using the integer list:
    return list(split_with_locations(text, split_locations))
Convert a single string into a list of substrings
    split along punctuation and word boundaries. Keep
    whitespace intact by always attaching it to the
    previous token.

    Arguments:
    ----------
        text : str
        normalize_ascii : bool, perform some replacements
            on non-ascii characters to canonicalize the
            string (defaults to True).

    Returns:
    --------
        list<str>, list of substring tokens.