ciseau/word_tokenizer.py
def mark_regex(regex, text, split_locations):
    """
    Regex that adds a 'SHOULD_SPLIT' marker at the end
    location of each matching group of the given regex.

    Arguments
    ---------
        regex : re.Expression
        text : str, same length as split_locations
        split_locations : list<int>, split decisions.
    """
    for match in regex.finditer(text):
        end_match = match.end()
        if end_match < len(split_locations):
            split_locations[end_match] = SHOULD_SPLIT
def mark_regex(regex, text, split_locations):
    """
    Regex that adds a 'SHOULD_SPLIT' marker at the end
    location of each matching group of the given regex.

    Arguments
    ---------
        regex : re.Expression
        text : str, same length as split_locations
        split_locations : list<int>, split decisions.
    """
    for match in regex.finditer(text):
        end_match = match.end()
        if end_match < len(split_locations):
            split_locations[end_match] = SHOULD_SPLIT
Regex that adds a 'SHOULD_SPLIT' marker at the end
    location of each matching group of the given regex.

    Arguments
    ---------
        regex : re.Expression
        text : str, same length as split_locations
        split_locations : list<int>, split decisions.