ciseau/wiki_markup_processing.py
def to_raw_text_pairings(text, keep_whitespace=False, normalize_ascii=True):
    """
    A generator to convert raw text segments, with xml, and other
    non-textual content to a list of words without any markup.
    Additionally dates are replaced by `7777` for normalization,
    along with wikipedia anchors kept.

    Arguments
    ---------
       text: str, input text to tokenize, strip of markup.
       keep_whitespace : bool, should the output retain the
          whitespace of the input (so that char offsets in the
          output correspond to those in the input).

    Returns
    -------
        generator<list<list<str>>>, a generator for sentences, with
            within each sentence a list of the words separated.
    """
    out = text
    out = _remove_mvar(out)
    out = _remove_squiggly_bracket(out)
    out = _remove_table(out)
    out = remove_markup(out)
    out = remove_wikipedia_link.sub(anchor_replacer, out)
    out = remove_bullets_nbsps.sub(empty_space, out)
    out = remove_math_sections(out)
    out = remove_html(out)
    for sentence in sent_tokenize(out, keep_whitespace, normalize_ascii):
        yield sentence
def to_raw_text_pairings(text, keep_whitespace=False, normalize_ascii=True):
    """
    A generator to convert raw text segments, with xml, and other
    non-textual content to a list of words without any markup.
    Additionally dates are replaced by `7777` for normalization,
    along with wikipedia anchors kept.

    Arguments
    ---------
       text: str, input text to tokenize, strip of markup.
       keep_whitespace : bool, should the output retain the
          whitespace of the input (so that char offsets in the
          output correspond to those in the input).

    Returns
    -------
        generator<list<list<str>>>, a generator for sentences, with
            within each sentence a list of the words separated.
    """
    out = text
    out = _remove_mvar(out)
    out = _remove_squiggly_bracket(out)
    out = _remove_table(out)
    out = remove_markup(out)
    out = remove_wikipedia_link.sub(anchor_replacer, out)
    out = remove_bullets_nbsps.sub(empty_space, out)
    out = remove_math_sections(out)
    out = remove_html(out)
    for sentence in sent_tokenize(out, keep_whitespace, normalize_ascii):
        yield sentence
A generator to convert raw text segments, with xml, and other
    non-textual content to a list of words without any markup.
    Additionally dates are replaced by `7777` for normalization,
    along with wikipedia anchors kept.

    Arguments
    ---------
       text: str, input text to tokenize, strip of markup.
       keep_whitespace : bool, should the output retain the
          whitespace of the input (so that char offsets in the
          output correspond to those in the input).

    Returns
    -------
        generator<list<list<str>>>, a generator for sentences, with
            within each sentence a list of the words separated.