streamcorpus_pipeline/offsets.py
def add_xpaths_to_stream_item(si):
    '''Mutably tag tokens with xpath offsets.

    Given some stream item, this will tag all tokens from all taggings
    in the document that contain character offsets. Note that some
    tokens may not have computable xpath offsets, so an xpath offset
    for those tokens will not be set. (See the documentation and
    comments for ``char_offsets_to_xpaths`` for what it means for a
    token to have a computable xpath.)

    If a token can have its xpath offset computed, it is added to its
    set of offsets with a ``OffsetType.XPATH_CHARS`` key.
    '''
    def sentences_to_xpaths(sentences):
        tokens = sentences_to_char_tokens(sentences)
        offsets = char_tokens_to_char_offsets(tokens)
        return char_offsets_to_xpaths(html, offsets)

    def xprange_to_offset(xprange):
        return Offset(type=OffsetType.XPATH_CHARS,
                      first=xprange.start_offset, length=0,
                      xpath=xprange.start_xpath,
                      content_form='clean_html', value=None,
                      xpath_end=xprange.end_xpath,
                      xpath_end_offset=xprange.end_offset)

    html = unicode(si.body.clean_html, 'utf-8')
    for sentences in si.body.sentences.itervalues():
        tokens = sentences_to_char_tokens(sentences)
        for token, xprange in izip(tokens, sentences_to_xpaths(sentences)):
            if xprange is None:
                continue
            offset = xprange_to_offset(xprange)
            token.offsets[OffsetType.XPATH_CHARS] = offset
def add_xpaths_to_stream_item(si):
    '''Mutably tag tokens with xpath offsets.

    Given some stream item, this will tag all tokens from all taggings
    in the document that contain character offsets. Note that some
    tokens may not have computable xpath offsets, so an xpath offset
    for those tokens will not be set. (See the documentation and
    comments for ``char_offsets_to_xpaths`` for what it means for a
    token to have a computable xpath.)

    If a token can have its xpath offset computed, it is added to its
    set of offsets with a ``OffsetType.XPATH_CHARS`` key.
    '''
    def sentences_to_xpaths(sentences):
        tokens = sentences_to_char_tokens(sentences)
        offsets = char_tokens_to_char_offsets(tokens)
        return char_offsets_to_xpaths(html, offsets)

    def xprange_to_offset(xprange):
        return Offset(type=OffsetType.XPATH_CHARS,
                      first=xprange.start_offset, length=0,
                      xpath=xprange.start_xpath,
                      content_form='clean_html', value=None,
                      xpath_end=xprange.end_xpath,
                      xpath_end_offset=xprange.end_offset)

    html = unicode(si.body.clean_html, 'utf-8')
    for sentences in si.body.sentences.itervalues():
        tokens = sentences_to_char_tokens(sentences)
        for token, xprange in izip(tokens, sentences_to_xpaths(sentences)):
            if xprange is None:
                continue
            offset = xprange_to_offset(xprange)
            token.offsets[OffsetType.XPATH_CHARS] = offset
Mutably tag tokens with xpath offsets.

    Given some stream item, this will tag all tokens from all taggings
    in the document that contain character offsets. Note that some
    tokens may not have computable xpath offsets, so an xpath offset
    for those tokens will not be set. (See the documentation and
    comments for ``char_offsets_to_xpaths`` for what it means for a
    token to have a computable xpath.)

    If a token can have its xpath offset computed, it is added to its
    set of offsets with a ``OffsetType.XPATH_CHARS`` key.