research/brotlidump.py
def readComplexCode(self, hskip, alphabet):
        """Read complex code"""
        stream = self.stream
        #read the lengths for the length code
        lengths = [1,2,3,4,0,5,17,6,16,7,8,9,10,11,12,13,14,15][hskip:]
        codeLengths = {}
        total = 0
        lol = LengthOfLengthAlphabet('##'+alphabet.name)
        #lengthCode will be used for coding the lengths of the new code
        #we use it for display until now; definition comes below
        lengthCode = LengthAlphabet('#'+alphabet.name)
        lengthIter = iter(lengths)
        lengthsLeft = len(lengths)
        while total<32 and lengthsLeft>0:
            lengthsLeft -= 1
            newSymbol = next(lengthIter)
            lol.description = str(lengthCode[newSymbol])
            length = self.verboseRead(lol)
            if length:
                codeLengths[newSymbol] = length
                total += 32>>length
        if total>32: raise ValueError("Stream format")
        if len(codeLengths)==1: codeLengths[list(codeLengths.keys())[0]] = 0
        #Now set the encoding of the lengthCode
        lengthCode.setLength(codeLengths)
        print("***** Lengths for {} will be coded as:".format(alphabet.name))
        lengthCode.showCode()
        #Now determine the symbol lengths with the lengthCode
        symbolLengths = {}
        total = 0
        lastLength = 8
        alphabetIter = iter(alphabet)
        while total<32768:
            #look ahead to see what is going to happen
            length = lengthCode.decodePeek(
                self.stream.peek(lengthCode.maxLength))[1].index
            #in every branch, set lengthCode.description to explanatory text
            #lengthCode calls format(symbol, extra) with this string
            if length==0:
                symbol = next(alphabetIter)
                lengthCode.description = 'symbol {} unused'.format(symbol)
                self.verboseRead(lengthCode)
                #unused symbol
                continue
            if length==16:
                lengthCode.description = \
                    '{1}+3 symbols of length '+str(lastLength)
                extra = self.verboseRead(lengthCode)
                #scan series of 16s (repeat counts)
                #start with repeat count 2
                repeat = 2
                startSymbol = next(alphabetIter)
                endSymbol = next(alphabetIter)
                symbolLengths[startSymbol.index] = \
                    symbolLengths[endSymbol.index] = lastLength
                #count the two just defined symbols
                total += 2*32768>>lastLength
                #note: loop may end because we're there
                #even if a 16 _appears_ to follow
                while True:
                    #determine last symbol
                    oldRepeat = repeat
                    repeat = (repeat-2<<2)+extra+3
                    #read as many symbols as repeat increased
                    for i in range(oldRepeat, repeat):
                        endSymbol = next(alphabetIter)
                        symbolLengths[endSymbol.index] = lastLength
                    #compute new total; it may be end of loop
                    total += (repeat-oldRepeat)*32768>>lastLength
                    if total>=32768: break
                    #see if there is more to do
                    length = lengthCode.decodePeek(
                        self.stream.peek(lengthCode.maxLength))[1].index
                    if length!=16: break
                    lengthCode.description = 'total {}+{{1}} symbols'.format(
                        (repeat-2<<2)+3)
                    extra = self.verboseRead(lengthCode)
            elif length==17:
                #read, and show explanation
                lengthCode.description = '{1}+3 unused'
                extra = self.verboseRead(lengthCode)
                #scan series of 17s (groups of zero counts)
                #start with repeat count 2
                repeat = 2
                startSymbol = next(alphabetIter)
                endSymbol = next(alphabetIter)
                #note: loop will not end with total==32768,
                #since total doesn't change here
                while True:
                    #determine last symbol
                    oldRepeat = repeat
                    repeat = (repeat-2<<3)+extra+3
                    #read as many symbols as repeat increases
                    for i in range(repeat-oldRepeat):
                        endSymbol = next(alphabetIter)
                    #see if there is more to do
                    length = lengthCode.decodePeek(
                        self.stream.peek(lengthCode.maxLength))[1].index
                    if length!=17: break
                    lengthCode.description = 'total {}+{{1}} unused'.format(
                        (repeat-2<<3)+3)
                    extra = self.verboseRead(lengthCode)
            else:
                symbol = next(alphabetIter)
                #double braces for format
                char = str(symbol)
                if char in '{}': char *= 2
                lengthCode.description = \
                    'Length for {} is {{0.index}} bits'.format(char)
                #output is not needed (will be 0)
                self.verboseRead(lengthCode)
                symbolLengths[symbol.index] = length
                total += 32768>>length
                lastLength = length
        assert total==32768
        alphabet.setLength(symbolLengths)
        print('End of table. Prefix code '+alphabet.name+':')
        alphabet.showCode()
def readComplexCode(self, hskip, alphabet):
        """Read complex code"""
        stream = self.stream
        #read the lengths for the length code
        lengths = [1,2,3,4,0,5,17,6,16,7,8,9,10,11,12,13,14,15][hskip:]
        codeLengths = {}
        total = 0
        lol = LengthOfLengthAlphabet('##'+alphabet.name)
        #lengthCode will be used for coding the lengths of the new code
        #we use it for display until now; definition comes below
        lengthCode = LengthAlphabet('#'+alphabet.name)
        lengthIter = iter(lengths)
        lengthsLeft = len(lengths)
        while total<32 and lengthsLeft>0:
            lengthsLeft -= 1
            newSymbol = next(lengthIter)
            lol.description = str(lengthCode[newSymbol])
            length = self.verboseRead(lol)
            if length:
                codeLengths[newSymbol] = length
                total += 32>>length
        if total>32: raise ValueError("Stream format")
        if len(codeLengths)==1: codeLengths[list(codeLengths.keys())[0]] = 0
        #Now set the encoding of the lengthCode
        lengthCode.setLength(codeLengths)
        print("***** Lengths for {} will be coded as:".format(alphabet.name))
        lengthCode.showCode()
        #Now determine the symbol lengths with the lengthCode
        symbolLengths = {}
        total = 0
        lastLength = 8
        alphabetIter = iter(alphabet)
        while total<32768:
            #look ahead to see what is going to happen
            length = lengthCode.decodePeek(
                self.stream.peek(lengthCode.maxLength))[1].index
            #in every branch, set lengthCode.description to explanatory text
            #lengthCode calls format(symbol, extra) with this string
            if length==0:
                symbol = next(alphabetIter)
                lengthCode.description = 'symbol {} unused'.format(symbol)
                self.verboseRead(lengthCode)
                #unused symbol
                continue
            if length==16:
                lengthCode.description = \
                    '{1}+3 symbols of length '+str(lastLength)
                extra = self.verboseRead(lengthCode)
                #scan series of 16s (repeat counts)
                #start with repeat count 2
                repeat = 2
                startSymbol = next(alphabetIter)
                endSymbol = next(alphabetIter)
                symbolLengths[startSymbol.index] = \
                    symbolLengths[endSymbol.index] = lastLength
                #count the two just defined symbols
                total += 2*32768>>lastLength
                #note: loop may end because we're there
                #even if a 16 _appears_ to follow
                while True:
                    #determine last symbol
                    oldRepeat = repeat
                    repeat = (repeat-2<<2)+extra+3
                    #read as many symbols as repeat increased
                    for i in range(oldRepeat, repeat):
                        endSymbol = next(alphabetIter)
                        symbolLengths[endSymbol.index] = lastLength
                    #compute new total; it may be end of loop
                    total += (repeat-oldRepeat)*32768>>lastLength
                    if total>=32768: break
                    #see if there is more to do
                    length = lengthCode.decodePeek(
                        self.stream.peek(lengthCode.maxLength))[1].index
                    if length!=16: break
                    lengthCode.description = 'total {}+{{1}} symbols'.format(
                        (repeat-2<<2)+3)
                    extra = self.verboseRead(lengthCode)
            elif length==17:
                #read, and show explanation
                lengthCode.description = '{1}+3 unused'
                extra = self.verboseRead(lengthCode)
                #scan series of 17s (groups of zero counts)
                #start with repeat count 2
                repeat = 2
                startSymbol = next(alphabetIter)
                endSymbol = next(alphabetIter)
                #note: loop will not end with total==32768,
                #since total doesn't change here
                while True:
                    #determine last symbol
                    oldRepeat = repeat
                    repeat = (repeat-2<<3)+extra+3
                    #read as many symbols as repeat increases
                    for i in range(repeat-oldRepeat):
                        endSymbol = next(alphabetIter)
                    #see if there is more to do
                    length = lengthCode.decodePeek(
                        self.stream.peek(lengthCode.maxLength))[1].index
                    if length!=17: break
                    lengthCode.description = 'total {}+{{1}} unused'.format(
                        (repeat-2<<3)+3)
                    extra = self.verboseRead(lengthCode)
            else:
                symbol = next(alphabetIter)
                #double braces for format
                char = str(symbol)
                if char in '{}': char *= 2
                lengthCode.description = \
                    'Length for {} is {{0.index}} bits'.format(char)
                #output is not needed (will be 0)
                self.verboseRead(lengthCode)
                symbolLengths[symbol.index] = length
                total += 32768>>length
                lastLength = length
        assert total==32768
        alphabet.setLength(symbolLengths)
        print('End of table. Prefix code '+alphabet.name+':')
        alphabet.showCode()
Read complex code