def lex!
			r = Array.new
			@buffer.gsub!(/(?:\r\n|\n\r)/, "\n")
			while @code_start_re =~ @buffer
				r << [:string, $`] unless $`.empty?
				if CODE_EQUIVALENT.has_key?($&)
					CODE_EQUIVALENT[$&].each do |c|
						r << [:code, c]
					end
					@buffer = $'
				else
					csi = $&
					residual = $'
					if PARAMETER_AND_LETTER =~ residual
						r << [:code, $&]
						@buffer = $'
					else
						@buffer = csi + residual
						return r
					end
				end
			end
			r << [:string, @buffer] unless @buffer.empty?
			@buffer = ''
			return r
		end
def lex!
			r = Array.new
			@buffer.gsub!(/(?:\r\n|\n\r)/, "\n")
			while @code_start_re =~ @buffer
				r << [:string, $`] unless $`.empty?
				if CODE_EQUIVALENT.has_key?($&)
					CODE_EQUIVALENT[$&].each do |c|
						r << [:code, c]
					end
					@buffer = $'
				else
					csi = $&
					residual = $'
					if PARAMETER_AND_LETTER =~ residual
						r << [:code, $&]
						@buffer = $'
					else
						@buffer = csi + residual
						return r
					end
				end
			end
			r << [:string, @buffer] unless @buffer.empty?
			@buffer = ''
			return r
		end
returns array of tokens while deleting the tokenized part from buffer