def validate_tokens(lexer)
      if lexer.lex_tokens.empty?
        complain("No token list defined")
      end
      if !lexer.lex_tokens.respond_to?(:to_ary)
        complain("Tokens must be a list or enumerable")
      end

      terminals = []
      lexer.lex_tokens.each do |token|
        if !identifier?(token)
          complain("Bad token name `#{token}`")
        end
        if terminals.include?(token)
          complain("Token `#{token}` already defined")
        end
        terminals << token
      end
    end
def validate_tokens(lexer)
      if lexer.lex_tokens.empty?
        complain("No token list defined")
      end
      if !lexer.lex_tokens.respond_to?(:to_ary)
        complain("Tokens must be a list or enumerable")
      end

      terminals = []
      lexer.lex_tokens.each do |token|
        if !identifier?(token)
          complain("Bad token name `#{token}`")
        end
        if terminals.include?(token)
          complain("Token `#{token}` already defined")
        end
        terminals << token
      end
    end
Validate provided tokens

 @api private