def tokenise(pattern: TOKEN_REGEXP, exclude: nil)
      filter_proc = filter_to_proc(exclude)
      @input.scan(pattern).map(&:downcase).reject { |token| filter_proc.call(token) }
    end
def tokenise(pattern: TOKEN_REGEXP, exclude: nil)
      filter_proc = filter_to_proc(exclude)
      @input.scan(pattern).map(&:downcase).reject { |token| filter_proc.call(token) }
    end
Initialises state with the string to be tokenised.

 @param [String] input   The string to tokenise
 Converts a string into an array of tokens using a regular expression.
 If a regexp is not provided a default one is used. See `Tokenizer.TOKEN_REGEXP`.

 Use `exclude` to remove tokens from the final list. `exclude` can be a string,
 a regular expression, a lambda, a symbol, or an array of one or more of those types.
 This allows for powerful and flexible tokenisation strategies.

 If a symbol is passed, it must name a predicate method.

 @example
  WordsCounted::Tokeniser.new("Hello World").tokenise
  # => ['hello', 'world']

 @example With `pattern`
  WordsCounted::Tokeniser.new("Hello-Mohamad").tokenise(pattern: /[^-]+/)
  # => ['hello', 'mohamad']

 @example With `exclude` as a string
  WordsCounted::Tokeniser.new("Hello Sami").tokenise(exclude: "hello")
  # => ['sami']

 @example With `exclude` as a regexp
  WordsCounted::Tokeniser.new("Hello Dani").tokenise(exclude: /hello/i)
  # => ['dani']

 @example With `exclude` as a lambda
  WordsCounted::Tokeniser.new("Goodbye Sami").tokenise(
    exclude: ->(token) { token.length > 6 }
  )
  # => ['sami']

 @example With `exclude` as a symbol
  WordsCounted::Tokeniser.new("Hello محمد").tokenise(exclude: :ascii_only?)
  # => ['محمد']

 @example With `exclude` as an array of strings
  WordsCounted::Tokeniser.new("Goodbye Sami and hello Dani").tokenise(
    exclude: ["goodbye hello"]
  )
  # => ['sami', 'and', dani']

 @example With `exclude` as an array of regular expressions
  WordsCounted::Tokeniser.new("Goodbye and hello Dani").tokenise(
    exclude: [/goodbye/i, /and/i]
  )
  # => ['hello', 'dani']

 @example With `exclude` as an array of lambdas
  t = WordsCounted::Tokeniser.new("Special Agent 007")
  t.tokenise(
    exclude: [
      ->(t) { t.to_i.odd? },
      ->(t) { t.length > 5}
    ]
  )
  # => ['agent']

 @example With `exclude` as a mixed array
  t = WordsCounted::Tokeniser.new("Hello! اسماءنا هي محمد، كارولينا، سامي، وداني")
  t.tokenise(
    exclude: [
      :ascii_only?,
      /محمد/,
      ->(t) { t.length > 6},
      "و"
    ]
  )
  # => ["هي", "سامي", "وداني"]

 @param [Regexp] pattern   The string to tokenise
 @param [Array<String, Regexp, Lambda, Symbol>, String, Regexp, Lambda, Symbol, nil] exclude     The filter to apply
 @return [Array] The array of filtered tokens