def add_page(url)
      unless robot_txt_allowed?(url)
        @scanned_pages << url
        return nil
      end
      page = Page.new(url)
      @pages << page
      @scanned_pages << url
    end
def add_page(url)
      unless robot_txt_allowed?(url)
        @scanned_pages << url
        return nil
      end
      page = Page.new(url)
      @pages << page
      @scanned_pages << url
    end
create Page and add to to site