Parquet stream-conversion fails to embed images/audio files from gated repos
e.g. for https://huggingface.co/datasets/SinKove/synthetic_mammography_csaw

it seems to be an issue with `datasets` not passing the token to `embed_table_storage` when generating a dataset 

```json
{
    "error": "An error occurred while generating the dataset",
    "cause_exception": "DatasetGenerationError",
    "cause_message": "An error occurred while generating the dataset",
    "cause_traceback": [
        "Traceback (most recent call last):\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/utils/_errors.py\", line 261, in hf_raise_for_status\n response.raise_for_status()\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/requests/models.py\", line 1021, in raise_for_status\n raise HTTPError(http_error_msg, response=self)\n",
        "requests.exceptions.HTTPError: 401 Client Error: Unauthorized for url: https://huggingface.co/api/datasets/SinKove/synthetic_mammography_csaw/revision/490f9d72f59e2599b5d1eca5973b37a913270445\n",
        "\nThe above exception was the direct cause of the following exception:\n\n",
        "Traceback (most recent call last):\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/hf_file_system.py\", line 100, in _repo_and_revision_exist\n self._api.repo_info(repo_id, revision=revision, repo_type=repo_type)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py\", line 118, in _inner_fn\n return fn(*args, **kwargs)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/hf_api.py\", line 1868, in repo_info\n return method(\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py\", line 118, in _inner_fn\n return fn(*args, **kwargs)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/hf_api.py\", line 1741, in dataset_info\n hf_raise_for_status(r)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/utils/_errors.py\", line 277, in hf_raise_for_status\n raise GatedRepoError(message, response) from e\n",
        "huggingface_hub.utils._errors.GatedRepoError: 401 Client Error. (Request ID: Root=1-6538ec77-55cac38800ae3763321e921e;1f6f0b56-19bb-4ea0-bfd6-0a1081822277)\n\nCannot access gated repo for url https://huggingface.co/api/datasets/SinKove/synthetic_mammography_csaw/revision/490f9d72f59e2599b5d1eca5973b37a913270445.\nRepo dataset SinKove/synthetic_mammography_csaw is gated. You must be authenticated to access it.\n",
        "\nThe above exception was the direct cause of the following exception:\n\n",
        "Traceback (most recent call last):\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/builder.py\", line 1703, in _prepare_split_single\n num_examples, num_bytes = writer.finalize()\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/arrow_writer.py\", line 586, in finalize\n self.write_examples_on_file()\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/arrow_writer.py\", line 448, in write_examples_on_file\n self.write_batch(batch_examples=batch_examples)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/arrow_writer.py\", line 559, in write_batch\n self.write_table(pa_table, writer_batch_size)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/arrow_writer.py\", line 574, in write_table\n pa_table = embed_table_storage(pa_table)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/table.py\", line 2306, in embed_table_storage\n arrays = [\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/table.py\", line 2307, in <listcomp>\n embed_array_storage(table[name], feature) if require_storage_embed(feature) else table[name]\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/table.py\", line 1831, in wrapper\n return pa.chunked_array([func(chunk, *args, **kwargs) for chunk in array.chunks])\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/table.py\", line 1831, in <listcomp>\n return pa.chunked_array([func(chunk, *args, **kwargs) for chunk in array.chunks])\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/table.py\", line 2176, in embed_array_storage\n return feature.embed_storage(array)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/features/image.py\", line 266, in embed_storage\n [\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/features/image.py\", line 267, in <listcomp>\n (path_to_bytes(x[\"path\"]) if x[\"bytes\"] is None else x[\"bytes\"]) if x is not None else None\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/utils/py_utils.py\", line 306, in wrapper\n return func(value) if value is not None else None\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/features/image.py\", line 261, in path_to_bytes\n with xopen(path, \"rb\") as f:\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/download/streaming_download_manager.py\", line 496, in xopen\n file_obj = fsspec.open(file, mode=mode, *args, **kwargs).open()\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/core.py\", line 439, in open\n return open_files(\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/core.py\", line 282, in open_files\n fs, fs_token, paths = get_fs_token_paths(\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/core.py\", line 606, in get_fs_token_paths\n fs = filesystem(protocol, **inkwargs)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/registry.py\", line 261, in filesystem\n return cls(**storage_options)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/spec.py\", line 76, in __call__\n obj = super().__call__(*args, **kwargs)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/implementations/zip.py\", line 58, in __init__\n self.fo = fo.__enter__() # the whole instance is a context\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/core.py\", line 102, in __enter__\n f = self.fs.open(self.path, mode=mode)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/spec.py\", line 1199, in open\n f = self._open(\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/hf_file_system.py\", line 200, in _open\n return HfFileSystemFile(self, path, mode=mode, revision=revision, **kwargs)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/hf_file_system.py\", line 392, in __init__\n super().__init__(fs, path, **kwargs)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/spec.py\", line 1555, in __init__\n self.size = self.details[\"size\"]\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/spec.py\", line 1568, in details\n self._details = self.fs.info(self.path)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/hf_file_system.py\", line 351, in info\n resolved_path = self.resolve_path(path)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/hf_file_system.py\", line 149, in resolve_path\n raise FileNotFoundError(path) from err\n",
        "FileNotFoundError: SinKove/synthetic_mammography_csaw@490f9d72f59e2599b5d1eca5973b37a913270445/data/train.zip\n",
        "\nThe above exception was the direct cause of the following exception:\n\n",
        "Traceback (most recent call last):\n",
        " File \"/src/services/worker/src/worker/job_manager.py\", line 168, in process\n job_result = self.job_runner.compute()\n",
        " File \"/src/services/worker/src/worker/job_runners/config/parquet_and_info.py\", line 1348, in compute\n compute_config_parquet_and_info_response(\n",
        " File \"/src/services/worker/src/worker/job_runners/config/parquet_and_info.py\", line 1255, in compute_config_parquet_and_info_response\n parquet_operations, partial = stream_convert_to_parquet(\n",
        " File \"/src/services/worker/src/worker/job_runners/config/parquet_and_info.py\", line 852, in stream_convert_to_parquet\n builder._prepare_split(\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/builder.py\", line 1555, in _prepare_split\n for job_id, done, content in self._prepare_split_single(\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/builder.py\", line 1712, in _prepare_split_single\n raise DatasetGenerationError(\"An error occurred while generating the dataset\") from e\n",
        "datasets.builder.DatasetGenerationError: An error occurred while generating the dataset\n"
    ]
}
```
Parquet stream-conversion fails to embed images/audio files from gated repos: e.g. for https://huggingface.co/datasets/SinKove/synthetic_mammography_csaw

it seems to be an issue with `datasets` not passing the token to `embed_table_storage` when generating a dataset 

```json
{
    "error": "An error occurred while generating the dataset",
    "cause_exception": "DatasetGenerationError",
    "cause_message": "An error occurred while generating the dataset",
    "cause_traceback": [
        "Traceback (most recent call last):\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/utils/_errors.py\", line 261, in hf_raise_for_status\n response.raise_for_status()\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/requests/models.py\", line 1021, in raise_for_status\n raise HTTPError(http_error_msg, response=self)\n",
        "requests.exceptions.HTTPError: 401 Client Error: Unauthorized for url: https://huggingface.co/api/datasets/SinKove/synthetic_mammography_csaw/revision/490f9d72f59e2599b5d1eca5973b37a913270445\n",
        "\nThe above exception was the direct cause of the following exception:\n\n",
        "Traceback (most recent call last):\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/hf_file_system.py\", line 100, in _repo_and_revision_exist\n self._api.repo_info(repo_id, revision=revision, repo_type=repo_type)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py\", line 118, in _inner_fn\n return fn(*args, **kwargs)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/hf_api.py\", line 1868, in repo_info\n return method(\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py\", line 118, in _inner_fn\n return fn(*args, **kwargs)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/hf_api.py\", line 1741, in dataset_info\n hf_raise_for_status(r)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/utils/_errors.py\", line 277, in hf_raise_for_status\n raise GatedRepoError(message, response) from e\n",
        "huggingface_hub.utils._errors.GatedRepoError: 401 Client Error. (Request ID: Root=1-6538ec77-55cac38800ae3763321e921e;1f6f0b56-19bb-4ea0-bfd6-0a1081822277)\n\nCannot access gated repo for url https://huggingface.co/api/datasets/SinKove/synthetic_mammography_csaw/revision/490f9d72f59e2599b5d1eca5973b37a913270445.\nRepo dataset SinKove/synthetic_mammography_csaw is gated. You must be authenticated to access it.\n",
        "\nThe above exception was the direct cause of the following exception:\n\n",
        "Traceback (most recent call last):\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/builder.py\", line 1703, in _prepare_split_single\n num_examples, num_bytes = writer.finalize()\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/arrow_writer.py\", line 586, in finalize\n self.write_examples_on_file()\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/arrow_writer.py\", line 448, in write_examples_on_file\n self.write_batch(batch_examples=batch_examples)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/arrow_writer.py\", line 559, in write_batch\n self.write_table(pa_table, writer_batch_size)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/arrow_writer.py\", line 574, in write_table\n pa_table = embed_table_storage(pa_table)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/table.py\", line 2306, in embed_table_storage\n arrays = [\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/table.py\", line 2307, in <listcomp>\n embed_array_storage(table[name], feature) if require_storage_embed(feature) else table[name]\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/table.py\", line 1831, in wrapper\n return pa.chunked_array([func(chunk, *args, **kwargs) for chunk in array.chunks])\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/table.py\", line 1831, in <listcomp>\n return pa.chunked_array([func(chunk, *args, **kwargs) for chunk in array.chunks])\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/table.py\", line 2176, in embed_array_storage\n return feature.embed_storage(array)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/features/image.py\", line 266, in embed_storage\n [\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/features/image.py\", line 267, in <listcomp>\n (path_to_bytes(x[\"path\"]) if x[\"bytes\"] is None else x[\"bytes\"]) if x is not None else None\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/utils/py_utils.py\", line 306, in wrapper\n return func(value) if value is not None else None\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/features/image.py\", line 261, in path_to_bytes\n with xopen(path, \"rb\") as f:\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/download/streaming_download_manager.py\", line 496, in xopen\n file_obj = fsspec.open(file, mode=mode, *args, **kwargs).open()\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/core.py\", line 439, in open\n return open_files(\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/core.py\", line 282, in open_files\n fs, fs_token, paths = get_fs_token_paths(\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/core.py\", line 606, in get_fs_token_paths\n fs = filesystem(protocol, **inkwargs)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/registry.py\", line 261, in filesystem\n return cls(**storage_options)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/spec.py\", line 76, in __call__\n obj = super().__call__(*args, **kwargs)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/implementations/zip.py\", line 58, in __init__\n self.fo = fo.__enter__() # the whole instance is a context\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/core.py\", line 102, in __enter__\n f = self.fs.open(self.path, mode=mode)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/spec.py\", line 1199, in open\n f = self._open(\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/hf_file_system.py\", line 200, in _open\n return HfFileSystemFile(self, path, mode=mode, revision=revision, **kwargs)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/hf_file_system.py\", line 392, in __init__\n super().__init__(fs, path, **kwargs)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/spec.py\", line 1555, in __init__\n self.size = self.details[\"size\"]\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/fsspec/spec.py\", line 1568, in details\n self._details = self.fs.info(self.path)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/hf_file_system.py\", line 351, in info\n resolved_path = self.resolve_path(path)\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/huggingface_hub/hf_file_system.py\", line 149, in resolve_path\n raise FileNotFoundError(path) from err\n",
        "FileNotFoundError: SinKove/synthetic_mammography_csaw@490f9d72f59e2599b5d1eca5973b37a913270445/data/train.zip\n",
        "\nThe above exception was the direct cause of the following exception:\n\n",
        "Traceback (most recent call last):\n",
        " File \"/src/services/worker/src/worker/job_manager.py\", line 168, in process\n job_result = self.job_runner.compute()\n",
        " File \"/src/services/worker/src/worker/job_runners/config/parquet_and_info.py\", line 1348, in compute\n compute_config_parquet_and_info_response(\n",
        " File \"/src/services/worker/src/worker/job_runners/config/parquet_and_info.py\", line 1255, in compute_config_parquet_and_info_response\n parquet_operations, partial = stream_convert_to_parquet(\n",
        " File \"/src/services/worker/src/worker/job_runners/config/parquet_and_info.py\", line 852, in stream_convert_to_parquet\n builder._prepare_split(\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/builder.py\", line 1555, in _prepare_split\n for job_id, done, content in self._prepare_split_single(\n",
        " File \"/src/services/worker/.venv/lib/python3.9/site-packages/datasets/builder.py\", line 1712, in _prepare_split_single\n raise DatasetGenerationError(\"An error occurred while generating the dataset\") from e\n",
        "datasets.builder.DatasetGenerationError: An error occurred while generating the dataset\n"
    ]
}
```