Set huggingface libraries to load finewebedu data

Author: aahladc

algoperf/_version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '1.0.1'
+__version_tuple__ = version_tuple = (1, 0, 1)
+
+__commit_id__ = commit_id = 'ga673f5835'

algoperf/workloads/finewebedu_lm/input_pipeline.py

@@ -6,6 +6,7 @@
 
 import jax
 import tensorflow as tf
+import datasets as hf_datasets
 
 from algoperf import data_utils
 
@@ -83,7 +84,8 @@ def get_lm_dataset(
   shuffle_seed = jax.random.randint(data_rng, (), -(2**31), 2**31 - 1)
 
   data_dir = os.path.join(data_dir, TFDS_SPLIT_NAME[split])
-  tokens_ds = tf.data.Dataset.load(data_dir)
+  ds = hf_datasets.load_from_disk(data_dir)
+  tokens_ds = ds.to_tf_dataset()
 
   # tokens
   tokens_ds = tokens_ds.flat_map(tf.data.Dataset.from_tensor_slices)

scoring/utils/slurm/run_jobs.sh

@@ -91,6 +91,7 @@ DOCKER_CMD=(
   docker run
   -v /opt/data/:/data/
   -v "$HOME/experiment_runs:/experiment_runs"
+  -v "$HOME/algorithmic-efficiency/:/algorithmic-efficiency/"
   -v "$HOME/submissions_algorithms/:/algorithmic-efficiency/submissions_algorithms"
   -v "$HOME/algorithmic-efficiency/docker/scripts/startup.sh:/algorithmic-efficiency/docker/scripts/startup.sh"
   --gpus all