sample held out workloads

Author: priyakasimbeg

scoring/generate_held_out_workloads.py

@@ -9,7 +9,8 @@
 
 flags.DEFINE_integer('held_out_workloads_seed',
                      None,
-                     'Random seed for scoring.')
+                     'Random seed for scoring.'
+                     'AlgoPerf v0.5 seed: 3438810845')
 flags.DEFINE_string('output_filename',
                     'held_out_workloads.json',
                     'Path to file to record sampled held_out workloads.')
@@ -19,7 +20,10 @@
     'librispeech': [
         'librispeech_conformer_attention_temperature',
         'librispeech_conformer_layernorm',
-        'librispeech_conformer_gelu'
+        # 'librispeech_conformer_gelu', # Removed due to bug in target setting procedure
+        'librispeech_deepspeech_no_resnet',
+        'librispeech_deepspeech_norm_and_spec_aug',
+        'librispeech_deepspeech_tanh'
     ],
     'imagenet': [
         'imagenet_resnet_silu',

scoring/held_out_workloads_algoperf_v05.json

@@ -0,0 +1 @@
+["librispeech_conformer_layernorm", "imagenet_resnet_large_bn_init", "ogbg_model_size", "wmt_glu_tanh", "fastmri_tanh", "criteo1tb_embed_init"]