diff --git a/mlperf_logging/compliance_checker/training_6.1.0/open_common.yaml b/mlperf_logging/compliance_checker/training_6.1.0/open_common.yaml index 4b7761e..ead4c00 100644 --- a/mlperf_logging/compliance_checker/training_6.1.0/open_common.yaml +++ b/mlperf_logging/compliance_checker/training_6.1.0/open_common.yaml @@ -2,5 +2,5 @@ - KEY: NAME: submission_benchmark REQ: EXACTLY_ONE - CHECK: " v['value'] in ['flux1', 'llama31_8b', 'llama2_70b_lora', 'gpt_oss_20b', 'deepseekv3_671b'] " + CHECK: " v['value'] in ['flux1', 'llama31_8b', 'llama2_70b_lora', 'gpt_oss_20b', 'deepseekv3_671b', 'qwen35_397b_grpo'] " POST: " enqueue_config('training_6.1.0/open_{}.yaml'.format(v['value'])) " diff --git a/mlperf_logging/compliance_checker/training_6.1.0/open_qwen35_397b_grpo.yaml b/mlperf_logging/compliance_checker/training_6.1.0/open_qwen35_397b_grpo.yaml new file mode 100644 index 0000000..0282a32 --- /dev/null +++ b/mlperf_logging/compliance_checker/training_6.1.0/open_qwen35_397b_grpo.yaml @@ -0,0 +1,94 @@ +- KEY: + NAME: global_batch_size + REQ: EXACTLY_ONE + CHECK: " is_integer(v['value']) and v['value'] > 0 " + +- KEY: + NAME: max_sequence_length + REQ: EXACTLY_ONE + CHECK: " v['value'] == 65536 " + +- KEY: + NAME: eval_samples + REQ: EXACTLY_ONE + CHECK: " v['value'] == 256 " + +- KEY: + NAME: init_checkpoint_step + REQ: EXACTLY_ONE + CHECK: " v['value'] == 0 " + +- KEY: + NAME: max_steps + REQ: EXACTLY_ONE + +- KEY: + NAME: opt_name + REQ: EXACTLY_ONE + CHECK: " v['value'] == 'adamw' " + +- KEY: + NAME: opt_base_learning_rate + REQ: EXACTLY_ONE + +- KEY: + NAME: opt_adamw_beta_1 + REQ: EXACTLY_ONE + +- KEY: + NAME: opt_adamw_beta_2 + REQ: EXACTLY_ONE + +- KEY: + NAME: opt_adamw_epsilon + REQ: EXACTLY_ONE + +- KEY: + NAME: opt_adamw_weight_decay + REQ: EXACTLY_ONE + +- KEY: + NAME: opt_gradient_clip_norm + REQ: EXACTLY_ONE + +- KEY: + NAME: opt_learning_rate_warmup_steps + REQ: EXACTLY_ONE + +- KEY: + NAME: opt_learning_rate_decay_schedule + REQ: EXACTLY_ONE + +- KEY: + NAME: generation_training_rollout_temperature + REQ: EXACTLY_ONE + +- KEY: + NAME: generation_training_rollout_top_p + REQ: EXACTLY_ONE + +- KEY: + NAME: generation_validation_rollout_temperature + REQ: EXACTLY_ONE + +- KEY: + NAME: generation_validation_rollout_top_p + REQ: EXACTLY_ONE + +- KEY: + NAME: num_prompts_per_step + REQ: EXACTLY_ONE + CHECK: " is_integer(v['value']) and v['value'] > 0 " + +- KEY: + NAME: num_generations_per_prompt + REQ: EXACTLY_ONE + CHECK: " is_integer(v['value']) and v['value'] > 0 " + +- KEY: + NAME: eval_accuracy + REQ: AT_LEAST_ONE + CHECK: + - "'samples_count' in v['metadata']" + - "0.0 <= v['value'] <= 1.0" + ATLEAST_ONE_CHECK: "v['value'] > 0.69" diff --git a/mlperf_logging/mllog/constants.py b/mlperf_logging/mllog/constants.py index 937c282..be832b6 100644 --- a/mlperf_logging/mllog/constants.py +++ b/mlperf_logging/mllog/constants.py @@ -58,6 +58,7 @@ LLAMA31_8B = "llama31_8b" FLUX1 = "flux1" DEEPSEEKV3_671B = "deepseekv3_671b" +QWEN35_397B_GRPO = "qwen35_397b_grpo" # Constant values - model info ADAGRAD = "adagrad" @@ -178,6 +179,12 @@ INIT_CHECKPOINT_STEP = "init_checkpoint_step" LORA_ALPHA = "lora_alpha" MOE_AUX_LOSS_COEFF = "moe_aux_loss_coeff" +GENERATION_TRAINING_ROLLOUT_TEMPERATURE = "generation_training_rollout_temperature" +GENERATION_TRAINING_ROLLOUT_TOP_P = "generation_training_rollout_top_p" +GENERATION_VALIDATION_ROLLOUT_TEMPERATURE = "generation_validation_rollout_temperature" +GENERATION_VALIDATION_ROLLOUT_TOP_P = "generation_validation_rollout_top_p" +NUM_GENERATIONS_PER_PROMPT = "num_generations_per_prompt" +NUM_PROMPTS_PER_STEP = "num_prompts_per_step" # Log keys - misc. BBOX = "bbox" SEGM = "segm" diff --git a/mlperf_logging/result_summarizer/config.yaml b/mlperf_logging/result_summarizer/config.yaml index 6b8779f..fca0274 100644 --- a/mlperf_logging/result_summarizer/config.yaml +++ b/mlperf_logging/result_summarizer/config.yaml @@ -117,6 +117,7 @@ columns: llama31_8b: ["Benchmark results (minutes)", "Small LLM", "C4", "Llama31-8b"] gpt_oss_20b: ["Benchmark results (minutes)", "LLM", "C4", "GPT-OSS-20B"] deepseekv3_671b: ["Benchmark results (minutes)", "LLM", "C4", "DeepSeekV3-671B"] + qwen35_397b_grpo: ["Benchmark results (minutes)", "LLM post training", "R2E-Gym-Subset (easy split)", "Qwen3.5-397B-A17B"] default: [" ", " ", " "] hpc: