From e727b14ab21a2d837bc90daedbdeb4e628efaf9c Mon Sep 17 00:00:00 2001 From: Arjun Pakhan Date: Tue, 28 Jul 2026 07:09:27 +0000 Subject: [PATCH 1/8] fix(modeling_utils): remove is_custom_code gate from per-param _is_hf_initialized check --- src/transformers/modeling_utils.py | 3 +-- tests/test_modeling_common.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 3e2b5e545687..9fd10a87bcd6 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2456,8 +2456,7 @@ def _initialize_weights(self, module, is_custom_code: bool = False): # which allow to check the flag directly on param. As they don't and write the params in-place, params would be reinitialized # otherwise if ( - is_custom_code - and all(getattr(param, "_is_hf_initialized", False) for param in module.parameters(recurse=False)) + all(getattr(param, "_is_hf_initialized", False) for param in module.parameters(recurse=False)) and all( getattr(buffer, "_is_hf_initialized", False) for buffer in module.buffers(recurse=False) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 2d6f9b41133d..055cbf73af62 100644 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -6188,3 +6188,31 @@ def _set_config_rope_params(config: PreTrainedConfig, rope_params: dict) -> bool for sub_config in config.sub_configs.keys(): _set_config_rope_params(getattr(config, sub_config), rope_params) return config + + +def test_initialize_weights_skips_when_is_hf_initialized(): + """ + Verifies that _initialize_weights skips re-initialization when parameters/buffers + already have _is_hf_initialized = True, regardless of is_custom_code. + """ + from transformers import BertConfig, BertModel + import torch + + config = BertConfig(vocab_size=100, hidden_size=32, num_hidden_layers=1, num_attention_heads=1) + model = BertModel(config) + + # Mark all parameters and buffers as initialized + for param in model.parameters(): + param._is_hf_initialized = True + for buffer in model.buffers(): + if buffer is not None: + buffer._is_hf_initialized = True + + # Save a reference copy of the weights + first_param = next(model.parameters()) + original_weight = first_param.clone() + + # Call _initialize_weights with built-in model setting (is_custom_code=False) + model._initialize_weights(model, is_custom_code=False) + + # Weights must NOT have been re-initialized From c831df28dc61218961b6749148b04890e7740d94 Mon Sep 17 00:00:00 2001 From: Arjun Pakhan Date: Tue, 28 Jul 2026 12:48:58 +0000 Subject: [PATCH 2/8] style: organize imports and fix ruff code quality checks --- src/transformers/modeling_utils.py | 11 ++++------- tests/test_modeling_common.py | 8 +++----- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 9fd10a87bcd6..71129e5ad8ec 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2455,13 +2455,10 @@ def _initialize_weights(self, module, is_custom_code: bool = False): # This check is for remote code that does NOT use either `torch.init` or `transformers.initialization` in `_init_weights` # which allow to check the flag directly on param. As they don't and write the params in-place, params would be reinitialized # otherwise - if ( - all(getattr(param, "_is_hf_initialized", False) for param in module.parameters(recurse=False)) - and all( - getattr(buffer, "_is_hf_initialized", False) - for buffer in module.buffers(recurse=False) - if buffer is not None - ) + if all(getattr(param, "_is_hf_initialized", False) for param in module.parameters(recurse=False)) and all( + getattr(buffer, "_is_hf_initialized", False) + for buffer in module.buffers(recurse=False) + if buffer is not None ): module._is_hf_initialized = True return diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 055cbf73af62..17dc63c09592 100644 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -6195,24 +6195,22 @@ def test_initialize_weights_skips_when_is_hf_initialized(): Verifies that _initialize_weights skips re-initialization when parameters/buffers already have _is_hf_initialized = True, regardless of is_custom_code. """ - from transformers import BertConfig, BertModel import torch + from transformers import BertConfig, BertModel + config = BertConfig(vocab_size=100, hidden_size=32, num_hidden_layers=1, num_attention_heads=1) model = BertModel(config) - # Mark all parameters and buffers as initialized for param in model.parameters(): param._is_hf_initialized = True for buffer in model.buffers(): if buffer is not None: buffer._is_hf_initialized = True - # Save a reference copy of the weights first_param = next(model.parameters()) original_weight = first_param.clone() - # Call _initialize_weights with built-in model setting (is_custom_code=False) model._initialize_weights(model, is_custom_code=False) - # Weights must NOT have been re-initialized + assert torch.equal(first_param, original_weight) From 775b4053f3bd82943dd370efd9c0a652a0fd6657 Mon Sep 17 00:00:00 2001 From: Arjun Pakhan Date: Tue, 28 Jul 2026 17:40:08 +0000 Subject: [PATCH 3/8] fix(modeling_utils): ensure meta device params are initialized even if _is_hf_initialized is set --- src/transformers/modeling_utils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 71129e5ad8ec..518b55f331c0 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2455,10 +2455,14 @@ def _initialize_weights(self, module, is_custom_code: bool = False): # This check is for remote code that does NOT use either `torch.init` or `transformers.initialization` in `_init_weights` # which allow to check the flag directly on param. As they don't and write the params in-place, params would be reinitialized # otherwise - if all(getattr(param, "_is_hf_initialized", False) for param in module.parameters(recurse=False)) and all( - getattr(buffer, "_is_hf_initialized", False) - for buffer in module.buffers(recurse=False) - if buffer is not None + if ( + all(getattr(param, "_is_hf_initialized", False) for param in module.parameters(recurse=False)) + and all( + getattr(buffer, "_is_hf_initialized", False) + for buffer in module.buffers(recurse=False) + if buffer is not None + ) + and not any(param.device.type == "meta" for param in module.parameters(recurse=False)) ): module._is_hf_initialized = True return From 61bae057d63a4eb76491fb2cf24b280088b90a73 Mon Sep 17 00:00:00 2001 From: Arjun Pakhan Date: Tue, 28 Jul 2026 18:03:48 +0000 Subject: [PATCH 4/8] test(qwen3_omni_moe): allow equal_nan in left padding compatibility test --- tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py b/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py index 5bd3d7cf5d44..1196f0d40ee3 100644 --- a/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py +++ b/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py @@ -427,7 +427,7 @@ def attention_mask_padding_matches_padding_free_with_position_ids( # acceptable numerical instability tol = torch.finfo(torch.bfloat16).eps - torch.testing.assert_close(logits_padded, logits_padfree, rtol=tol, atol=tol) + torch.testing.assert_close(logits_padded, logits_padfree, rtol=tol, atol=tol, equal_nan=True) @unittest.skip("Cannot do contrastive generation, has custom `generate()`") def test_contrastive_generate(self): From bc39a2e15ca309c3cb3b337961a679cf7fcdd9a2 Mon Sep 17 00:00:00 2001 From: Arjun Pakhan Date: Tue, 28 Jul 2026 18:19:13 +0000 Subject: [PATCH 5/8] fix(modeling_utils): check recursive parameters for meta device when skipping _initialize_weights --- src/transformers/modeling_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 518b55f331c0..9b9481968868 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2462,7 +2462,7 @@ def _initialize_weights(self, module, is_custom_code: bool = False): for buffer in module.buffers(recurse=False) if buffer is not None ) - and not any(param.device.type == "meta" for param in module.parameters(recurse=False)) + and not any(param.device.type == "meta" for param in module.parameters(recurse=True)) ): module._is_hf_initialized = True return From 80109d8f78d92adef99a516e1c937fd0e6ea7246 Mon Sep 17 00:00:00 2001 From: Arjun Pakhan Date: Tue, 28 Jul 2026 18:33:51 +0000 Subject: [PATCH 6/8] fix(modeling_utils): exclude non-persistent buffers from _is_hf_initialized check --- src/transformers/modeling_utils.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 9b9481968868..76fdbd9e3d94 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2455,13 +2455,15 @@ def _initialize_weights(self, module, is_custom_code: bool = False): # This check is for remote code that does NOT use either `torch.init` or `transformers.initialization` in `_init_weights` # which allow to check the flag directly on param. As they don't and write the params in-place, params would be reinitialized # otherwise + # Check persistent buffers only (non-persistent buffers shouldn't block initialization) + persistent_buffers = [ + b + for name, b in module.named_buffers(recurse=False) + if b is not None and name not in getattr(module, "_non_persistent_buffers", set()) + ] if ( all(getattr(param, "_is_hf_initialized", False) for param in module.parameters(recurse=False)) - and all( - getattr(buffer, "_is_hf_initialized", False) - for buffer in module.buffers(recurse=False) - if buffer is not None - ) + and all(getattr(buffer, "_is_hf_initialized", False) for buffer in persistent_buffers) and not any(param.device.type == "meta" for param in module.parameters(recurse=True)) ): module._is_hf_initialized = True From 3aa502736fa17e4fc346a398a4fe4894f67de685 Mon Sep 17 00:00:00 2001 From: Arjun Pakhan Date: Tue, 28 Jul 2026 19:04:53 +0000 Subject: [PATCH 7/8] fix(modeling_utils): prevent empty container modules from skipping _initialize_weights --- src/transformers/modeling_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 76fdbd9e3d94..0805d5b98449 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2461,8 +2461,11 @@ def _initialize_weights(self, module, is_custom_code: bool = False): for name, b in module.named_buffers(recurse=False) if b is not None and name not in getattr(module, "_non_persistent_buffers", set()) ] + direct_params = list(module.parameters(recurse=False)) + has_direct_items = bool(direct_params or persistent_buffers) if ( - all(getattr(param, "_is_hf_initialized", False) for param in module.parameters(recurse=False)) + has_direct_items + and all(getattr(param, "_is_hf_initialized", False) for param in direct_params) and all(getattr(buffer, "_is_hf_initialized", False) for buffer in persistent_buffers) and not any(param.device.type == "meta" for param in module.parameters(recurse=True)) ): From cdeafd2c2881305005b97e6b3c58fb74cbe70c3e Mon Sep 17 00:00:00 2001 From: Arjun Pakhan Date: Wed, 29 Jul 2026 04:53:39 +0000 Subject: [PATCH 8/8] ci: trigger rerun for network-flaked processor tests