From 7d859ee70036e22b14eabeb127159a35d9513847 Mon Sep 17 00:00:00 2001 From: taking-lying-flat <1615405@qq.com> Date: Wed, 16 Sep 2026 22:28:19 +0800 Subject: [PATCH 1/3] fix(lisa): select language layers in multimodal models Signed-off-by: taking-lying-flat <1615405@qq.com> --- swift/callbacks/lisa.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/swift/callbacks/lisa.py b/swift/callbacks/lisa.py index e44896336a..e16f8a80e4 100644 --- a/swift/callbacks/lisa.py +++ b/swift/callbacks/lisa.py @@ -1,6 +1,7 @@ # Copyright (c) ModelScope Contributors. All rights reserved. import numpy as np import torch +from accelerate.utils import extract_model_from_parallel from typing import TYPE_CHECKING from .base import TrainerCallback @@ -19,10 +20,21 @@ def __init__(self, args: 'TrainingArguments', trainer: 'Trainer'): def on_train_begin(self, args, state, control, **kwargs): # Wait until the model exists and the optimizer has registered all layers. - self.model = kwargs['model'] + self.model = extract_model_from_parallel(kwargs['model'], keep_torch_compile=False) + model_arch = getattr(getattr(self.model, 'model_meta', None), 'model_arch', None) + layer_prefixes = getattr(model_arch, 'language_model', None) + if getattr(model_arch, 'module_list', None): + layer_prefixes = [model_arch.module_list] + excluded_prefixes = [ + prefix for key in ('vision_tower', 'aligner', 'generator') for prefix in getattr(model_arch, key, []) + ] layers_name = None layers = None for name, module in self.model.named_modules(): + if layer_prefixes and not any(name == prefix or name.startswith(f'{prefix}.') for prefix in layer_prefixes): + continue + if any(name == prefix or name.startswith(f'{prefix}.') for prefix in excluded_prefixes): + continue if isinstance(module, torch.nn.ModuleList): layers_name = name layers = module From 897b0d3da6697965f6cb6de9b3205a0cf1215ec3 Mon Sep 17 00:00:00 2001 From: taking-lying-flat <1615405@qq.com> Date: Thu, 17 Sep 2026 06:52:49 +0800 Subject: [PATCH 2/3] fix(lisa): reject missing or ambiguous layer mappings Signed-off-by: taking-lying-flat <1615405@qq.com> --- swift/callbacks/lisa.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/swift/callbacks/lisa.py b/swift/callbacks/lisa.py index e16f8a80e4..00f1c85273 100644 --- a/swift/callbacks/lisa.py +++ b/swift/callbacks/lisa.py @@ -21,27 +21,33 @@ def __init__(self, args: 'TrainingArguments', trainer: 'Trainer'): def on_train_begin(self, args, state, control, **kwargs): # Wait until the model exists and the optimizer has registered all layers. self.model = extract_model_from_parallel(kwargs['model'], keep_torch_compile=False) - model_arch = getattr(getattr(self.model, 'model_meta', None), 'model_arch', None) + model_meta = getattr(self.model, 'model_meta', None) + model_arch = getattr(model_meta, 'model_arch', None) + module_list = getattr(model_arch, 'module_list', None) layer_prefixes = getattr(model_arch, 'language_model', None) - if getattr(model_arch, 'module_list', None): - layer_prefixes = [model_arch.module_list] + if module_list: + layer_prefixes = [module_list] + if getattr(model_meta, 'is_multimodal', False) and not layer_prefixes: + raise ValueError('LISA requires model_arch.module_list or language_model for multimodal models.') excluded_prefixes = [ prefix for key in ('vision_tower', 'aligner', 'generator') for prefix in getattr(model_arch, key, []) ] - layers_name = None - layers = None + layer_names = [] for name, module in self.model.named_modules(): + if not isinstance(module, torch.nn.ModuleList) or (module_list and name != module_list): + continue if layer_prefixes and not any(name == prefix or name.startswith(f'{prefix}.') for prefix in layer_prefixes): continue if any(name == prefix or name.startswith(f'{prefix}.') for prefix in excluded_prefixes): continue - if isinstance(module, torch.nn.ModuleList): - layers_name = name - layers = module - break - assert layers_name is not None - self.layers_attribute = layers_name - self.total_layers = len(layers) + # Nested lists (e.g. MoE experts) belong to the enclosing layer stack. + if not any(not parent or name.startswith(f'{parent}.') for parent in layer_names): + layer_names.append(name) + if len(layer_names) != 1: + raise ValueError('LISA could not uniquely identify the layer list. ' + f'Set model_arch.module_list explicitly. Candidates: {layer_names}') + self.layers_attribute = layer_names[0] + self.total_layers = len(self.model.get_submodule(self.layers_attribute)) self.active_layers_indices = [] self.switch_active_layers() From d93b3f64b78a306bd4dc31186be9ce781cf57e31 Mon Sep 17 00:00:00 2001 From: taking-lying-flat <1615405@qq.com> Date: Thu, 17 Sep 2026 06:57:45 +0800 Subject: [PATCH 3/3] fix(lisa): preserve fallback outside registered model scopes Signed-off-by: taking-lying-flat <1615405@qq.com> --- swift/callbacks/lisa.py | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/swift/callbacks/lisa.py b/swift/callbacks/lisa.py index 00f1c85273..e16f8a80e4 100644 --- a/swift/callbacks/lisa.py +++ b/swift/callbacks/lisa.py @@ -21,33 +21,27 @@ def __init__(self, args: 'TrainingArguments', trainer: 'Trainer'): def on_train_begin(self, args, state, control, **kwargs): # Wait until the model exists and the optimizer has registered all layers. self.model = extract_model_from_parallel(kwargs['model'], keep_torch_compile=False) - model_meta = getattr(self.model, 'model_meta', None) - model_arch = getattr(model_meta, 'model_arch', None) - module_list = getattr(model_arch, 'module_list', None) + model_arch = getattr(getattr(self.model, 'model_meta', None), 'model_arch', None) layer_prefixes = getattr(model_arch, 'language_model', None) - if module_list: - layer_prefixes = [module_list] - if getattr(model_meta, 'is_multimodal', False) and not layer_prefixes: - raise ValueError('LISA requires model_arch.module_list or language_model for multimodal models.') + if getattr(model_arch, 'module_list', None): + layer_prefixes = [model_arch.module_list] excluded_prefixes = [ prefix for key in ('vision_tower', 'aligner', 'generator') for prefix in getattr(model_arch, key, []) ] - layer_names = [] + layers_name = None + layers = None for name, module in self.model.named_modules(): - if not isinstance(module, torch.nn.ModuleList) or (module_list and name != module_list): - continue if layer_prefixes and not any(name == prefix or name.startswith(f'{prefix}.') for prefix in layer_prefixes): continue if any(name == prefix or name.startswith(f'{prefix}.') for prefix in excluded_prefixes): continue - # Nested lists (e.g. MoE experts) belong to the enclosing layer stack. - if not any(not parent or name.startswith(f'{parent}.') for parent in layer_names): - layer_names.append(name) - if len(layer_names) != 1: - raise ValueError('LISA could not uniquely identify the layer list. ' - f'Set model_arch.module_list explicitly. Candidates: {layer_names}') - self.layers_attribute = layer_names[0] - self.total_layers = len(self.model.get_submodule(self.layers_attribute)) + if isinstance(module, torch.nn.ModuleList): + layers_name = name + layers = module + break + assert layers_name is not None + self.layers_attribute = layers_name + self.total_layers = len(layers) self.active_layers_indices = [] self.switch_active_layers()