diff --git a/swift/template/base.py b/swift/template/base.py index 2a0ce04f51..660d83cf77 100644 --- a/swift/template/base.py +++ b/swift/template/base.py @@ -1497,6 +1497,13 @@ def _truncate(self, input_ids: List[int], labels: Optional[List[int]], encoded, loss_scale = torch.tensor(loss_scale)[protected].tolist() loss_scale[0] = 0 encoded['loss_scale'] = loss_scale + token_type_ids = encoded.get('token_type_ids') + if token_type_ids is not None: + if isinstance(token_type_ids, torch.Tensor): + # ERNIE-VL keeps a leading batch dimension: [1, seq_len]. + encoded['token_type_ids'] = token_type_ids[..., protected] + else: + encoded['token_type_ids'] = torch.tensor(token_type_ids)[protected].tolist() for key in ('mm_token_type_ids', 'image_token_types'): token_types = encoded.get(key) if token_types is not None: diff --git a/swift/template/templates/gemma.py b/swift/template/templates/gemma.py index 9ce1203bfc..34666dcb2f 100644 --- a/swift/template/templates/gemma.py +++ b/swift/template/templates/gemma.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Literal, Optional -from swift.utils import get_logger, upper_bound +from swift.utils import get_logger, lower_bound from ..base import Template from ..constant import LLMTemplateType, MLLMTemplateType from ..register import TemplateMeta, register_template @@ -48,7 +48,7 @@ def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]: raw_image = inputs.images processor = self.processor if encoded['labels'] is not None: - n = upper_bound(0, len(encoded['labels']), lambda idx: encoded['labels'][idx] == -100) + n = lower_bound(0, len(encoded['labels']), lambda idx: encoded['labels'][idx] != -100) n2 = len(encoded['labels']) - n encoded['token_type_ids'] = [0] * n + [1] * n2 else: diff --git a/swift/template/templates/glm.py b/swift/template/templates/glm.py index 442309a997..b309048480 100644 --- a/swift/template/templates/glm.py +++ b/swift/template/templates/glm.py @@ -577,6 +577,9 @@ def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]: encoded['input_ids'] = input_ids[:1] + [self.processor.pad_token_id] * image_token_len + input_ids[1:] if labels is not None: encoded['labels'] = labels[:1] + [-100] * image_token_len + labels[1:] + loss_scale = encoded.get('loss_scale') + if loss_scale is not None: + encoded['loss_scale'] = loss_scale[:1] + [0.] * image_token_len + loss_scale[1:] if len(image) > 0: encoded['images'] = [[img.to(dtype=self.model_info.torch_dtype)] for img in inputs2['images']] if 'cross_images' in inputs2: @@ -586,11 +589,13 @@ def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]: return encoded def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]: + if any(b.get('cross_images') for b in batch) and not all(b.get('cross_images') for b in batch): + raise ValueError('CogAgent requires an image for every sample in a multimodal batch.') res = super()._data_collator(batch, padding_to=padding_to) keys = ['images', 'cross_images'] for key in keys: - if key in batch[0]: - res[key] = [b[key][0] for b in batch] + if any(b.get(key) for b in batch): + res[key] = [b[key][0] if b.get(key) else [] for b in batch] return res @@ -648,6 +653,9 @@ def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]: encoded['input_ids'] = input_ids[:1] + [self.processor.pad_token_id] * video_token_len + input_ids[1:] if labels is not None: encoded['labels'] = labels[:1] + [-100] * video_token_len + labels[1:] + loss_scale = encoded.get('loss_scale') + if loss_scale is not None: + encoded['loss_scale'] = loss_scale[:1] + [0.] * video_token_len + loss_scale[1:] if len(video) > 0: dtype = model.dtype encoded['images'] = [[img.to(dtype=dtype)] for img in inputs2['images']] diff --git a/swift/template/templates/seed.py b/swift/template/templates/seed.py index 0babdcc8ed..296c6681e7 100644 --- a/swift/template/templates/seed.py +++ b/swift/template/templates/seed.py @@ -199,11 +199,11 @@ def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]: input_ids = encoded['input_ids'] idx_list = findall(input_ids, -100) pixel_values = None + labels = encoded.get('labels') loss_scale = encoded.get('loss_scale', None) images = inputs.images processor = self.processor if images: - labels = encoded.get('labels') image_inputs = processor.image_processor(images) num_patches = image_inputs['num_patches_list'] pixel_values = image_inputs['pixel_values'] @@ -227,10 +227,10 @@ def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, An embedding = model.language_model.get_input_embeddings() device = embedding.weight.device input_ids = inputs['input_ids'] + inputs_embeds = embedding(input_ids).to(device=device) pixel_values = inputs.get('pixel_values') if pixel_values is not None: vit_embeds = model.extract_feature(pixel_values) - inputs_embeds = embedding(input_ids) B, N, C = inputs_embeds.shape inputs_embeds = inputs_embeds.reshape(B * N, C) @@ -242,7 +242,6 @@ def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, An inputs_embeds = inputs_embeds.reshape(B, N, C) elif is_deepspeed_enabled(): - inputs_embeds = embedding(input_ids).to(device=device) dummy_pixel_values = torch.zeros((1, 3, 32, 32), device=device, dtype=inputs_embeds.dtype) vit_embeds = model.extract_feature(dummy_pixel_values).to(device=device) inputs_embeds = inputs_embeds + vit_embeds.mean() * 0.