diff --git a/src/len_bot/media/images.py b/src/len_bot/media/images.py index d0702321a..8785fabc8 100644 --- a/src/len_bot/media/images.py +++ b/src/len_bot/media/images.py @@ -2,12 +2,19 @@ import base64 import io +import math from PIL import Image, ImageOps FORMATS = {"PNG": "image/png", "JPEG": "image/jpeg", "WEBP": "image/webp", "GIF": "image/gif"} +# An animated image reaches the model as a single picture. Sampling several frames into +# a grid keeps the motion: the first frame alone is often a blurred transitional one. +GRID_MAX_FRAMES = 6 +GRID_CELL_MAX = 640 +GRID_GAP = 6 + def validate_image(data: bytes, *, max_bytes: int, max_pixels: int): if not data or len(data) > max_bytes: @@ -22,23 +29,68 @@ def validate_image(data: bytes, *, max_bytes: int, max_pixels: int): return mime_type -def prepare_image(data: bytes, *, max_bytes: int, max_pixels: int, max_dimension: int): +def _frame_indices(count: int, limit: int) -> list[int]: + """Evenly spaced frame positions, always including the first and the last one.""" + if count <= limit: + return list(range(count)) + step = (count - 1) / (limit - 1) + return sorted({round(index * step) for index in range(limit)}) + + +def _flatten(image, max_dimension: int): + frame = ImageOps.exif_transpose(image).convert("RGBA") + frame.thumbnail((max_dimension, max_dimension), Image.Resampling.LANCZOS) + background = Image.new("RGBA", frame.size, "white") + background.alpha_composite(frame) + return background.convert("RGB") + + +def _contact_sheet(image, indices: list[int], *, max_dimension: int, max_pixels: int): + """Frames left to right, top to bottom, on one white sheet.""" + frames = [] + for index in indices: + image.seek(index) + frames.append(_flatten(image, GRID_CELL_MAX)) + columns = 2 if len(frames) <= 2 else (3 if len(frames) <= 9 else 4) + rows = math.ceil(len(frames) / columns) + cell_width = max(frame.width for frame in frames) + cell_height = max(frame.height for frame in frames) + sheet = Image.new("RGB", (columns * cell_width + GRID_GAP * (columns - 1), + rows * cell_height + GRID_GAP * (rows - 1)), "white") + for position, frame in enumerate(frames): + column, row = position % columns, position // columns + left = column * (cell_width + GRID_GAP) + (cell_width - frame.width) // 2 + top = row * (cell_height + GRID_GAP) + (cell_height - frame.height) // 2 + sheet.paste(frame, (left, top)) + width, height = sheet.size + scale = min(1.0, max_dimension / max(width, height), math.sqrt(max_pixels / (width * height))) + if scale < 1: + sheet = sheet.resize((max(1, round(width * scale)), max(1, round(height * scale))), + Image.Resampling.LANCZOS) + return sheet + + +def prepare_image(data: bytes, *, max_bytes: int, max_pixels: int, max_dimension: int, + frames: int = GRID_MAX_FRAMES): + """``frames`` is how many frames an animated image may contribute; 1 keeps the first only.""" validate_image(data, max_bytes=max_bytes, max_pixels=max_pixels) with Image.open(io.BytesIO(data)) as image: - animated = getattr(image, "n_frames", 1) > 1 - image.seek(0) - frame = ImageOps.exif_transpose(image).convert("RGBA") - frame.thumbnail((max_dimension, max_dimension), Image.Resampling.LANCZOS) - background = Image.new("RGBA", frame.size, "white") - background.alpha_composite(frame) + frame_count = getattr(image, "n_frames", 1) + animated = frame_count > 1 + if animated and frames > 1: + sheet = _contact_sheet(image, _frame_indices(frame_count, min(frames, GRID_MAX_FRAMES)), + max_dimension=max_dimension, max_pixels=max_pixels) + else: + image.seek(0) + sheet = _flatten(image, max_dimension) output = io.BytesIO() # The frame is already composited onto white and flattened to RGB, so a # lossless encoding preserves nothing an alpha channel would have kept. # It only inflates the request body: measured on real group images, six # pictures reach 13.6 MB at p90 and 37 MB at worst as PNG, which is what # the relay drops mid-upload. - background.convert("RGB").save(output, format="JPEG", quality=85, optimize=True) - return output.getvalue(), animated, frame.width, frame.height + sheet.save(output, format="JPEG", quality=85, optimize=True) + return output.getvalue(), animated, sheet.width, sheet.height def image_block(data: bytes): diff --git a/src/len_bot/next/chat/context.py b/src/len_bot/next/chat/context.py index 404f213b2..9a9e082c5 100644 --- a/src/len_bot/next/chat/context.py +++ b/src/len_bot/next/chat/context.py @@ -459,8 +459,28 @@ def conversation_entries(entries: list[tuple[int, dict]]) -> list[tuple[int, dic if not (message['role'] == 'user' and isinstance(message['content'], str) and message['content'].startswith('宿主启动状态:'))] + @staticmethod + def _unspoken_thought(message: dict) -> dict: + """A plain-text assistant entry is an inner thought, not a sent reply. + + Kept as an assistant message, it becomes a few-shot example of answering without + the outlet tool, and later turns keep copying that shape (silent "listening"). + Re-label it as a read-only note so the model sees what it thought without + imitating the format. Entries carrying tool_calls must stay untouched so + tool messages keep their preceding assistant call. + """ + if message.get('role') != 'assistant' or message.get('tool_calls'): + return message + content = message.get('content') + if not isinstance(content, str) or not content.strip(): + return message + return {'role': 'user', + 'content': '[上一轮未发出的内部想法,不是发言;不要模仿这种写法,要开口必须调用 say 工具] ' + + content.strip()} + def project_entries(self, entries: list[tuple[int, dict]]) -> list[tuple[int, dict]]: entries = self.conversation_entries(entries) + entries = [(seq, self._unspoken_thought(message)) for seq, message in entries] return [(seq, projected) for seq, message in project_old_results( entries, self.config.compaction.keep_recent_tokens) for projected in project_messages([message], self.config.models.roles.mind.history_policy)] diff --git a/src/len_bot/next/chat/tools.py b/src/len_bot/next/chat/tools.py index 9f496152f..04c793b3d 100644 --- a/src/len_bot/next/chat/tools.py +++ b/src/len_bot/next/chat/tools.py @@ -424,11 +424,14 @@ def tool_names(self) -> set[str]: return {tool["function"]["name"] for tool in self.tools} async def describe_image(self, turn_id: str, asset: ImageAsset) -> str: + info = {"width": asset.width, "height": asset.height, "animated_grid": asset.animated} + if asset.animated: + info["grid_note"] = ("动图已抽取多帧按时间顺序排成网格,从左到右、从上到下;请按同一画面随时间的" + "连续变化描述动作,不要把格子本身当成画面内容。") messages = [ {"role": "system", "content": read_prompt("next_vision.md")}, {"role": "user", "content": [ - {"type": "text", "text": encode({"width": asset.width, "height": asset.height, - "animated_first_frame_only": asset.animated})}, + {"type": "text", "text": encode(info)}, image_block(asset.jpeg), ]}, ] diff --git a/src/len_bot/next/launcher.py b/src/len_bot/next/launcher.py index 87f98b8e3..5cbb495bb 100644 --- a/src/len_bot/next/launcher.py +++ b/src/len_bot/next/launcher.py @@ -49,8 +49,13 @@ def wait(process: subprocess.Popen) -> int: def start(command: list[str], cwd: Path = root, env: dict | None = None) -> subprocess.Popen: nonlocal child + # Windows: the desktop shell starts us with CREATE_NO_WINDOW, so we have no console to hand + # down; a plain CREATE_NEW_PROCESS_GROUP would make the host allocate a visible console + # window. CREATE_NO_WINDOW gives it a console without a window, and CTRL_BREAK still works. + flags = (subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW + if sys.platform == 'win32' else 0) child = subprocess.Popen(command, cwd=cwd, env=env, start_new_session=sys.platform != 'win32', - creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if sys.platform == 'win32' else 0) + creationflags=flags) if stopping: child.terminate() return child diff --git a/src/len_bot/next/learning/sticker_collection.py b/src/len_bot/next/learning/sticker_collection.py index 85d366af9..436e6216a 100644 --- a/src/len_bot/next/learning/sticker_collection.py +++ b/src/len_bot/next/learning/sticker_collection.py @@ -237,7 +237,8 @@ async def _process(self, scene: str, candidate: dict) -> None: mime_type, width, height, animated = await asyncio.to_thread(inspect_image, data) self.records.save_original(scene, id, data=data, mime_type=mime_type, width=width, height=height, animated=animated) - jpeg, animated_first_frame_only, width, height = await prepare_pixels(data, self.config.images) + jpeg, animated_first_frame_only, width, height = await prepare_pixels( + data, self.config.images, frames=1) except TimeoutError as error: if deadline.expired(): raise TimeoutError(f"sticker download and preparation exceeded " diff --git a/src/len_bot/next/media/images.py b/src/len_bot/next/media/images.py index 599804aeb..8547e688e 100644 --- a/src/len_bot/next/media/images.py +++ b/src/len_bot/next/media/images.py @@ -11,6 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator from ..configuration.chat import ImageSettings +from ...media.images import GRID_MAX_FRAMES from ..tools.http_read import fetch_public from ..storage.store import ImageAsset, Store from ..trials.replay_images import RecordedImages @@ -55,10 +56,10 @@ def image_url(data: dict[str, object]) -> str: return url -async def prepare_pixels(data: bytes, settings: ImageSettings) -> tuple[bytes, bool, int, int]: +async def prepare_pixels(data: bytes, settings: ImageSettings, *, frames: int = GRID_MAX_FRAMES) -> tuple[bytes, bool, int, int]: process = await asyncio.create_subprocess_exec( sys.executable, "-m", __name__, "prepare", - str(settings.max_bytes), str(settings.max_pixels), str(settings.max_dimension), + str(settings.max_bytes), str(settings.max_pixels), str(settings.max_dimension), str(frames), stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) @@ -81,7 +82,7 @@ def _display(scene: str, arguments: LookArguments, asset: ImageAsset, *, result = { "scene": scene, "platform_message_id": arguments.message, "image": arguments.image, "jpeg_bytes": len(asset.jpeg), "width": asset.width, "height": asset.height, - "animated_first_frame_only": asset.animated, + "animated_grid": asset.animated, "fetched_at": datetime.fromtimestamp(asset.fetched_at, timezone.utc).isoformat(), "description": asset.description, "description_model": asset.description_model, "described_at": (None if asset.described_at is None else @@ -89,7 +90,7 @@ def _display(scene: str, arguments: LookArguments, asset: ImageAsset, *, "pixels_reused": pixels_reused, "description_reused": description_reused, "pixels_source": pixels_source, } - return json.dumps(result, ensure_ascii=False) + "\n描述来自准备后的 JPEG;动图只分析第一帧。" + return json.dumps(result, ensure_ascii=False) + "\n描述来自准备后的 JPEG;动图已按时间顺序抽帧拼成网格,从左到右、从上到下读。" async def load_look_image(store: Store, scene: str, arguments: LookArguments, settings: ImageSettings, *, @@ -168,28 +169,28 @@ def shown_image(scene: str, arguments: LookArguments, asset: ImageAsset, *, result = { "scene": scene, "platform_message_id": arguments.message, "image": arguments.image, "jpeg_bytes": len(asset.jpeg), "width": asset.width, "height": asset.height, - "animated_first_frame_only": asset.animated, + "animated_grid": asset.animated, "fetched_at": datetime.fromtimestamp(asset.fetched_at, timezone.utc).isoformat(), "pixels_reused": pixels_reused, "pixels_source": pixels_source, } - return json.dumps(result, ensure_ascii=False) + "\n图片附在本轮接下来的请求里;动图只有第一帧。" + return json.dumps(result, ensure_ascii=False) + "\n图片附在本轮接下来的请求里;动图已抽成多帧网格,从左到右、从上到下读。" def _worker() -> None: from len_bot.media.images import prepare_image - max_bytes, max_pixels, max_dimension = map(int, sys.argv[2:5]) + max_bytes, max_pixels, max_dimension, frames = map(int, sys.argv[2:6]) jpeg, animated, width, height = prepare_image( sys.stdin.buffer.read(max_bytes + 1), - max_bytes=max_bytes, max_pixels=max_pixels, max_dimension=max_dimension, + max_bytes=max_bytes, max_pixels=max_pixels, max_dimension=max_dimension, frames=frames, ) metadata = {"animated": animated, "width": width, "height": height} sys.stdout.buffer.write(json.dumps(metadata).encode("ascii") + b"\n" + jpeg) if __name__ == "__main__": - if len(sys.argv) != 5 or sys.argv[1] != "prepare": - raise SystemExit("look image worker expects prepare and three limits") + if len(sys.argv) != 6 or sys.argv[1] != "prepare": + raise SystemExit("look image worker expects prepare and four limits") try: _worker() except Exception as error: diff --git a/src/len_bot/next/platform/messages.py b/src/len_bot/next/platform/messages.py index af24f6f71..2b2308c27 100644 --- a/src/len_bot/next/platform/messages.py +++ b/src/len_bot/next/platform/messages.py @@ -79,6 +79,19 @@ def plain_text(message: ChatMessage) -> str: for segment in message.segments) +# [local patch] 12-name-clip: 群名片可以是整句话,原样进上下文容易被当成发言。 +# uid 已经能唯一定位一个人,名片只需要够认,不需要完整。 +CARD_LIMIT = 16 + + +def _display_name(message: ChatMessage) -> str | None: + """The name shown in the transcript; a long free-form card is clipped.""" + name = (message.sender.card or message.sender.nickname or '').strip() + if not name: + return None + return name if len(name) <= CARD_LIMIT else name[:CARD_LIMIT] + '…' + + def _speaker(message: ChatMessage) -> str: if message.is_self: if message.send_status == "failed": @@ -88,7 +101,7 @@ def _speaker(message: ChatMessage) -> str: if message.send_status == "simulated": return "我(模拟)" return "我" - name = message.sender.card or message.sender.nickname + name = _display_name(message) return f"{name}({message.sender.uid})" if name else f"{message.sender.uid}" @@ -184,7 +197,7 @@ def render_batch(items: Sequence[tuple[ChatMessage, ChatMessage | None, dict[int lines.append(f"{message.scene}|{local.date()}|{timezone} {local.strftime('%z')}") previous = header # Names and IDs are data on one line, not new batch headings. - name = '我' if message.is_self else message.sender.card or message.sender.nickname + name = '我' if message.is_self else _display_name(message) speaker = (f"{message.sender.uid}" if name is None else f"{json.dumps(name, ensure_ascii=False)}({message.sender.uid})") labels = [] @@ -202,5 +215,8 @@ def render_batch(items: Sequence[tuple[ChatMessage, ChatMessage | None, dict[int excerpt = json.dumps(_body(reply.segments)[:40], ensure_ascii=False) who = json.dumps(_speaker(reply), ensure_ascii=False) lines.append(f" 引用节选 {who}{'(已撤回)' if reply.recalled else ''}:{excerpt}") - lines.extend(' ' + line for line in _body(message.segments, audio).split('\n')) + body = _body(message.segments, audio) + # [local patch] 12-name-clip: 没有正文时不留空行,否则像「说了句空话」。 + if body.strip(): + lines.extend(' ' + line for line in body.split('\n')) return '\n'.join(lines) diff --git a/src/len_bot/prompts/next_response_choice.md b/src/len_bot/prompts/next_response_choice.md index 241ecf419..e56eaa760 100644 --- a/src/len_bot/prompts/next_response_choice.md +++ b/src/len_bot/prompts/next_response_choice.md @@ -14,6 +14,8 @@ - 群友之间在聊,你没有新内容可补充时,就旁听。 - 一段文字、一张表情、继续旁听,都是完整的选择。 - 当前关注里写着你最近 10 分钟说了几句。最近说得多时,起哄、夸奖和单纯的情绪回个表情就够,认真问你的事照常回答。 +- 对方只发来图片或表情、一个字都没写时:挑一张贴得上的表情,用 react 发出去,这一张就是完整回复。不要把图里有什么讲出来,也不要复述、点评这张图。对方写了字,或者明确问你图里是什么,才需要用文字说。 +- 只发图、又分不清是表情包还是要你处理的截图时,先看清再定:是截图、报错、一屏文字这类需要你处理的内容,就照常回话。 怎么接: - 亲近、打趣、纠正、求助、抱怨是不同的话头,按对方这句话的实际意思接。对方的新反馈会改变你刚才的判断。 diff --git a/tests/test_next_unspoken_thought.py b/tests/test_next_unspoken_thought.py new file mode 100644 index 000000000..4be1eb1ea --- /dev/null +++ b/tests/test_next_unspoken_thought.py @@ -0,0 +1,28 @@ +"""A plain-text assistant turn is an unspoken thought, not a sample reply to imitate.""" + +from __future__ import annotations + +from len_bot.next.chat.context import ChatContext + + +def test_plain_text_assistant_turn_is_relabelled_as_an_unspoken_note(): + message = {'role': 'assistant', 'content': ' 先看看他们在聊什么 '} + projected = ChatContext._unspoken_thought(message) + assert projected['role'] == 'user' + assert projected['content'].startswith('[上一轮未发出的内部想法') + assert projected['content'].endswith('先看看他们在聊什么') + + +def test_assistant_turn_with_tool_calls_keeps_its_role_and_content(): + message = {'role': 'assistant', 'content': '', + 'tool_calls': [{'id': 'call_1', 'type': 'function', + 'function': {'name': 'say', 'arguments': '{}'}}]} + assert ChatContext._unspoken_thought(message) is message + + +def test_everything_else_passes_through_untouched(): + for message in ({'role': 'tool', 'content': 'ok'}, + {'role': 'user', 'content': '在吗'}, + {'role': 'assistant', 'content': ' '}, + {'role': 'assistant', 'content': None}): + assert ChatContext._unspoken_thought(message) is message