Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 61 additions & 9 deletions src/len_bot/media/images.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,19 @@

import base64
import io
import math

from PIL import Image, ImageOps


FORMATS = {"PNG": "image/png", "JPEG": "image/jpeg", "WEBP": "image/webp", "GIF": "image/gif"}

# An animated image reaches the model as a single picture. Sampling several frames into
# a grid keeps the motion: the first frame alone is often a blurred transitional one.
GRID_MAX_FRAMES = 6
GRID_CELL_MAX = 640
GRID_GAP = 6


def validate_image(data: bytes, *, max_bytes: int, max_pixels: int):
if not data or len(data) > max_bytes:
Expand All @@ -22,23 +29,68 @@ def validate_image(data: bytes, *, max_bytes: int, max_pixels: int):
return mime_type


def prepare_image(data: bytes, *, max_bytes: int, max_pixels: int, max_dimension: int):
def _frame_indices(count: int, limit: int) -> list[int]:
"""Evenly spaced frame positions, always including the first and the last one."""
if count <= limit:
return list(range(count))
step = (count - 1) / (limit - 1)
return sorted({round(index * step) for index in range(limit)})


def _flatten(image, max_dimension: int):
frame = ImageOps.exif_transpose(image).convert("RGBA")
frame.thumbnail((max_dimension, max_dimension), Image.Resampling.LANCZOS)
background = Image.new("RGBA", frame.size, "white")
background.alpha_composite(frame)
return background.convert("RGB")


def _contact_sheet(image, indices: list[int], *, max_dimension: int, max_pixels: int):
"""Frames left to right, top to bottom, on one white sheet."""
frames = []
for index in indices:
image.seek(index)
frames.append(_flatten(image, GRID_CELL_MAX))
columns = 2 if len(frames) <= 2 else (3 if len(frames) <= 9 else 4)
rows = math.ceil(len(frames) / columns)
cell_width = max(frame.width for frame in frames)
cell_height = max(frame.height for frame in frames)
sheet = Image.new("RGB", (columns * cell_width + GRID_GAP * (columns - 1),
rows * cell_height + GRID_GAP * (rows - 1)), "white")
for position, frame in enumerate(frames):
column, row = position % columns, position // columns
left = column * (cell_width + GRID_GAP) + (cell_width - frame.width) // 2
top = row * (cell_height + GRID_GAP) + (cell_height - frame.height) // 2
sheet.paste(frame, (left, top))
width, height = sheet.size
scale = min(1.0, max_dimension / max(width, height), math.sqrt(max_pixels / (width * height)))
if scale < 1:
sheet = sheet.resize((max(1, round(width * scale)), max(1, round(height * scale))),
Image.Resampling.LANCZOS)
return sheet


def prepare_image(data: bytes, *, max_bytes: int, max_pixels: int, max_dimension: int,
frames: int = GRID_MAX_FRAMES):
"""``frames`` is how many frames an animated image may contribute; 1 keeps the first only."""
validate_image(data, max_bytes=max_bytes, max_pixels=max_pixels)
with Image.open(io.BytesIO(data)) as image:
animated = getattr(image, "n_frames", 1) > 1
image.seek(0)
frame = ImageOps.exif_transpose(image).convert("RGBA")
frame.thumbnail((max_dimension, max_dimension), Image.Resampling.LANCZOS)
background = Image.new("RGBA", frame.size, "white")
background.alpha_composite(frame)
frame_count = getattr(image, "n_frames", 1)
animated = frame_count > 1
if animated and frames > 1:
sheet = _contact_sheet(image, _frame_indices(frame_count, min(frames, GRID_MAX_FRAMES)),
max_dimension=max_dimension, max_pixels=max_pixels)
else:
image.seek(0)
sheet = _flatten(image, max_dimension)
output = io.BytesIO()
# The frame is already composited onto white and flattened to RGB, so a
# lossless encoding preserves nothing an alpha channel would have kept.
# It only inflates the request body: measured on real group images, six
# pictures reach 13.6 MB at p90 and 37 MB at worst as PNG, which is what
# the relay drops mid-upload.
background.convert("RGB").save(output, format="JPEG", quality=85, optimize=True)
return output.getvalue(), animated, frame.width, frame.height
sheet.save(output, format="JPEG", quality=85, optimize=True)
return output.getvalue(), animated, sheet.width, sheet.height


def image_block(data: bytes):
Expand Down
20 changes: 20 additions & 0 deletions src/len_bot/next/chat/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -459,8 +459,28 @@ def conversation_entries(entries: list[tuple[int, dict]]) -> list[tuple[int, dic
if not (message['role'] == 'user' and isinstance(message['content'], str)
and message['content'].startswith('宿主启动状态:'))]

@staticmethod
def _unspoken_thought(message: dict) -> dict:
"""A plain-text assistant entry is an inner thought, not a sent reply.

Kept as an assistant message, it becomes a few-shot example of answering without
the outlet tool, and later turns keep copying that shape (silent "listening").
Re-label it as a read-only note so the model sees what it thought without
imitating the format. Entries carrying tool_calls must stay untouched so
tool messages keep their preceding assistant call.
"""
if message.get('role') != 'assistant' or message.get('tool_calls'):
return message
content = message.get('content')
if not isinstance(content, str) or not content.strip():
return message
return {'role': 'user',
'content': '[上一轮未发出的内部想法,不是发言;不要模仿这种写法,要开口必须调用 say 工具] '
+ content.strip()}

def project_entries(self, entries: list[tuple[int, dict]]) -> list[tuple[int, dict]]:
entries = self.conversation_entries(entries)
entries = [(seq, self._unspoken_thought(message)) for seq, message in entries]
return [(seq, projected) for seq, message in project_old_results(
entries, self.config.compaction.keep_recent_tokens)
for projected in project_messages([message], self.config.models.roles.mind.history_policy)]
Expand Down
7 changes: 5 additions & 2 deletions src/len_bot/next/chat/tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -424,11 +424,14 @@ def tool_names(self) -> set[str]:
return {tool["function"]["name"] for tool in self.tools}

async def describe_image(self, turn_id: str, asset: ImageAsset) -> str:
info = {"width": asset.width, "height": asset.height, "animated_grid": asset.animated}
if asset.animated:
info["grid_note"] = ("动图已抽取多帧按时间顺序排成网格,从左到右、从上到下;请按同一画面随时间的"
"连续变化描述动作,不要把格子本身当成画面内容。")
messages = [
{"role": "system", "content": read_prompt("next_vision.md")},
{"role": "user", "content": [
{"type": "text", "text": encode({"width": asset.width, "height": asset.height,
"animated_first_frame_only": asset.animated})},
{"type": "text", "text": encode(info)},
image_block(asset.jpeg),
]},
]
Expand Down
7 changes: 6 additions & 1 deletion src/len_bot/next/launcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,13 @@ def wait(process: subprocess.Popen) -> int:

def start(command: list[str], cwd: Path = root, env: dict | None = None) -> subprocess.Popen:
nonlocal child
# Windows: the desktop shell starts us with CREATE_NO_WINDOW, so we have no console to hand
# down; a plain CREATE_NEW_PROCESS_GROUP would make the host allocate a visible console
# window. CREATE_NO_WINDOW gives it a console without a window, and CTRL_BREAK still works.
flags = (subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW
if sys.platform == 'win32' else 0)
child = subprocess.Popen(command, cwd=cwd, env=env, start_new_session=sys.platform != 'win32',
creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if sys.platform == 'win32' else 0)
creationflags=flags)
if stopping:
child.terminate()
return child
Expand Down
3 changes: 2 additions & 1 deletion src/len_bot/next/learning/sticker_collection.py
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,8 @@ async def _process(self, scene: str, candidate: dict) -> None:
mime_type, width, height, animated = await asyncio.to_thread(inspect_image, data)
self.records.save_original(scene, id, data=data, mime_type=mime_type,
width=width, height=height, animated=animated)
jpeg, animated_first_frame_only, width, height = await prepare_pixels(data, self.config.images)
jpeg, animated_first_frame_only, width, height = await prepare_pixels(
data, self.config.images, frames=1)
except TimeoutError as error:
if deadline.expired():
raise TimeoutError(f"sticker download and preparation exceeded "
Expand Down
21 changes: 11 additions & 10 deletions src/len_bot/next/media/images.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from pydantic import BaseModel, ConfigDict, Field, field_validator

from ..configuration.chat import ImageSettings
from ...media.images import GRID_MAX_FRAMES
from ..tools.http_read import fetch_public
from ..storage.store import ImageAsset, Store
from ..trials.replay_images import RecordedImages
Expand Down Expand Up @@ -55,10 +56,10 @@ def image_url(data: dict[str, object]) -> str:
return url


async def prepare_pixels(data: bytes, settings: ImageSettings) -> tuple[bytes, bool, int, int]:
async def prepare_pixels(data: bytes, settings: ImageSettings, *, frames: int = GRID_MAX_FRAMES) -> tuple[bytes, bool, int, int]:
process = await asyncio.create_subprocess_exec(
sys.executable, "-m", __name__, "prepare",
str(settings.max_bytes), str(settings.max_pixels), str(settings.max_dimension),
str(settings.max_bytes), str(settings.max_pixels), str(settings.max_dimension), str(frames),
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
Expand All @@ -81,15 +82,15 @@ def _display(scene: str, arguments: LookArguments, asset: ImageAsset, *,
result = {
"scene": scene, "platform_message_id": arguments.message, "image": arguments.image,
"jpeg_bytes": len(asset.jpeg), "width": asset.width, "height": asset.height,
"animated_first_frame_only": asset.animated,
"animated_grid": asset.animated,
"fetched_at": datetime.fromtimestamp(asset.fetched_at, timezone.utc).isoformat(),
"description": asset.description, "description_model": asset.description_model,
"described_at": (None if asset.described_at is None else
datetime.fromtimestamp(asset.described_at, timezone.utc).isoformat()),
"pixels_reused": pixels_reused, "description_reused": description_reused,
"pixels_source": pixels_source,
}
return json.dumps(result, ensure_ascii=False) + "\n描述来自准备后的 JPEG;动图只分析第一帧。"
return json.dumps(result, ensure_ascii=False) + "\n描述来自准备后的 JPEG;动图已按时间顺序抽帧拼成网格,从左到右、从上到下读。"


async def load_look_image(store: Store, scene: str, arguments: LookArguments, settings: ImageSettings, *,
Expand Down Expand Up @@ -168,28 +169,28 @@ def shown_image(scene: str, arguments: LookArguments, asset: ImageAsset, *,
result = {
"scene": scene, "platform_message_id": arguments.message, "image": arguments.image,
"jpeg_bytes": len(asset.jpeg), "width": asset.width, "height": asset.height,
"animated_first_frame_only": asset.animated,
"animated_grid": asset.animated,
"fetched_at": datetime.fromtimestamp(asset.fetched_at, timezone.utc).isoformat(),
"pixels_reused": pixels_reused, "pixels_source": pixels_source,
}
return json.dumps(result, ensure_ascii=False) + "\n图片附在本轮接下来的请求里;动图只有第一帧。"
return json.dumps(result, ensure_ascii=False) + "\n图片附在本轮接下来的请求里;动图已抽成多帧网格,从左到右、从上到下读。"


def _worker() -> None:
from len_bot.media.images import prepare_image

max_bytes, max_pixels, max_dimension = map(int, sys.argv[2:5])
max_bytes, max_pixels, max_dimension, frames = map(int, sys.argv[2:6])
jpeg, animated, width, height = prepare_image(
sys.stdin.buffer.read(max_bytes + 1),
max_bytes=max_bytes, max_pixels=max_pixels, max_dimension=max_dimension,
max_bytes=max_bytes, max_pixels=max_pixels, max_dimension=max_dimension, frames=frames,
)
metadata = {"animated": animated, "width": width, "height": height}
sys.stdout.buffer.write(json.dumps(metadata).encode("ascii") + b"\n" + jpeg)


if __name__ == "__main__":
if len(sys.argv) != 5 or sys.argv[1] != "prepare":
raise SystemExit("look image worker expects prepare and three limits")
if len(sys.argv) != 6 or sys.argv[1] != "prepare":
raise SystemExit("look image worker expects prepare and four limits")
try:
_worker()
except Exception as error:
Expand Down
22 changes: 19 additions & 3 deletions src/len_bot/next/platform/messages.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,19 @@ def plain_text(message: ChatMessage) -> str:
for segment in message.segments)


# [local patch] 12-name-clip: 群名片可以是整句话,原样进上下文容易被当成发言。
# uid 已经能唯一定位一个人,名片只需要够认,不需要完整。
CARD_LIMIT = 16


def _display_name(message: ChatMessage) -> str | None:
"""The name shown in the transcript; a long free-form card is clipped."""
name = (message.sender.card or message.sender.nickname or '').strip()
if not name:
return None
return name if len(name) <= CARD_LIMIT else name[:CARD_LIMIT] + '…'


def _speaker(message: ChatMessage) -> str:
if message.is_self:
if message.send_status == "failed":
Expand All @@ -88,7 +101,7 @@ def _speaker(message: ChatMessage) -> str:
if message.send_status == "simulated":
return "我(模拟)"
return "我"
name = message.sender.card or message.sender.nickname
name = _display_name(message)
return f"{name}({message.sender.uid})" if name else f"{message.sender.uid}"


Expand Down Expand Up @@ -184,7 +197,7 @@ def render_batch(items: Sequence[tuple[ChatMessage, ChatMessage | None, dict[int
lines.append(f"{message.scene}|{local.date()}|{timezone} {local.strftime('%z')}")
previous = header
# Names and IDs are data on one line, not new batch headings.
name = '我' if message.is_self else message.sender.card or message.sender.nickname
name = '我' if message.is_self else _display_name(message)
speaker = (f"{message.sender.uid}" if name is None else
f"{json.dumps(name, ensure_ascii=False)}({message.sender.uid})")
labels = []
Expand All @@ -202,5 +215,8 @@ def render_batch(items: Sequence[tuple[ChatMessage, ChatMessage | None, dict[int
excerpt = json.dumps(_body(reply.segments)[:40], ensure_ascii=False)
who = json.dumps(_speaker(reply), ensure_ascii=False)
lines.append(f" 引用节选 {who}{'(已撤回)' if reply.recalled else ''}:{excerpt}")
lines.extend(' ' + line for line in _body(message.segments, audio).split('\n'))
body = _body(message.segments, audio)
# [local patch] 12-name-clip: 没有正文时不留空行,否则像「说了句空话」。
if body.strip():
lines.extend(' ' + line for line in body.split('\n'))
return '\n'.join(lines)
2 changes: 2 additions & 0 deletions src/len_bot/prompts/next_response_choice.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
- 群友之间在聊,你没有新内容可补充时,就旁听。
- 一段文字、一张表情、继续旁听,都是完整的选择。
- 当前关注里写着你最近 10 分钟说了几句。最近说得多时,起哄、夸奖和单纯的情绪回个表情就够,认真问你的事照常回答。
- 对方只发来图片或表情、一个字都没写时:挑一张贴得上的表情,用 react 发出去,这一张就是完整回复。不要把图里有什么讲出来,也不要复述、点评这张图。对方写了字,或者明确问你图里是什么,才需要用文字说。
- 只发图、又分不清是表情包还是要你处理的截图时,先看清再定:是截图、报错、一屏文字这类需要你处理的内容,就照常回话。

怎么接:
- 亲近、打趣、纠正、求助、抱怨是不同的话头,按对方这句话的实际意思接。对方的新反馈会改变你刚才的判断。
Expand Down
28 changes: 28 additions & 0 deletions tests/test_next_unspoken_thought.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
"""A plain-text assistant turn is an unspoken thought, not a sample reply to imitate."""

from __future__ import annotations

from len_bot.next.chat.context import ChatContext


def test_plain_text_assistant_turn_is_relabelled_as_an_unspoken_note():
message = {'role': 'assistant', 'content': ' 先看看他们在聊什么 '}
projected = ChatContext._unspoken_thought(message)
assert projected['role'] == 'user'
assert projected['content'].startswith('[上一轮未发出的内部想法')
assert projected['content'].endswith('先看看他们在聊什么')


def test_assistant_turn_with_tool_calls_keeps_its_role_and_content():
message = {'role': 'assistant', 'content': '',
'tool_calls': [{'id': 'call_1', 'type': 'function',
'function': {'name': 'say', 'arguments': '{}'}}]}
assert ChatContext._unspoken_thought(message) is message


def test_everything_else_passes_through_untouched():
for message in ({'role': 'tool', 'content': 'ok'},
{'role': 'user', 'content': '在吗'},
{'role': 'assistant', 'content': ' '},
{'role': 'assistant', 'content': None}):
assert ChatContext._unspoken_thought(message) is message