Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -279,10 +279,50 @@ type LegacyMigrationRow = (
bool,
);

/// Upper bound for the conversation data one migration page carries to the webview.
/// A page crosses IPC as a single JSON document and the webview then holds several
/// copies of it (parsed rows, converted messages, the import request body), so twenty
/// conversations with large transcripts or inline images ran the renderer out of memory
/// (#1136). A conversation larger than the budget still travels alone on its own page.
const MIGRATION_PAGE_BYTE_BUDGET: usize = 8 * 1024 * 1024;

fn migration_page_sync(
conn: &Connection,
cursor: Option<&str>,
home: &std::path::Path,
) -> Result<LegacyMigrationPage, String> {
migration_page_sync_with_budget(conn, cursor, home, MIGRATION_PAGE_BYTE_BUDGET)
}

fn migration_conversation_bytes(
segments: &[LegacyMigrationSegment],
checkpoint: &LegacyMigrationCheckpoint,
context_meta_json: &str,
selected_model_json: Option<&str>,
) -> usize {
let segment_bytes: usize = segments
.iter()
.map(|segment| {
segment.messages_json.len() + segment.summary_json.as_deref().map_or(0, str::len)
})
.sum();
let checkpoint_bytes: usize = checkpoint.index_jsonl.len()
+ checkpoint
.records
.iter()
.map(|record| record.blob_base64.as_deref().map_or(0, str::len))
.sum::<usize>();
segment_bytes
+ checkpoint_bytes
+ context_meta_json.len()
+ selected_model_json.map_or(0, str::len)
}

fn migration_page_sync_with_budget(
conn: &Connection,
cursor: Option<&str>,
home: &std::path::Path,
byte_budget: usize,
) -> Result<LegacyMigrationPage, String> {
const PAGE_SIZE: i64 = 20;
let after = cursor.unwrap_or("");
Expand Down Expand Up @@ -326,6 +366,9 @@ fn migration_page_sync(
.map_err(|e| format!("read legacy migration rows failed: {e}"))?;
drop(stmt);

let fetched = rows.len();
let mut page_bytes = 0usize;
let mut over_budget = false;
let mut conversations = Vec::with_capacity(rows.len());
for (
id,
Expand Down Expand Up @@ -373,6 +416,19 @@ fn migration_page_sync(
.map_err(|e| format!("query legacy segments failed: {e}"))?
.collect::<Result<Vec<_>, _>>()
.map_err(|e| format!("read legacy segments failed: {e}"))?;
let checkpoint = checkpoint_export(&id, home);
let bytes = migration_conversation_bytes(
&segments,
&checkpoint,
&context_meta_json,
selected_model_json.as_deref(),
);
if !conversations.is_empty() && page_bytes.saturating_add(bytes) > byte_budget {
// Leave this conversation for the next page; the cursor stops before it.
over_budget = true;
break;
}
page_bytes = page_bytes.saturating_add(bytes);
conversations.push(LegacyMigrationConversation {
id: id.clone(),
title,
Expand All @@ -392,11 +448,11 @@ fn migration_page_sync(
total_segment_count: total_segment_count.unwrap_or(segments.len() as i64),
total_message_count: total_message_count.unwrap_or(0),
segments,
checkpoint: checkpoint_export(&id, home),
checkpoint,
});
}
let next_cursor = conversations.last().map(|item| item.id.clone());
let complete = conversations.len() < PAGE_SIZE as usize;
let complete = !over_budget && fetched < PAGE_SIZE as usize;
Ok(LegacyMigrationPage {
conversations,
next_cursor: (!complete).then_some(next_cursor.unwrap_or_default()),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,67 @@ mod migration_tests {
assert_eq!(page.conversations[0].checkpoint.records.len(), 1);
}

#[test]
fn page_stops_at_byte_budget_and_resumes_after_last_exported_conversation() {
let conn = Connection::open_in_memory().expect("open fixture");
history_db::initialize_connection(&conn).expect("initialize fixture");
let home = tempfile::tempdir().unwrap();
// 600-byte transcripts: a 1000-byte budget fits one per page, the oversized one alone.
let body = format!(
"[{{\"role\":\"user\",\"id\":\"u\",\"content\":\"{}\"}}]",
"x".repeat(560)
);
for (id, repeat) in [("c-1", 1), ("c-2", 1), ("c-3", 3), ("c-4", 0)] {
conn.execute(
"INSERT INTO chatHistory (id,title,provider_id,model,context_meta_json,active_segment_index,total_segment_count,total_message_count,created_at,updated_at) VALUES (?1,'t','p','m','{}',0,1,1,1,1)",
[id],
)
.unwrap();
let messages = if repeat == 0 {
"[]".to_string()
} else {
body.repeat(repeat)
};
conn.execute(
"INSERT INTO chatHistorySegment (conversation_id,segment_index,segment_id,messages_json,message_count,created_at,updated_at) VALUES (?1,0,'s',?2,1,1,1)",
rusqlite::params![id, messages],
)
.unwrap();
}

let mut cursor: Option<String> = None;
let mut pages: Vec<Vec<String>> = Vec::new();
loop {
let page = migration_page_sync_with_budget(&conn, cursor.as_deref(), home.path(), 1000)
.unwrap();
pages.push(page.conversations.iter().map(|c| c.id.clone()).collect());
if page.complete {
assert!(page.next_cursor.is_none());
break;
}
let next = page
.next_cursor
.expect("incomplete page must carry a cursor");
assert_eq!(Some(&next), page.conversations.last().map(|c| &c.id));
cursor = Some(next);
assert!(pages.len() < 10, "paging did not terminate");
}
assert_eq!(
pages,
vec![
vec!["c-1".to_string()],
vec!["c-2".to_string()],
vec!["c-3".to_string()],
vec!["c-4".to_string()],
]
);

// The default budget keeps small conversations together on one page.
let page = migration_page_sync(&conn, None, home.path()).unwrap();
assert_eq!(page.conversations.len(), 4);
assert!(page.complete);
}

#[test]
fn malformed_segment_is_exported_for_per_conversation_isolation() {
let conn = Connection::open_in_memory().expect("open fixture");
Expand Down
Loading