From ecd8fb309e9226922ca8fa6fdd2111198c809fc9 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:16:03 +0000 Subject: [PATCH 1/2] perf(receiver): optimize bulk data replacement queries with multi-row chunk inserts Replaced individual row-by-row execute calls in `replace_divisions`, `replace_bib_chips`, `replace_participants`, and `replace_rd_data` with chunked multi-row batch inserts (`CHUNK_SIZE = 64`). This eliminates N+1 query execution overhead when importing large sets of division, participant, and bib-chip data. Co-authored-by: iwismer <10574297+iwismer@users.noreply.github.com> --- services/receiver/src/db.rs | 199 +++++++++++++++++++++++++----------- 1 file changed, 142 insertions(+), 57 deletions(-) diff --git a/services/receiver/src/db.rs b/services/receiver/src/db.rs index d6aaa335..75535195 100644 --- a/services/receiver/src/db.rs +++ b/services/receiver/src/db.rs @@ -552,21 +552,7 @@ impl Db { ) -> DbResult<()> { let tx = self.transaction()?; tx.execute_batch("DELETE FROM participants")?; - let mut stmt = tx.prepare( - "INSERT OR REPLACE INTO participants (bib, last, first, affiliation, gender, division) - VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - )?; - for p in participants { - stmt.execute(rusqlite::params![ - p.bib, - &p.last, - &p.first, - &p.affiliation, - &p.gender, - p.division - ])?; - } - drop(stmt); + insert_participants_batch(&tx, participants)?; tx.commit()?; Ok(()) } @@ -576,12 +562,7 @@ impl Db { pub fn replace_divisions(&mut self, divisions: &[(i32, String)]) -> DbResult<()> { let tx = self.transaction()?; tx.execute_batch("DELETE FROM divisions")?; - let mut stmt = - tx.prepare("INSERT OR REPLACE INTO divisions (divno, name) VALUES (?1, ?2)")?; - for (divno, name) in divisions { - stmt.execute(rusqlite::params![divno, name])?; - } - drop(stmt); + insert_divisions_batch(&tx, divisions)?; tx.commit()?; Ok(()) } @@ -599,36 +580,9 @@ impl Db { ) -> DbResult<()> { let tx = self.transaction()?; tx.execute_batch("DELETE FROM participants; DELETE FROM bib_chips; DELETE FROM divisions")?; - { - let mut stmt = tx.prepare( - "INSERT OR REPLACE INTO participants (bib, last, first, affiliation, gender, division) - VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - )?; - for p in participants { - stmt.execute(rusqlite::params![ - p.bib, - &p.last, - &p.first, - &p.affiliation, - &p.gender, - p.division - ])?; - } - } - { - let mut stmt = - tx.prepare("INSERT OR REPLACE INTO bib_chips (chip_id, bib) VALUES (?1, ?2)")?; - for (bib, chip_id) in chips { - stmt.execute(rusqlite::params![chip_id, bib])?; - } - } - { - let mut stmt = - tx.prepare("INSERT OR REPLACE INTO divisions (divno, name) VALUES (?1, ?2)")?; - for (divno, name) in divisions { - stmt.execute(rusqlite::params![divno, name])?; - } - } + insert_participants_batch(&tx, participants)?; + insert_bib_chips_batch(&tx, chips)?; + insert_divisions_batch(&tx, divisions)?; tx.commit()?; Ok(()) } @@ -637,12 +591,7 @@ impl Db { pub fn replace_bib_chips(&mut self, chips: &[(i64, String)]) -> DbResult<()> { let tx = self.transaction()?; tx.execute_batch("DELETE FROM bib_chips")?; - let mut stmt = - tx.prepare("INSERT OR REPLACE INTO bib_chips (chip_id, bib) VALUES (?1, ?2)")?; - for (bib, chip_id) in chips { - stmt.execute(rusqlite::params![chip_id, bib])?; - } - drop(stmt); + insert_bib_chips_batch(&tx, chips)?; tx.commit()?; Ok(()) } @@ -1902,6 +1851,142 @@ impl Db { } } +fn insert_participants_batch( + tx: &rusqlite::Transaction<'_>, + participants: &[crate::participants::Participant], +) -> DbResult<()> { + if participants.is_empty() { + return Ok(()); + } + const CHUNK_SIZE: usize = 64; + let mut full_query = String::from( + "INSERT OR REPLACE INTO participants (bib, last, first, affiliation, gender, division) VALUES ", + ); + for i in 0..CHUNK_SIZE { + if i > 0 { + full_query.push_str(", "); + } + use std::fmt::Write; + let _ = write!( + full_query, + "(?{}, ?{}, ?{}, ?{}, ?{}, ?{})", + i * 6 + 1, + i * 6 + 2, + i * 6 + 3, + i * 6 + 4, + i * 6 + 5, + i * 6 + 6 + ); + } + let mut chunk_stmt = tx.prepare_cached(&full_query)?; + let mut single_stmt = tx.prepare_cached( + "INSERT OR REPLACE INTO participants (bib, last, first, affiliation, gender, division) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + )?; + + for chunk in participants.chunks(CHUNK_SIZE) { + if chunk.len() == CHUNK_SIZE { + let mut params: Vec<&dyn rusqlite::types::ToSql> = Vec::with_capacity(CHUNK_SIZE * 6); + for p in chunk { + params.push(&p.bib); + params.push(&p.last); + params.push(&p.first); + params.push(&p.affiliation); + params.push(&p.gender); + params.push(&p.division); + } + chunk_stmt.execute(params.as_slice())?; + } else { + for p in chunk { + single_stmt.execute(rusqlite::params![ + p.bib, + &p.last, + &p.first, + &p.affiliation, + &p.gender, + p.division + ])?; + } + } + } + Ok(()) +} + +fn insert_divisions_batch( + tx: &rusqlite::Transaction<'_>, + divisions: &[(i32, String)], +) -> DbResult<()> { + if divisions.is_empty() { + return Ok(()); + } + const CHUNK_SIZE: usize = 64; + let mut full_query = String::from("INSERT OR REPLACE INTO divisions (divno, name) VALUES "); + for i in 0..CHUNK_SIZE { + if i > 0 { + full_query.push_str(", "); + } + use std::fmt::Write; + let _ = write!(full_query, "(?{}, ?{})", i * 2 + 1, i * 2 + 2); + } + let mut chunk_stmt = tx.prepare_cached(&full_query)?; + let mut single_stmt = tx.prepare_cached( + "INSERT OR REPLACE INTO divisions (divno, name) VALUES (?1, ?2)", + )?; + + for chunk in divisions.chunks(CHUNK_SIZE) { + if chunk.len() == CHUNK_SIZE { + let mut params: Vec<&dyn rusqlite::types::ToSql> = Vec::with_capacity(CHUNK_SIZE * 2); + for (divno, name) in chunk { + params.push(divno); + params.push(name); + } + chunk_stmt.execute(params.as_slice())?; + } else { + for (divno, name) in chunk { + single_stmt.execute(rusqlite::params![divno, name])?; + } + } + } + Ok(()) +} + +fn insert_bib_chips_batch( + tx: &rusqlite::Transaction<'_>, + chips: &[(i64, String)], +) -> DbResult<()> { + if chips.is_empty() { + return Ok(()); + } + const CHUNK_SIZE: usize = 64; + let mut full_query = String::from("INSERT OR REPLACE INTO bib_chips (chip_id, bib) VALUES "); + for i in 0..CHUNK_SIZE { + if i > 0 { + full_query.push_str(", "); + } + use std::fmt::Write; + let _ = write!(full_query, "(?{}, ?{})", i * 2 + 1, i * 2 + 2); + } + let mut chunk_stmt = tx.prepare_cached(&full_query)?; + let mut single_stmt = tx.prepare_cached( + "INSERT OR REPLACE INTO bib_chips (chip_id, bib) VALUES (?1, ?2)", + )?; + + for chunk in chips.chunks(CHUNK_SIZE) { + if chunk.len() == CHUNK_SIZE { + let mut params: Vec<&dyn rusqlite::types::ToSql> = Vec::with_capacity(CHUNK_SIZE * 2); + for (bib, chip_id) in chunk { + params.push(chip_id); + params.push(bib); + } + chunk_stmt.execute(params.as_slice())?; + } else { + for (bib, chip_id) in chunk { + single_stmt.execute(rusqlite::params![chip_id, bib])?; + } + } + } + Ok(()) +} + fn parse_event_type_column(raw: String, column: usize) -> rusqlite::Result { match raw.parse::() { Ok(et) => Ok(et), From 1b535e7226f8f819c4d18e705cb262c84ad4b4e9 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:28:13 +0000 Subject: [PATCH 2/2] perf(receiver): optimize bulk data replacement queries with multi-row chunk inserts Replaced individual row-by-row execute calls in `replace_divisions`, `replace_bib_chips`, `replace_participants`, and `replace_rd_data` with chunked multi-row batch inserts (`CHUNK_SIZE = 64`). This eliminates N+1 query execution overhead when importing large sets of division, participant, and bib-chip data. Co-authored-by: iwismer <10574297+iwismer@users.noreply.github.com> --- services/receiver/src/db.rs | 170 +++++++++++++++++++++++------------- 1 file changed, 109 insertions(+), 61 deletions(-) diff --git a/services/receiver/src/db.rs b/services/receiver/src/db.rs index 75535195..df13bf10 100644 --- a/services/receiver/src/db.rs +++ b/services/receiver/src/db.rs @@ -1855,33 +1855,34 @@ fn insert_participants_batch( tx: &rusqlite::Transaction<'_>, participants: &[crate::participants::Participant], ) -> DbResult<()> { + use std::fmt::Write; if participants.is_empty() { return Ok(()); } const CHUNK_SIZE: usize = 64; - let mut full_query = String::from( - "INSERT OR REPLACE INTO participants (bib, last, first, affiliation, gender, division) VALUES ", - ); - for i in 0..CHUNK_SIZE { - if i > 0 { - full_query.push_str(", "); - } - use std::fmt::Write; - let _ = write!( - full_query, - "(?{}, ?{}, ?{}, ?{}, ?{}, ?{})", - i * 6 + 1, - i * 6 + 2, - i * 6 + 3, - i * 6 + 4, - i * 6 + 5, - i * 6 + 6 + let mut chunk_stmt = if participants.len() >= CHUNK_SIZE { + let mut full_query = String::from( + "INSERT OR REPLACE INTO participants (bib, last, first, affiliation, gender, division) VALUES ", ); - } - let mut chunk_stmt = tx.prepare_cached(&full_query)?; - let mut single_stmt = tx.prepare_cached( - "INSERT OR REPLACE INTO participants (bib, last, first, affiliation, gender, division) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - )?; + for i in 0..CHUNK_SIZE { + if i > 0 { + full_query.push_str(", "); + } + let _ = write!( + full_query, + "(?{}, ?{}, ?{}, ?{}, ?{}, ?{})", + i * 6 + 1, + i * 6 + 2, + i * 6 + 3, + i * 6 + 4, + i * 6 + 5, + i * 6 + 6 + ); + } + Some(tx.prepare_cached(&full_query)?) + } else { + None + }; for chunk in participants.chunks(CHUNK_SIZE) { if chunk.len() == CHUNK_SIZE { @@ -1894,18 +1895,39 @@ fn insert_participants_batch( params.push(&p.gender); params.push(&p.division); } - chunk_stmt.execute(params.as_slice())?; + if let Some(stmt) = &mut chunk_stmt { + stmt.execute(params.as_slice())?; + } } else { + let mut sql = String::from( + "INSERT OR REPLACE INTO participants (bib, last, first, affiliation, gender, division) VALUES ", + ); + for i in 0..chunk.len() { + if i > 0 { + sql.push_str(", "); + } + let _ = write!( + sql, + "(?{}, ?{}, ?{}, ?{}, ?{}, ?{})", + i * 6 + 1, + i * 6 + 2, + i * 6 + 3, + i * 6 + 4, + i * 6 + 5, + i * 6 + 6 + ); + } + let mut stmt = tx.prepare_cached(&sql)?; + let mut params: Vec<&dyn rusqlite::types::ToSql> = Vec::with_capacity(chunk.len() * 6); for p in chunk { - single_stmt.execute(rusqlite::params![ - p.bib, - &p.last, - &p.first, - &p.affiliation, - &p.gender, - p.division - ])?; + params.push(&p.bib); + params.push(&p.last); + params.push(&p.first); + params.push(&p.affiliation); + params.push(&p.gender); + params.push(&p.division); } + stmt.execute(params.as_slice())?; } } Ok(()) @@ -1915,22 +1937,23 @@ fn insert_divisions_batch( tx: &rusqlite::Transaction<'_>, divisions: &[(i32, String)], ) -> DbResult<()> { + use std::fmt::Write; if divisions.is_empty() { return Ok(()); } const CHUNK_SIZE: usize = 64; - let mut full_query = String::from("INSERT OR REPLACE INTO divisions (divno, name) VALUES "); - for i in 0..CHUNK_SIZE { - if i > 0 { - full_query.push_str(", "); + let mut chunk_stmt = if divisions.len() >= CHUNK_SIZE { + let mut full_query = String::from("INSERT OR REPLACE INTO divisions (divno, name) VALUES "); + for i in 0..CHUNK_SIZE { + if i > 0 { + full_query.push_str(", "); + } + let _ = write!(full_query, "(?{}, ?{})", i * 2 + 1, i * 2 + 2); } - use std::fmt::Write; - let _ = write!(full_query, "(?{}, ?{})", i * 2 + 1, i * 2 + 2); - } - let mut chunk_stmt = tx.prepare_cached(&full_query)?; - let mut single_stmt = tx.prepare_cached( - "INSERT OR REPLACE INTO divisions (divno, name) VALUES (?1, ?2)", - )?; + Some(tx.prepare_cached(&full_query)?) + } else { + None + }; for chunk in divisions.chunks(CHUNK_SIZE) { if chunk.len() == CHUNK_SIZE { @@ -1939,36 +1962,48 @@ fn insert_divisions_batch( params.push(divno); params.push(name); } - chunk_stmt.execute(params.as_slice())?; + if let Some(stmt) = &mut chunk_stmt { + stmt.execute(params.as_slice())?; + } } else { + let mut sql = String::from("INSERT OR REPLACE INTO divisions (divno, name) VALUES "); + for i in 0..chunk.len() { + if i > 0 { + sql.push_str(", "); + } + let _ = write!(sql, "(?{}, ?{})", i * 2 + 1, i * 2 + 2); + } + let mut stmt = tx.prepare_cached(&sql)?; + let mut params: Vec<&dyn rusqlite::types::ToSql> = Vec::with_capacity(chunk.len() * 2); for (divno, name) in chunk { - single_stmt.execute(rusqlite::params![divno, name])?; + params.push(divno); + params.push(name); } + stmt.execute(params.as_slice())?; } } Ok(()) } -fn insert_bib_chips_batch( - tx: &rusqlite::Transaction<'_>, - chips: &[(i64, String)], -) -> DbResult<()> { +fn insert_bib_chips_batch(tx: &rusqlite::Transaction<'_>, chips: &[(i64, String)]) -> DbResult<()> { + use std::fmt::Write; if chips.is_empty() { return Ok(()); } const CHUNK_SIZE: usize = 64; - let mut full_query = String::from("INSERT OR REPLACE INTO bib_chips (chip_id, bib) VALUES "); - for i in 0..CHUNK_SIZE { - if i > 0 { - full_query.push_str(", "); + let mut chunk_stmt = if chips.len() >= CHUNK_SIZE { + let mut full_query = + String::from("INSERT OR REPLACE INTO bib_chips (chip_id, bib) VALUES "); + for i in 0..CHUNK_SIZE { + if i > 0 { + full_query.push_str(", "); + } + let _ = write!(full_query, "(?{}, ?{})", i * 2 + 1, i * 2 + 2); } - use std::fmt::Write; - let _ = write!(full_query, "(?{}, ?{})", i * 2 + 1, i * 2 + 2); - } - let mut chunk_stmt = tx.prepare_cached(&full_query)?; - let mut single_stmt = tx.prepare_cached( - "INSERT OR REPLACE INTO bib_chips (chip_id, bib) VALUES (?1, ?2)", - )?; + Some(tx.prepare_cached(&full_query)?) + } else { + None + }; for chunk in chips.chunks(CHUNK_SIZE) { if chunk.len() == CHUNK_SIZE { @@ -1977,11 +2012,24 @@ fn insert_bib_chips_batch( params.push(chip_id); params.push(bib); } - chunk_stmt.execute(params.as_slice())?; + if let Some(stmt) = &mut chunk_stmt { + stmt.execute(params.as_slice())?; + } } else { + let mut sql = String::from("INSERT OR REPLACE INTO bib_chips (chip_id, bib) VALUES "); + for i in 0..chunk.len() { + if i > 0 { + sql.push_str(", "); + } + let _ = write!(sql, "(?{}, ?{})", i * 2 + 1, i * 2 + 2); + } + let mut stmt = tx.prepare_cached(&sql)?; + let mut params: Vec<&dyn rusqlite::types::ToSql> = Vec::with_capacity(chunk.len() * 2); for (bib, chip_id) in chunk { - single_stmt.execute(rusqlite::params![chip_id, bib])?; + params.push(chip_id); + params.push(bib); } + stmt.execute(params.as_slice())?; } } Ok(())