Skip to content

Commit 7086b4e

Browse files
committed
Refactor token estimation and test code formatting
Replaced manual token estimation rounding with div_ceil for improved clarity and accuracy in benches and llm_response. Added #[allow(dead_code)] to test-only code to suppress warnings. Minor formatting improvements in imports and test code for readability.
1 parent abd5f0e commit 7086b4e

7 files changed

Lines changed: 30 additions & 14 deletions

File tree

benches/toon_bench/benches/toon_bench.rs

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,11 @@ fn create_users(count: usize) -> Vec<User> {
3131
.map(|i| User {
3232
id: i as u64,
3333
name: format!("User{}", i),
34-
role: if i % 3 == 0 { "admin".into() } else { "user".into() },
34+
role: if i % 3 == 0 {
35+
"admin".into()
36+
} else {
37+
"user".into()
38+
},
3539
active: i % 2 == 0,
3640
})
3741
.collect()
@@ -103,8 +107,8 @@ fn benchmark_output_size(c: &mut Criterion) {
103107
);
104108

105109
// Estimate tokens (~4 chars per token)
106-
let json_tokens = (json_str.len() + 3) / 4;
107-
let toon_tokens = (toon_str.len() + 3) / 4;
110+
let json_tokens = json_str.len().div_ceil(4);
111+
let toon_tokens = toon_str.len().div_ceil(4);
108112
println!("JSON tokens (est): {}", json_tokens);
109113
println!("TOON tokens (est): {}", toon_tokens);
110114
println!(

crates/rustapi-core/src/error.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@ pub fn get_environment() -> Environment {
157157
/// Note: This only works if the environment hasn't been accessed yet.
158158
/// Returns `Ok(())` if successful, `Err(env)` if already set.
159159
#[cfg(test)]
160+
#[allow(dead_code)]
160161
pub fn set_environment_for_test(env: Environment) -> Result<(), Environment> {
161162
ENVIRONMENT.set(env)
162163
}

crates/rustapi-core/src/extract.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1294,6 +1294,7 @@ mod tests {
12941294
#[test]
12951295
fn test_extension_extractor_missing() {
12961296
#[derive(Clone, Debug)]
1297+
#[allow(dead_code)]
12971298
struct MyData(String);
12981299

12991300
let request = create_test_request_with_headers(Method::GET, "/test", vec![]);

crates/rustapi-core/src/middleware/metrics.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -363,7 +363,7 @@ mod tests {
363363
let response = handler();
364364

365365
let http_response = crate::response::IntoResponse::into_response(response);
366-
let body = http_response.into_body();
366+
let _body = http_response.into_body();
367367

368368
// The body should contain rustapi_info metric
369369
// We can't easily read the body here, but we verified the metric is registered

crates/rustapi-core/src/path_validation.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -629,7 +629,7 @@ mod tests {
629629
1 => format!("/{}//test", content), // Double slash
630630
2 => format!("/{}/{{", content), // Unclosed brace
631631
3 => format!("/{}/{{}}", content), // Empty parameter
632-
4 => format!("/{}/{{1{content}}}", content = content), // Parameter starts with digit
632+
4 => format!("/{content}/{{1{content}}}", content = content), // Parameter starts with digit
633633
_ => content.clone(),
634634
};
635635

crates/rustapi-extras/src/sqlx/mod.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,6 @@ pub fn convert_sqlx_error(err: sqlx::Error) -> ApiError {
156156
/// Note: This implementation is provided in rustapi-core with the `sqlx` feature flag.
157157
/// The extension trait `SqlxErrorExt` is provided here for convenience when you need
158158
/// explicit conversion control.
159-
160159
#[cfg(test)]
161160
mod tests {
162161
use super::*;

crates/rustapi-toon/src/llm_response.rs

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,9 @@ use bytes::Bytes;
3939
use http::{header, StatusCode};
4040
use http_body_util::Full;
4141
use rustapi_core::{ApiError, IntoResponse, Response};
42-
use rustapi_openapi::{MediaType, Operation, OperationModifier, ResponseModifier, ResponseSpec, SchemaRef};
42+
use rustapi_openapi::{
43+
MediaType, Operation, OperationModifier, ResponseModifier, ResponseSpec, SchemaRef,
44+
};
4345
use serde::Serialize;
4446
use std::collections::HashMap;
4547

@@ -135,7 +137,7 @@ fn estimate_tokens(text: &str) -> usize {
135137
// Simple heuristic: ~4 chars per token
136138
// Accounts for whitespace and punctuation overhead
137139
let char_count = text.len();
138-
(char_count + 3) / 4 // Round up
140+
char_count.div_ceil(4) // Round up
139141
}
140142

141143
/// Calculate token savings percentage.
@@ -155,8 +157,14 @@ impl<T: Serialize> IntoResponse for LlmResponse<T> {
155157

156158
// Calculate token counts if enabled
157159
let (json_tokens, toon_tokens, savings) = if self.include_token_headers {
158-
let json_tokens = json_result.as_ref().map(|s| estimate_tokens(s)).unwrap_or(0);
159-
let toon_tokens = toon_result.as_ref().map(|s| estimate_tokens(s)).unwrap_or(0);
160+
let json_tokens = json_result
161+
.as_ref()
162+
.map(|s| estimate_tokens(s))
163+
.unwrap_or(0);
164+
let toon_tokens = toon_result
165+
.as_ref()
166+
.map(|s| estimate_tokens(s))
167+
.unwrap_or(0);
160168
let savings = calculate_savings(json_tokens, toon_tokens);
161169
(Some(json_tokens), Some(toon_tokens), Some(savings))
162170
} else {
@@ -187,10 +195,13 @@ impl<T: Serialize> IntoResponse for LlmResponse<T> {
187195
let mut builder = http::Response::builder()
188196
.status(StatusCode::OK)
189197
.header(header::CONTENT_TYPE, content_type)
190-
.header(X_FORMAT_USED, match self.format {
191-
OutputFormat::Json => "json",
192-
OutputFormat::Toon => "toon",
193-
});
198+
.header(
199+
X_FORMAT_USED,
200+
match self.format {
201+
OutputFormat::Json => "json",
202+
OutputFormat::Toon => "toon",
203+
},
204+
);
194205

195206
// Token counting headers
196207
if let Some(json_tokens) = json_tokens {

0 commit comments

Comments
 (0)