diff --git a/.sqlx/query-25c1b409c7444cdbe01c4aa09fbc6b3d2dc488e99dfb37cbb6b4ac1a13f76d63.json b/.sqlx/query-25c1b409c7444cdbe01c4aa09fbc6b3d2dc488e99dfb37cbb6b4ac1a13f76d63.json new file mode 100644 index 0000000..10ed222 --- /dev/null +++ b/.sqlx/query-25c1b409c7444cdbe01c4aa09fbc6b3d2dc488e99dfb37cbb6b4ac1a13f76d63.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE geode_versions SET downloads = $1 WHERE tag = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "25c1b409c7444cdbe01c4aa09fbc6b3d2dc488e99dfb37cbb6b4ac1a13f76d63" +} diff --git a/.sqlx/query-42c856db81d1581a8fe41e42fa424d312f8616475717b3f6b1e50cd95d4f86b3.json b/.sqlx/query-42c856db81d1581a8fe41e42fa424d312f8616475717b3f6b1e50cd95d4f86b3.json new file mode 100644 index 0000000..e1074e2 --- /dev/null +++ b/.sqlx/query-42c856db81d1581a8fe41e42fa424d312f8616475717b3f6b1e50cd95d4f86b3.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT tag FROM geode_versions WHERE downloads IS NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag", + "type_info": "Text", + "origin": { + "Table": { + "table": "geode_versions", + "name": "tag" + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "42c856db81d1581a8fe41e42fa424d312f8616475717b3f6b1e50cd95d4f86b3" +} diff --git a/.sqlx/query-94a9f2bc14d5a6c21eb7f638e5d70bcdc7112d9ad0d153aaad6bf7b00e27d74a.json b/.sqlx/query-4887f58d2c76c3c0f8e4dace53e1d338ffeaeac49f79453305453cd312d6f0f7.json similarity index 88% rename from .sqlx/query-94a9f2bc14d5a6c21eb7f638e5d70bcdc7112d9ad0d153aaad6bf7b00e27d74a.json rename to .sqlx/query-4887f58d2c76c3c0f8e4dace53e1d338ffeaeac49f79453305453cd312d6f0f7.json index 3933afc..3138dd8 100644 --- a/.sqlx/query-94a9f2bc14d5a6c21eb7f638e5d70bcdc7112d9ad0d153aaad6bf7b00e27d74a.json +++ b/.sqlx/query-4887f58d2c76c3c0f8e4dace53e1d338ffeaeac49f79453305453cd312d6f0f7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n\t\t\t\t mac as \"mac: _\", win as \"win: _\", android as \"android: _\", ios as \"ios: _\",\n\t\t\t\t tag, created_at, commit_hash, prerelease\n\t\t\t FROM geode_versions\n\t\t\t\t WHERE tag = $1", + "query": "SELECT\n\t\t\t\t mac as \"mac: _\", win as \"win: _\", android as \"android: _\", ios as \"ios: _\",\n\t\t\t\t tag, created_at, commit_hash, prerelease, downloads as \"downloads: Json\"\n\t\t\t FROM geode_versions\n\t\t\t\t WHERE tag = $1", "describe": { "columns": [ { @@ -178,6 +178,17 @@ "name": "prerelease" } } + }, + { + "ordinal": 8, + "name": "downloads: Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "geode_versions", + "name": "downloads" + } + } } ], "parameters": { @@ -193,8 +204,9 @@ false, false, false, - false + false, + true ] }, - "hash": "94a9f2bc14d5a6c21eb7f638e5d70bcdc7112d9ad0d153aaad6bf7b00e27d74a" + "hash": "4887f58d2c76c3c0f8e4dace53e1d338ffeaeac49f79453305453cd312d6f0f7" } diff --git a/migrations/20260721182316_add_download_link_to_geode_version.down.sql b/migrations/20260721182316_add_download_link_to_geode_version.down.sql new file mode 100644 index 0000000..52d0dc5 --- /dev/null +++ b/migrations/20260721182316_add_download_link_to_geode_version.down.sql @@ -0,0 +1 @@ +ALTER TABLE geode_versions DROP COLUMN downloads; diff --git a/migrations/20260721182316_add_download_link_to_geode_version.up.sql b/migrations/20260721182316_add_download_link_to_geode_version.up.sql new file mode 100644 index 0000000..cf96bef --- /dev/null +++ b/migrations/20260721182316_add_download_link_to_geode_version.up.sql @@ -0,0 +1 @@ +ALTER TABLE geode_versions ADD COLUMN downloads JSONB; diff --git a/src/config.rs b/src/config.rs index 76cf5df..80fcffc 100644 --- a/src/config.rs +++ b/src/config.rs @@ -27,7 +27,7 @@ pub struct AppData { static_storage: PublicDisk, public_storage: PublicDisk, private_storage: PrivateDisk, - mod_storage: Option, + cdn_storage: Option, disable_downloads: bool, max_download_mb: u32, port: u16, @@ -77,7 +77,7 @@ pub async fn build_config() -> anyhow::Result { .time_to_live(Duration::from_mins(10)) .build(); - let mod_storage = if let Some(s3_config) = S3Configuration::from_env()? { + let cdn_storage = if let Some(s3_config) = S3Configuration::from_env()? { let backend = Arc::new(S3Backend::new(&s3_config)?); Some(PublicDisk::new(backend, s3_config.public_url)) } else { @@ -103,7 +103,7 @@ pub async fn build_config() -> anyhow::Result { format!("{app_url}/storage"), ), private_storage: PrivateDisk::new(Arc::new(LocalBackend::new("storage/private"))), - mod_storage, + cdn_storage, disable_downloads, max_download_mb, port, @@ -181,8 +181,8 @@ impl AppData { &self.private_storage } - pub fn mod_storage(&self) -> Option<&PublicDisk> { - self.mod_storage.as_ref() + pub fn cdn_storage(&self) -> Option<&PublicDisk> { + self.cdn_storage.as_ref() } pub fn mods_cache(&self) -> &Cache>> { diff --git a/src/database/repository/geode_versions.rs b/src/database/repository/geode_versions.rs new file mode 100644 index 0000000..014853b --- /dev/null +++ b/src/database/repository/geode_versions.rs @@ -0,0 +1,22 @@ +use sqlx::PgConnection; + +use crate::{database::DatabaseError, types::models::loader_version::LoaderDownloads}; + +#[tracing::instrument(skip_all, fields(tag = %tag, downloads = ?downloads))] +pub async fn update_downloads( + tag: &str, + downloads: &LoaderDownloads, + conn: &mut PgConnection, +) -> Result<(), DatabaseError> { + sqlx::query!( + "UPDATE geode_versions SET downloads = $1 WHERE tag = $2", + // should never fail + &serde_json::to_value(downloads).expect("serialization failed"), + tag + ) + .execute(&mut *conn) + .await + .inspect_err(|e| tracing::error!("{:?}", e))?; + + Ok(()) +} diff --git a/src/database/repository/mod.rs b/src/database/repository/mod.rs index 7253922..e15fe17 100644 --- a/src/database/repository/mod.rs +++ b/src/database/repository/mod.rs @@ -2,6 +2,7 @@ pub mod auth_tokens; pub mod dependencies; pub mod deprecations; pub mod developers; +pub mod geode_versions; pub mod github_login_attempts; pub mod github_web_logins; pub mod incompatibilities; diff --git a/src/endpoints/loader.rs b/src/endpoints/loader.rs index 3434566..88e9b20 100644 --- a/src/endpoints/loader.rs +++ b/src/endpoints/loader.rs @@ -1,11 +1,12 @@ use actix_web::{HttpResponse, Responder, get, post, web}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use std::str::FromStr; use utoipa::{IntoParams, ToSchema}; use sqlx::Acquire; use crate::endpoints::ApiError; +use crate::s3_worker::S3WorkerTask; use crate::{ config::AppData, extractors::auth::Auth, @@ -122,10 +123,12 @@ pub async fn create_version( return Err(ApiError::Authorization); } + let tag = payload.tag.trim_start_matches('v').to_string(); + let mut tx = pool.begin().await?; LoaderVersion::create_version( LoaderVersionCreate { - tag: payload.tag.trim_start_matches('v').to_string(), + tag: tag.clone(), prerelease: payload.prerelease, commit_hash: payload.commit_hash.clone(), win: payload.gd.win, @@ -139,6 +142,8 @@ pub async fn create_version( tx.commit().await?; + data.send_s3_task(S3WorkerTask::UploadLoader { tag }); + Ok(HttpResponse::NoContent()) } diff --git a/src/endpoints/mod_versions.rs b/src/endpoints/mod_versions.rs index c4cde46..b6ea876 100644 --- a/src/endpoints/mod_versions.rs +++ b/src/endpoints/mod_versions.rs @@ -241,7 +241,7 @@ pub async fn download_version( let url = mod_version .managed_download_link .as_deref() - .take_if(|_| data.mod_storage().is_some()) + .take_if(|_| data.cdn_storage().is_some()) .unwrap_or(&mod_version.download_link); if data.disable_downloads() || mod_version.status != ModVersionStatusEnum::Accepted { diff --git a/src/main.rs b/src/main.rs index f6746f9..1d43dd2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -38,8 +38,8 @@ async fn main() -> anyhow::Result<()> { let app_data = config::build_config().await?; app_data.static_storage().init().await?; app_data.private_storage().init().await?; - if let Some(mod_storage) = app_data.mod_storage() { - mod_storage.init().await?; + if let Some(cdn_storage) = app_data.cdn_storage() { + cdn_storage.init().await?; } if cli::maybe_cli(&app_data).await? { diff --git a/src/s3_worker.rs b/src/s3_worker.rs index 9b7e117..846b0fd 100644 --- a/src/s3_worker.rs +++ b/src/s3_worker.rs @@ -4,8 +4,12 @@ use actix_web::web; use bytes::Bytes; use crate::{ - config::AppData, database::repository::mod_versions::update_managed_download_link, mod_zip, - types::models::mod_gd_version::GDVersionEnum, + config::AppData, + database::repository::{ + geode_versions::update_downloads, mod_versions::update_managed_download_link, + }, + mod_zip, + types::models::{loader_version::LoaderDownloads, mod_gd_version::GDVersionEnum}, }; pub enum S3WorkerTask { @@ -15,14 +19,36 @@ pub enum S3WorkerTask { version: String, version_id: i32, }, + + UploadLoader { + tag: String, + }, } fn path_for_mod(mod_id: &str, version: &str) -> String { format!("mods/{mod_id}/{version}/{mod_id}.geode") } +fn path_for_loader(tag: &str, platform: &str) -> String { + format!("geode/{tag}/geode-v{tag}-{platform}.zip") +} + +fn path_for_resources(tag: &str) -> String { + format!("geode/{tag}/resources.zip") +} + +fn github_url_for_loader(tag: &str, platform: &str) -> String { + format!( + "https://github.com/geode-sdk/geode/releases/download/v{tag}/geode-v{tag}-{platform}.zip" + ) +} + +fn github_url_for_resources(tag: &str) -> String { + format!("https://github.com/geode-sdk/geode/releases/download/v{tag}/resources.zip") +} + async fn process_task(data: &AppData, task: S3WorkerTask) -> anyhow::Result<()> { - let storage = data.mod_storage().expect("mod storage must be set by now"); + let storage = data.cdn_storage().expect("mod storage must be set by now"); match task { S3WorkerTask::UploadMod { @@ -47,6 +73,65 @@ async fn process_task(data: &AppData, task: S3WorkerTask) -> anyhow::Result<()> public_url ); } + + S3WorkerTask::UploadLoader { tag } => { + tracing::info!("Preparing to upload Geode v{tag} to S3"); + + let mut zips = Vec::new(); + let mut downloads = LoaderDownloads::default(); + + zips.push(( + &mut downloads.win, + path_for_loader(&tag, "win"), + github_url_for_loader(&tag, "win"), + )); + zips.push(( + &mut downloads.mac, + path_for_loader(&tag, "mac"), + github_url_for_loader(&tag, "mac"), + )); + zips.push(( + &mut downloads.android32, + path_for_loader(&tag, "android32"), + github_url_for_loader(&tag, "android32"), + )); + zips.push(( + &mut downloads.android64, + path_for_loader(&tag, "android64"), + github_url_for_loader(&tag, "android64"), + )); + zips.push(( + &mut downloads.ios, + path_for_loader(&tag, "ios"), + github_url_for_loader(&tag, "ios"), + )); + zips.push(( + &mut downloads.resources, + path_for_resources(&tag), + github_url_for_resources(&tag), + )); + + for (dl, path, url) in zips { + let resp = data.http_client().get(&url).send().await?; + if let Err(e) = resp.error_for_status_ref() { + tracing::error!("Failed to download loader release from {url}: {e:?}",); + continue; + } + + let resp = resp.bytes().await?; + let public_url = storage.asset_url(&path); + storage.store(&path, &resp).await?; + + dl.url = public_url; + dl.hash = sha256::digest(&resp[..]); + } + + let mut tx = data.db().begin().await?; + update_downloads(&tag, &downloads, &mut tx).await?; + tx.commit().await?; + + tracing::info!("Uploaded new loader release to S3: {downloads:?}"); + } } Ok(()) @@ -54,7 +139,7 @@ async fn process_task(data: &AppData, task: S3WorkerTask) -> anyhow::Result<()> async fn cleanup_old_s3_files(data: &AppData) -> anyhow::Result<()> { let supported_gd = GDVersionEnum::supported_for_storage(); - let storage = data.mod_storage().expect("mod storage must be set by now"); + let storage = data.cdn_storage().expect("mod storage must be set by now"); let mut db = data.db().acquire().await?; @@ -173,8 +258,33 @@ async fn migrate_existing_mods_to_s3(data: &AppData) -> anyhow::Result<()> { Ok(()) } +async fn migrate_loader_versions_to_s3(data: &AppData) -> anyhow::Result<()> { + let mut db = data.db().acquire().await?; + + let versions = sqlx::query!("SELECT tag FROM geode_versions WHERE downloads IS NULL") + .fetch_all(&mut *db) + .await?; + + tracing::info!("Migrating {} Geode releases to S3", versions.len()); + + for record in versions { + if let Err(e) = process_task( + data, + S3WorkerTask::UploadLoader { + tag: record.tag.clone(), + }, + ) + .await + { + tracing::error!("error migrating Geode release {} to S3: {e:?}", record.tag); + } + } + + Ok(()) +} + pub async fn run_s3_worker(data: web::Data) { - if data.mod_storage().is_none() { + if data.cdn_storage().is_none() { return; } @@ -188,6 +298,13 @@ pub async fn run_s3_worker(data: web::Data) { } }); + let s_data2 = data.clone(); + tokio::spawn(async move { + if let Err(e) = migrate_loader_versions_to_s3(&s_data2).await { + tracing::error!("Error migrating loader versions to S3: {:?}", e); + } + }); + let mut interval = tokio::time::interval(Duration::from_mins(30)); loop { diff --git a/src/types/models/loader_version.rs b/src/types/models/loader_version.rs index 0879cf9..0363e3a 100644 --- a/src/types/models/loader_version.rs +++ b/src/types/models/loader_version.rs @@ -6,12 +6,15 @@ use crate::{ }, }; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use utoipa::ToSchema; use sqlx::{ PgConnection, Postgres, QueryBuilder, - types::chrono::{DateTime, Utc}, + types::{ + Json, + chrono::{DateTime, Utc}, + }, }; #[derive(Debug)] @@ -25,6 +28,22 @@ pub struct LoaderVersionCreate { pub ios: Option, } +#[derive(Serialize, Deserialize, Default, Debug, ToSchema)] +pub struct LoaderDownload { + pub url: String, + pub hash: String, +} + +#[derive(Serialize, Deserialize, Default, Debug, ToSchema)] +pub struct LoaderDownloads { + pub win: LoaderDownload, + pub mac: LoaderDownload, + pub android32: LoaderDownload, + pub android64: LoaderDownload, + pub ios: LoaderDownload, + pub resources: LoaderDownload, +} + #[derive(Serialize, Debug, ToSchema)] pub struct LoaderVersion { pub version: String, @@ -34,6 +53,7 @@ pub struct LoaderVersion { pub commit_hash: String, #[serde(with = "chrono_dt_secs")] pub created_at: DateTime, + pub downloads: Option, } #[derive(sqlx::FromRow, Debug)] @@ -46,6 +66,7 @@ pub struct LoaderVersionGetOne { pub win: Option, pub android: Option, pub ios: Option, + pub downloads: Option>, } pub struct GetVersionsQuery { @@ -72,6 +93,7 @@ impl LoaderVersionGetOne { android64: self.android, ios: self.ios, }, + downloads: self.downloads.map(|x| x.0), } } } @@ -86,7 +108,7 @@ impl LoaderVersion { ) -> Result, DatabaseError> { let mut query_builder: QueryBuilder = QueryBuilder::new( r#"SELECT - mac, win, android, ios, tag, commit_hash, created_at, prerelease + mac, win, android, ios, tag, commit_hash, created_at, prerelease, downloads FROM geode_versions "#, ); @@ -139,7 +161,7 @@ impl LoaderVersion { query_builder.push(" AND prerelease=FALSE "); } - query_builder.push(" ORDER BY "); + query_builder.push(" AND downloads IS NOT NULL ORDER BY "); if gd.is_none() { if let Some(p) = platform { @@ -179,7 +201,7 @@ impl LoaderVersion { LoaderVersionGetOne, r#"SELECT mac as "mac: _", win as "win: _", android as "android: _", ios as "ios: _", - tag, created_at, commit_hash, prerelease + tag, created_at, commit_hash, prerelease, downloads as "downloads: Json" FROM geode_versions WHERE tag = $1"#, tag @@ -228,8 +250,8 @@ impl LoaderVersion { let mut query_builder = QueryBuilder::new( r#" - SELECT - mac, win, android, ios, tag, created_at, commit_hash, prerelease + SELECT + mac, win, android, ios, tag, created_at, commit_hash, prerelease, downloads FROM geode_versions "#, );