From 2607903f7b764597275b10b8d5b65b2f21829473 Mon Sep 17 00:00:00 2001 From: "NG (Graham)" Date: Sat, 24 May 2025 23:56:48 -0400 Subject: [PATCH] Add arc thumbnail support as part of #10 --- Cargo.lock | 205 ++++++++++++++++++++++++- cdn/Cargo.toml | 2 + cdn/src/main.rs | 1 + cdn/src/robocraft/brawl_data.rs | 4 +- cdn/src/robocraft/campaign_data.rs | 4 +- cdn/src/robocraft/clan_avatar.rs | 4 +- cdn/src/robocraft/factory/arc.rs | 82 ++++++++++ cdn/src/robocraft/factory/mod.rs | 1 + cdn/src/robocraft/mod.rs | 1 + cdn/src/robocraft/user_avatar.rs | 4 +- rc_core/src/factory/adapter_enum.rs | 2 +- rc_core/src/persist/vehicle_factory.rs | 3 + rc_factory/src/arc/adapter.rs | 17 +- 13 files changed, 310 insertions(+), 20 deletions(-) create mode 100644 cdn/src/robocraft/factory/arc.rs create mode 100644 cdn/src/robocraft/factory/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 5277e30..6915170 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -223,6 +223,17 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "ahash" version = "0.7.8" @@ -350,6 +361,15 @@ dependencies = [ "num-traits 0.2.19", ] +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "argon2" version = "0.5.3" @@ -593,9 +613,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "bytecheck" @@ -646,6 +666,25 @@ dependencies = [ "bytes", ] +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "cc" version = "1.2.10" @@ -666,6 +705,8 @@ dependencies = [ "clap", "env_logger", "log", + "tokio", + "zip", ] [[package]] @@ -717,6 +758,16 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.5.27" @@ -778,6 +829,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "convert_case" version = "0.4.0" @@ -919,6 +976,12 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "deflate64" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" + [[package]] name = "der" version = "0.7.10" @@ -940,6 +1003,17 @@ dependencies = [ "serde", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "derive_more" version = "0.99.20" @@ -1149,11 +1223,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.35" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", + "libz-rs-sys", "miniz_oxide", ] @@ -1350,8 +1425,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.13.3+wasi-0.2.2", + "wasm-bindgen", "windows-targets 0.52.6", ] @@ -1884,6 +1961,15 @@ dependencies = [ "libc", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -2028,6 +2114,26 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "liblzma" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66352d7a8ac12d4877b6e6ea5a9b7650ee094257dc40889955bea5bc5b08c1d0" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5839bad90c3cc2e0b8c4ed8296b80e86040240f81d46b9c0e9bc8dd51ddd3af1" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.13" @@ -2056,6 +2162,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +dependencies = [ + "zlib-rs", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -2097,9 +2212,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.25" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "loom" @@ -2159,9 +2274,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.8.3" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" +checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" dependencies = [ "adler2", ] @@ -2490,6 +2605,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "pear" version = "0.2.9" @@ -3798,6 +3923,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "simdutf8" version = "0.1.5" @@ -5210,6 +5341,20 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] [[package]] name = "zerovec" @@ -5233,6 +5378,50 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "zip" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "153a6fff49d264c4babdcfa6b4d534747f520e56e8f0f384f3b808c4b64cc1fd" +dependencies = [ + "aes", + "arbitrary", + "bzip2", + "constant_time_eq", + "crc32fast", + "deflate64", + "flate2", + "getrandom 0.3.1", + "hmac", + "indexmap", + "liblzma", + "memchr", + "pbkdf2", + "sha1", + "time", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zlib-rs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" + +[[package]] +name = "zopfli" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfc5ee405f504cd4984ecc6f14d02d55cfda60fa4b689434ef4102aae150cd7" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.13.3" diff --git a/cdn/Cargo.toml b/cdn/Cargo.toml index 7ec4bc3..94e0e5e 100644 --- a/cdn/Cargo.toml +++ b/cdn/Cargo.toml @@ -13,3 +13,5 @@ actix-files.workspace = true clap.workspace = true log.workspace = true env_logger.workspace = true +tokio = { version = "1.43", features = [ "rt-multi-thread" ] } +zip = "4" diff --git a/cdn/src/main.rs b/cdn/src/main.rs index 41d384c..23ae9c2 100644 --- a/cdn/src/main.rs +++ b/cdn/src/main.rs @@ -27,6 +27,7 @@ async fn main() -> std::io::Result<()> { .service(robocraft::clan_avatar::get) .service(robocraft::brawl_data::get) .service(robocraft::campaign_data::get) + .service(robocraft::factory::arc::get) }) .bind((cli_args.ip, cli_args.port))? .run() diff --git a/cdn/src/robocraft/brawl_data.rs b/cdn/src/robocraft/brawl_data.rs index 8423f5e..685a864 100644 --- a/cdn/src/robocraft/brawl_data.rs +++ b/cdn/src/robocraft/brawl_data.rs @@ -1,8 +1,8 @@ -use actix_web::{dev::ResourcePath, get, web::{Data, Path}, Responder}; +use actix_web::{get, web::{Data, Path}, Responder}; #[get("/brawldata/Live/{name}")] pub async fn get(cli: Data, name: Path) -> impl Responder { - let path = std::path::PathBuf::from(&cli.data_robocraft).join("brawldata").join(format!("{}.jpg", name.path())); + let path = std::path::PathBuf::from(&cli.data_robocraft).join("brawldata").join(format!("{}.jpg", *name)); log::debug!("RC asset at {} (exists? {})", path.display(), path.exists()); actix_files::NamedFile::open_async(path).await } diff --git a/cdn/src/robocraft/campaign_data.rs b/cdn/src/robocraft/campaign_data.rs index 4048cb6..da9631a 100644 --- a/cdn/src/robocraft/campaign_data.rs +++ b/cdn/src/robocraft/campaign_data.rs @@ -1,8 +1,8 @@ -use actix_web::{dev::ResourcePath, get, web::{Data, Path}, Responder}; +use actix_web::{get, web::{Data, Path}, Responder}; #[get("/campaigndata/Live/{name}")] pub async fn get(cli: Data, name: Path) -> impl Responder { - let path = std::path::PathBuf::from(&cli.data_robocraft).join("campaigndata").join(format!("{}.jpg", name.path())); + let path = std::path::PathBuf::from(&cli.data_robocraft).join("campaigndata").join(format!("{}.jpg", *name)); log::debug!("RC asset at {} (exists? {})", path.display(), path.exists()); actix_files::NamedFile::open_async(path).await } diff --git a/cdn/src/robocraft/clan_avatar.rs b/cdn/src/robocraft/clan_avatar.rs index 498d96c..47c514b 100644 --- a/cdn/src/robocraft/clan_avatar.rs +++ b/cdn/src/robocraft/clan_avatar.rs @@ -1,8 +1,8 @@ -use actix_web::{dev::ResourcePath, get, web::{Data, Path}, Responder}; +use actix_web::{get, web::{Data, Path}, Responder}; #[get("/clanavatar/Live/{name}")] pub async fn get(cli: Data, name: Path) -> impl Responder { - let path = std::path::PathBuf::from(&cli.data_robocraft).join("clanavatar").join(format!("{}.jpg", name.path())); + let path = std::path::PathBuf::from(&cli.data_robocraft).join("clanavatar").join(format!("{}.jpg", *name)); log::debug!("RC asset at {} (exists? {})", path.display(), path.exists()); actix_files::NamedFile::open_async(path).await } diff --git a/cdn/src/robocraft/factory/arc.rs b/cdn/src/robocraft/factory/arc.rs new file mode 100644 index 0000000..ea80585 --- /dev/null +++ b/cdn/src/robocraft/factory/arc.rs @@ -0,0 +1,82 @@ +use std::io::Read; + +use actix_web::{get, web::{Data, Path}, HttpResponse}; + +static ZIP_FILE: std::sync::Mutex>>> = std::sync::Mutex::new(None); + +#[get("/roboshop/arc/Live/{id}")] +pub async fn get(cli: Data, id: Path) -> HttpResponse { + let id: u32 = *id; + let zip_path = std::path::PathBuf::from(&cli.data_robocraft).join("rc_archive_thumbnails.zip"); + try_find_file(zip_path, id).await +} + +async fn try_find_file(zip_path: std::path::PathBuf, id: u32) -> HttpResponse { + let result = tokio::task::spawn_blocking(move || get_file_in_zip(zip_path, id)).await.unwrap(); + match result { + Ok(bytes) => { + log::debug!("Found id {} in factory arc", id); + actix_web::HttpResponseBuilder::new(actix_web::http::StatusCode::OK) + .append_header(("Content-Type", "image/jpeg")) + .body(bytes) + }, + Err(e) => { + log::debug!("Failed to find id {} in factory arc: {}", id, e); + match e { + zip::result::ZipError::Io(e) => { + actix_web::HttpResponseBuilder::new(actix_web::http::StatusCode::INTERNAL_SERVER_ERROR) + .body(format!("zip io error: {}", e.to_string())) + }, + zip::result::ZipError::InvalidArchive(e) => { + actix_web::HttpResponseBuilder::new(actix_web::http::StatusCode::INTERNAL_SERVER_ERROR) + .body(format!("invalid zip file: {}", e.to_string())) + }, + zip::result::ZipError::UnsupportedArchive(e) => { + actix_web::HttpResponseBuilder::new(actix_web::http::StatusCode::INTERNAL_SERVER_ERROR) + .body(format!("unsupported zip file: {}", e.to_string())) + }, + zip::result::ZipError::FileNotFound => { + actix_web::HttpResponseBuilder::new(actix_web::http::StatusCode::NOT_FOUND) + .body(format!("file not found in zip archive")) + }, + zip::result::ZipError::InvalidPassword => { + actix_web::HttpResponseBuilder::new(actix_web::http::StatusCode::INTERNAL_SERVER_ERROR) + .body(format!("invalid zip password")) + }, + _ => actix_web::HttpResponseBuilder::new(actix_web::http::StatusCode::INTERNAL_SERVER_ERROR) + .body(format!("unknown zip error")), + } + } + } +} + +fn get_file_in_zip(zip_path: std::path::PathBuf, id: u32) -> zip::result::ZipResult> { + let prefix = format!("rc_archive_thumbnails/{} - ", id); + let mut lock = ZIP_FILE.lock().unwrap(); + if let Some(archive) = lock.as_mut() { + Ok(read_file_with_prefix(&prefix, archive)?) + } else { + // this is usually very slow, but thankfully should only run the first time this endpoint is hit + let file = std::fs::File::open(zip_path)?; + let buf = std::io::BufReader::new(file); + let mut archive = zip::read::ZipArchive::new(buf)?; + let bytes = read_file_with_prefix(&prefix, &mut archive)?; + *lock = Some(archive); + Ok(bytes) + } + + +} + +fn read_file_with_prefix(prefix: &str, archive: &mut zip::read::ZipArchive>) -> zip::result::ZipResult> { + let index = if let Some((index, _)) = archive.file_names().enumerate().find(|(_, name)| name.starts_with(prefix)) { + index + } else { + return Err(zip::result::ZipError::FileNotFound); + }; + let mut buf = Vec::new(); + let mut file = archive.by_index(index)?; + file.read_to_end(&mut buf)?; + Ok(buf) + +} diff --git a/cdn/src/robocraft/factory/mod.rs b/cdn/src/robocraft/factory/mod.rs new file mode 100644 index 0000000..02d3672 --- /dev/null +++ b/cdn/src/robocraft/factory/mod.rs @@ -0,0 +1 @@ +pub mod arc; diff --git a/cdn/src/robocraft/mod.rs b/cdn/src/robocraft/mod.rs index 3e7b560..600b6fb 100644 --- a/cdn/src/robocraft/mod.rs +++ b/cdn/src/robocraft/mod.rs @@ -3,3 +3,4 @@ pub mod user_avatar; pub mod clan_avatar; pub mod brawl_data; pub mod campaign_data; +pub mod factory; diff --git a/cdn/src/robocraft/user_avatar.rs b/cdn/src/robocraft/user_avatar.rs index 9015cbb..5cb3d57 100644 --- a/cdn/src/robocraft/user_avatar.rs +++ b/cdn/src/robocraft/user_avatar.rs @@ -1,8 +1,8 @@ -use actix_web::{dev::ResourcePath, get, web::{Data, Path}, Responder}; +use actix_web::{get, web::{Data, Path}, Responder}; #[get("/customavatar/Live/{name}")] pub async fn get(cli: Data, name: Path) -> impl Responder { - let path = std::path::PathBuf::from(&cli.data_robocraft).join("customavatars").join(format!("{}.jpg", name.path())); + let path = std::path::PathBuf::from(&cli.data_robocraft).join("customavatars").join(format!("{}.jpg", *name)); log::debug!("RC asset at {} (exists? {})", path.display(), path.exists()); actix_files::NamedFile::open_async(path).await } diff --git a/rc_core/src/factory/adapter_enum.rs b/rc_core/src/factory/adapter_enum.rs index 69dde44..8bced44 100644 --- a/rc_core/src/factory/adapter_enum.rs +++ b/rc_core/src/factory/adapter_enum.rs @@ -34,7 +34,7 @@ impl rc_factory::VehicleFactoryAdapter for Factory { impl Factory { pub async fn from_config(conf: &crate::persist::FactoryConfig) -> Result> { Ok(match &conf.adapter { - crate::persist::AdapterSettings::Arc(x) => Self::Arc(rc_factory::arc::ArcAdapter::init(&x.uri, x.show_expired).await?), + crate::persist::AdapterSettings::Arc(x) => Self::Arc(rc_factory::arc::ArcAdapter::init(&x.uri, x.show_expired, x.override_cdn.clone()).await?), crate::persist::AdapterSettings::None => Self::None, }) } diff --git a/rc_core/src/persist/vehicle_factory.rs b/rc_core/src/persist/vehicle_factory.rs index c562cd6..acfc46c 100644 --- a/rc_core/src/persist/vehicle_factory.rs +++ b/rc_core/src/persist/vehicle_factory.rs @@ -23,6 +23,9 @@ pub struct ArcFactorySettings { pub uri: String, #[serde(default = "default_true")] pub show_expired: bool, + /// should probably end with /roboshop/arc/Live/ + #[serde(default)] + pub override_cdn: Option, } fn default_true() -> bool { diff --git a/rc_factory/src/arc/adapter.rs b/rc_factory/src/arc/adapter.rs index 76ef109..9029103 100644 --- a/rc_factory/src/arc/adapter.rs +++ b/rc_factory/src/arc/adapter.rs @@ -3,15 +3,18 @@ use sea_orm::{ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder} pub struct ArcAdapter { orm: sea_orm::DatabaseConnection, ignore_expiry: bool, + cdn: Option, } impl ArcAdapter { - pub async fn init(uri: &str, show_expired: bool) -> Result{ + pub async fn init(uri: &str, show_expired: bool, override_cdn: Option) -> Result{ log::debug!("Connecting to Archive of RoboCraft (ARC) vehicle factory database URI: {}", uri); let db = sea_orm::Database::connect(uri).await?; + let good_cdn = override_cdn.map(|s| if s.ends_with("/") { s } else { format!("{}/", s) }); Ok(Self { orm: db, - ignore_expiry: show_expired + ignore_expiry: show_expired, + cdn: good_cdn, }) } @@ -19,6 +22,14 @@ impl ArcAdapter { super::entities::robot_metadata::Entity::find() .order_by_desc(super::entities::robot_metadata::Column::RentCount) } + + fn thumbnail_url(&self, meta: String, id: u32) -> String { + if let Some(cdn) = &self.cdn { + format!("{}{}", cdn, id) + } else { + meta + } + } } #[async_trait::async_trait] @@ -41,7 +52,7 @@ impl crate::VehicleFactoryAdapter for ArcAdapter { id: meta.id as _, name: meta.name, description: meta.description, - thumbnail: meta.thumbnail, + thumbnail: self.thumbnail_url(meta.thumbnail, meta.id), added_by: meta.added_by, added_by_display_name: meta.added_by_display_name, added_date: crate::traits::parse_rc_date(&meta.added_date).unwrap_or_default(),