From aedec62b8d837a3d0cfed1e3dac310ae61ac3e4a Mon Sep 17 00:00:00 2001 From: MaxSignal Date: Sat, 13 Dec 2025 19:23:07 +0000 Subject: [PATCH] Implementing upload functionality for ArcAdapter and improving CRF search. (#62) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Description This PR implements upload functionality for ArcAdapter and improves CRF search. adapter.rs First, regarding the CRF search feature, when there is no search query (i.e., on the default page), there was a bug where sorting by added date and other sort options did not work correctly, so the default query was removed. Also, the added-date sort order was reversed, so that was fixed as well. Next, about the upload feature. First, in order to create cube_amounts, which stores in JSON the count of each part of the robot contained at the end of the ROBOT_CUBES table, we count—by type—the number of all byte sequences (part IDs) excluding the first 4 bytes (total part count) and excluding the last 4 bytes (coordinates) out of each following 8 bytes, and then create a JSON string where the keys are the part IDs converted to decimal and the values are the counts. After that, we insert Base64-encoded data into cube_data and colour_data in ROBOT_CUBES, and insert the above JSON string into cube_amounts. Next, regarding ROBOT_METADATA, the thumbnail URL is set to the internal CDN, and the actual thumbnail data is saved under data/robocraft/thumbnails. The added date and expiration date are converted to match the same format as the other data before insertion, and the other data is inserted as-is. arc.rs If the requested file is not present in the ZIP file, it was changed to scan the JPG files existing in data/robocraft/thumbnails. package_release.py Made it so that data/robocraft/thumbnails is created. ### Game Robocraft ### Please confirm - [x] I am the legal owner or represent the owner of all work submitted - [x] I consent to my submission being added to this FOSS project - [x] This PR used LLMs to generate some or all of the code changes Reviewed-on: https://git.ngram.ca/OpenJam/rc-servers/pulls/62 Co-authored-by: MaxSignal Co-committed-by: MaxSignal --- cdn/src/robocraft/factory/arc.rs | 35 ++++- rc_core/src/factory/adapter_enum.rs | 2 +- rc_core/src/persist/settings.rs | 2 +- rc_core/src/persist/vehicle_factory.rs | 8 +- rc_factory/Cargo.toml | 2 + rc_factory/src/arc/adapter.rs | 197 ++++++++++++++++--------- utils/package_release.py | 1 + 7 files changed, 165 insertions(+), 82 deletions(-) diff --git a/cdn/src/robocraft/factory/arc.rs b/cdn/src/robocraft/factory/arc.rs index ed6baf3..c9c963b 100644 --- a/cdn/src/robocraft/factory/arc.rs +++ b/cdn/src/robocraft/factory/arc.rs @@ -8,10 +8,11 @@ static ZIP_FILE: std::sync::Mutex, id: Path) -> HttpResponse { let id: u32 = *id; let zip_path = std::path::PathBuf::from(&cli.data_robocraft).join("rc_archive_thumbnails.zip"); - try_find_file(zip_path, id).await + let thumb_dir_path = std::path::PathBuf::from(&cli.data_robocraft).join("factorythumbnails"); + try_find_file(zip_path, thumb_dir_path, id).await } -async fn try_find_file(zip_path: std::path::PathBuf, id: u32) -> HttpResponse { +async fn try_find_file(zip_path: std::path::PathBuf, thumb_dir: std::path::PathBuf, id: u32) -> HttpResponse { let result = tokio::task::spawn_blocking(move || get_file_in_zip(zip_path, id)).await.unwrap(); match result { Ok(bytes) => { @@ -20,6 +21,19 @@ async fn try_find_file(zip_path: std::path::PathBuf, id: u32) -> HttpResponse { .append_header(("Content-Type", "image/jpeg")) .body(bytes) }, + Err(zip::result::ZipError::FileNotFound) => { + let result = tokio::task::spawn_blocking(move || get_file_in_thumbnails(thumb_dir, id)).await.unwrap(); + match result { + Ok(bytes) => { + log::debug!("Found id {} in thumbnails dir", id); + actix_web::HttpResponseBuilder::new(actix_web::http::StatusCode::OK) + .append_header(("Content-Type", "image/jpeg")) + .body(bytes) + }, + Err(_) => actix_web::HttpResponseBuilder::new(actix_web::http::StatusCode::NOT_FOUND) + .body("file not found in zip archive or thumbnails dir".to_string()), + } + }, Err(e) => { log::debug!("Failed to find id {} in factory arc: {}", id, e); match e { @@ -35,10 +49,6 @@ async fn try_find_file(zip_path: std::path::PathBuf, id: u32) -> HttpResponse { actix_web::HttpResponseBuilder::new(actix_web::http::StatusCode::INTERNAL_SERVER_ERROR) .body(format!("unsupported zip file: {}", e)) }, - zip::result::ZipError::FileNotFound => { - actix_web::HttpResponseBuilder::new(actix_web::http::StatusCode::NOT_FOUND) - .body("file not found in zip archive".to_string()) - }, zip::result::ZipError::InvalidPassword => { actix_web::HttpResponseBuilder::new(actix_web::http::StatusCode::INTERNAL_SERVER_ERROR) .body("invalid zip password".to_string()) @@ -68,6 +78,19 @@ fn get_file_in_zip(zip_path: std::path::PathBuf, id: u32) -> zip::result::ZipRes } +fn get_file_in_thumbnails(dir: std::path::PathBuf, id: u32) -> std::io::Result> { + let prefix = format!("{} - ", id); + for ent in std::fs::read_dir(&dir)? { + let ent = ent?; + let name = ent.file_name().to_string_lossy().into_owned(); + if name.starts_with(&prefix) && name.ends_with(".jpg") { + return std::fs::read(ent.path()); + } + } + + Err(std::io::Error::new(std::io::ErrorKind::NotFound, "thumbnail not found")) +} + fn read_file_with_prefix(prefix: &str, archive: &mut zip::read::ZipArchive>) -> zip::result::ZipResult> { let index = if let Some((index, _)) = archive.file_names().enumerate().find(|(_, name)| name.starts_with(prefix)) { index diff --git a/rc_core/src/factory/adapter_enum.rs b/rc_core/src/factory/adapter_enum.rs index 5f2a278..7188c1b 100644 --- a/rc_core/src/factory/adapter_enum.rs +++ b/rc_core/src/factory/adapter_enum.rs @@ -34,7 +34,7 @@ impl oj_rc_factory::VehicleFactoryAdapter for Factory { impl Factory { pub async fn from_config(conf: &crate::persist::FactoryConfig) -> Result> { Ok(match &conf.adapter { - crate::persist::AdapterSettings::Arc(x) => Self::Arc(oj_rc_factory::arc::ArcAdapter::init(&x.uri, x.show_expired, x.override_cdn.clone(), x.spoof_username).await?), + crate::persist::AdapterSettings::Arc(x) => Self::Arc(oj_rc_factory::arc::ArcAdapter::init(&x.uri, x.show_expired, x.cdn.clone(), x.override_cdn, x.spoof_username).await?), crate::persist::AdapterSettings::None => Self::None, }) } diff --git a/rc_core/src/persist/settings.rs b/rc_core/src/persist/settings.rs index f5f626a..e911e1a 100644 --- a/rc_core/src/persist/settings.rs +++ b/rc_core/src/persist/settings.rs @@ -124,7 +124,7 @@ fn default_server_conf() -> ServerSettings { } } -fn default_cdn_root_url() -> String { +pub fn default_cdn_root_url() -> String { "http://127.0.0.1:8010".to_owned() } diff --git a/rc_core/src/persist/vehicle_factory.rs b/rc_core/src/persist/vehicle_factory.rs index 039d763..63ad49b 100644 --- a/rc_core/src/persist/vehicle_factory.rs +++ b/rc_core/src/persist/vehicle_factory.rs @@ -32,12 +32,18 @@ pub struct ArcFactorySettings { #[serde(default = "default_true")] pub show_expired: bool, /// should probably end with /roboshop/arc/Live/ + #[serde(default = "default_arc_live_url")] + pub cdn: String, #[serde(default)] - pub override_cdn: Option, + pub override_cdn: bool, #[serde(default = "default_true")] pub spoof_username: bool, } +fn default_arc_live_url() -> String { + format!("{}/roboshop/arc/Live/", super::settings::default_cdn_root_url()) +} + fn default_true() -> bool { true } diff --git a/rc_factory/Cargo.toml b/rc_factory/Cargo.toml index b96aa79..0cca727 100644 --- a/rc_factory/Cargo.toml +++ b/rc_factory/Cargo.toml @@ -15,3 +15,5 @@ chrono.workspace = true log.workspace = true base64.workspace = true hex.workspace = true +serde_json = "1.0" +tokio = "1.48.0" diff --git a/rc_factory/src/arc/adapter.rs b/rc_factory/src/arc/adapter.rs index f1828d3..33a5048 100644 --- a/rc_factory/src/arc/adapter.rs +++ b/rc_factory/src/arc/adapter.rs @@ -1,21 +1,26 @@ -use sea_orm::{ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder}; +use sea_orm::{ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder, ActiveModelTrait, Set, TransactionTrait}; +use base64::Engine; +use std::collections::HashMap; +use std::path::Path; pub struct ArcAdapter { orm: sea_orm::DatabaseConnection, ignore_expiry: bool, - cdn: Option, + cdn: String, + override_cdn: bool, spoof_users: bool, } impl ArcAdapter { - pub async fn init(uri: &str, show_expired: bool, override_cdn: Option, username_spoofing: bool) -> Result{ + pub async fn init(uri: &str, show_expired: bool, cdn: String, override_cdn: bool, username_spoofing: bool) -> Result{ log::debug!("Connecting to Archive of RoboCraft (ARC) vehicle factory database URI: {}", uri); let db = sea_orm::Database::connect(uri).await?; - let good_cdn = override_cdn.map(|s| if s.ends_with("/") { s } else { format!("{}/", s) }); + let good_cdn = if cdn.ends_with('/') { cdn } else { format!("{}/", cdn) }; let adapter = Self { orm: db, ignore_expiry: show_expired, cdn: good_cdn, + override_cdn: override_cdn, spoof_users: username_spoofing, }; // do query to ensure database is ok @@ -30,8 +35,8 @@ impl ArcAdapter { } fn thumbnail_url(&self, meta: String, id: u32) -> String { - if let Some(cdn) = &self.cdn { - format!("{}{}", cdn, id) + if self.override_cdn { + format!("{}{}", &self.cdn, id) } else { meta } @@ -85,72 +90,67 @@ impl crate::VehicleFactoryAdapter for ArcAdapter { async fn list(&self, query: libfj::robocraft::ListQuery) -> Result, Box> { log::debug!("Search vehicles with query {:?}", query); - let query_params = if query.default_page { - log::debug!("Default vehicle list query"); - self.default_query() - } else { - let mut query_builder = super::entities::robot_metadata::Entity::find(); - match query.order { - libfj::robocraft::FactoryOrderType::Suggested => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::RentCount); }, - libfj::robocraft::FactoryOrderType::CombatRating => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::CombatRating); }, - libfj::robocraft::FactoryOrderType::CosmeticRating => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::CosmeticRating); }, - libfj::robocraft::FactoryOrderType::Added => { query_builder = query_builder.order_by_asc(super::entities::robot_metadata::Column::AddedDate); }, - libfj::robocraft::FactoryOrderType::CPU => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::Cpu); }, - libfj::robocraft::FactoryOrderType::MostBought => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::BuyCount); }, - } - if !query.text_filter.is_empty() { - let query_text = format!("%{}%", query.text_filter.replace('%', "")); - if query.player_filter { - query_builder = query_builder.filter( - sea_orm::sea_query::Condition::any() - .add(super::entities::robot_metadata::Column::AddedBy.like(query_text.clone())) - .add(super::entities::robot_metadata::Column::AddedByDisplayName.like(query_text)) - ); - } else { - match query.text_search_field { - libfj::robocraft::FactoryTextSearchField::All => { - query_builder = query_builder.filter( - sea_orm::sea_query::Condition::any() - .add(super::entities::robot_metadata::Column::AddedBy.like(query_text.clone())) - .add(super::entities::robot_metadata::Column::AddedByDisplayName.like(query_text.clone())) - .add(super::entities::robot_metadata::Column::Name.like(query_text.clone())) - .add(super::entities::robot_metadata::Column::Description.like(query_text)) - ); - }, - libfj::robocraft::FactoryTextSearchField::Name => { - query_builder = query_builder.filter( - sea_orm::sea_query::Condition::any() - .add(super::entities::robot_metadata::Column::Name.like(query_text.clone())) - ); - }, - libfj::robocraft::FactoryTextSearchField::Player => { - query_builder = query_builder.filter( - sea_orm::sea_query::Condition::any() - .add(super::entities::robot_metadata::Column::AddedBy.like(query_text.clone())) - .add(super::entities::robot_metadata::Column::AddedByDisplayName.like(query_text)) - ); - }, - } + let mut query_builder = super::entities::robot_metadata::Entity::find(); + match query.order { + libfj::robocraft::FactoryOrderType::Suggested => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::RentCount); }, + libfj::robocraft::FactoryOrderType::CombatRating => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::CombatRating); }, + libfj::robocraft::FactoryOrderType::CosmeticRating => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::CosmeticRating); }, + libfj::robocraft::FactoryOrderType::Added => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::AddedDate); }, + libfj::robocraft::FactoryOrderType::CPU => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::Cpu); }, + libfj::robocraft::FactoryOrderType::MostBought => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::BuyCount); }, + } + if !query.text_filter.is_empty() { + let query_text = format!("%{}%", query.text_filter.replace('%', "")); + if query.player_filter { + query_builder = query_builder.filter( + sea_orm::sea_query::Condition::any() + .add(super::entities::robot_metadata::Column::AddedBy.like(query_text.clone())) + .add(super::entities::robot_metadata::Column::AddedByDisplayName.like(query_text)) + ); + } else { + match query.text_search_field { + libfj::robocraft::FactoryTextSearchField::All => { + query_builder = query_builder.filter( + sea_orm::sea_query::Condition::any() + .add(super::entities::robot_metadata::Column::AddedBy.like(query_text.clone())) + .add(super::entities::robot_metadata::Column::AddedByDisplayName.like(query_text.clone())) + .add(super::entities::robot_metadata::Column::Name.like(query_text.clone())) + .add(super::entities::robot_metadata::Column::Description.like(query_text)) + ); + }, + libfj::robocraft::FactoryTextSearchField::Name => { + query_builder = query_builder.filter( + sea_orm::sea_query::Condition::any() + .add(super::entities::robot_metadata::Column::Name.like(query_text.clone())) + ); + }, + libfj::robocraft::FactoryTextSearchField::Player => { + query_builder = query_builder.filter( + sea_orm::sea_query::Condition::any() + .add(super::entities::robot_metadata::Column::AddedBy.like(query_text.clone())) + .add(super::entities::robot_metadata::Column::AddedByDisplayName.like(query_text)) + ); + }, } } - // movement filters not supported - // weapon filters not supported - if query.minimum_cpu > 0 { - query_builder = query_builder.filter(super::entities::robot_metadata::Column::Cpu.gte(query.minimum_cpu as u32)); - } - if query.maximum_cpu < usize::MAX { - query_builder = query_builder.filter(super::entities::robot_metadata::Column::Cpu.lte(query.maximum_cpu as u32)); - } - if query.buyable { - query_builder = query_builder.filter(super::entities::robot_metadata::Column::Buyable.ne(0)); - } - if !self.ignore_expiry { - let now_str = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true).trim_end_matches('Z').to_owned(); - log::debug!("Expiry must exceed `{}`", now_str); - query_builder = query_builder.filter(super::entities::robot_metadata::Column::ExpiryDate.gte(now_str)) - } - query_builder - }; + } + // movement filters not supported + // weapon filters not supported + if query.minimum_cpu > 0 { + query_builder = query_builder.filter(super::entities::robot_metadata::Column::Cpu.gte(query.minimum_cpu as u32)); + } + if query.maximum_cpu < usize::MAX { + query_builder = query_builder.filter(super::entities::robot_metadata::Column::Cpu.lte(query.maximum_cpu as u32)); + } + if query.buyable { + query_builder = query_builder.filter(super::entities::robot_metadata::Column::Buyable.ne(0)); + } + if !self.ignore_expiry { + let now_str = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true).trim_end_matches('Z').to_owned(); + log::debug!("Expiry must exceed `{}`", now_str); + query_builder = query_builder.filter(super::entities::robot_metadata::Column::ExpiryDate.gte(now_str)) + } + let query_params = query_builder; // FIXME add support for query.prepend_featured_bot let metadata_pages = query_params.paginate(&self.orm, query.page_size as u64); @@ -187,8 +187,59 @@ impl crate::VehicleFactoryAdapter for ArcAdapter { Ok(infos) } - async fn upload(&self, _vehicle: crate::VehicleUploadInfo) -> Result> { - log::info!("Arc adapter does not support uploading factory vehicles"); - Ok(false) + async fn upload(&self, vehicle: crate::VehicleUploadInfo) -> Result> { + let transaction = self.orm.begin().await?; + + let cube_amounts = { + let mut counts: HashMap = HashMap::new(); + let data = &vehicle.cube_data[4..]; + for chunk in data.chunks_exact(8) { + let id_bytes: [u8; 4] = chunk[0..4].try_into().unwrap(); + let part_id = u32::from_le_bytes(id_bytes); + + *counts.entry(part_id).or_insert(0) += 1; + } + let mut str_map: HashMap = HashMap::new(); + for (k, v) in counts { + str_map.insert(k.to_string(), v); + } + + serde_json::to_string(&str_map).unwrap_or_else(|_| "{}".to_string()) + }; + let cubes = super::entities::robot_cubes::ActiveModel { + cube_data: Set(base64::prelude::BASE64_STANDARD.encode(&vehicle.cube_data)), + colour_data: Set(base64::prelude::BASE64_STANDARD.encode(&vehicle.colour_data)), + cube_amounts: Set(cube_amounts), + ..Default::default() + }.insert(&transaction).await?; + + let now = chrono::Utc::now(); + let meta = super::entities::robot_metadata::ActiveModel { + name: Set(vehicle.name), + description: Set(vehicle.description), + thumbnail: Set(format!("{}{}", &self.cdn, cubes.id)), + added_by: Set(vehicle.added_by), + added_by_display_name: Set(vehicle.added_by_display_name), + added_date: Set(now.format("%Y-%m-%dT%H:%M:%S").to_string()), + expiry_date: Set((now + chrono::Duration::days(365 * 2)).format("%Y-%m-%dT%H:%M:%S").to_string()), + cpu: Set(vehicle.cpu), + total_robot_ranking: Set(vehicle.total_robot_ranking as i32), + rent_count: Set(0), + buy_count: Set(0), + buyable: Set(1), + featured: Set(0), + combat_rating: Set(3.0), + cosmetic_rating: Set(3.0), + ..Default::default() + }.insert(&transaction).await?; + transaction.commit().await?; + + let entry_name = format!("{} - {}.jpg", meta.id, meta.name); + let thumbnail = vehicle.thumbnail; + tokio::task::spawn_blocking(move || { + std::fs::write(Path::new("../data/robocraft/factorythumbnails").join(&entry_name), &thumbnail).ok(); + }); + + Ok(true) } } diff --git a/utils/package_release.py b/utils/package_release.py index f5e860b..982c8b1 100755 --- a/utils/package_release.py +++ b/utils/package_release.py @@ -58,6 +58,7 @@ BLANK_FOLDERS_TO_CREATE = [ "data/robocraft/campaigndata", "data/robocraft/clanavatar", "data/robocraft/customavatars", + "data/robocraft/factorythumbnails", ] def add_folder_to_zip(archive: zipfile.ZipFile, root_dir: str):