bonkmaykr/ojrc
1
0
Fork 0
mirror of https://git.ngram.ca/OpenJam/rc-servers synced 2026-08-23 23:08:52 +00:00
Code Activity

Implementing upload functionality for ArcAdapter and improving CRF search. (#62)

Browse Source 
### Description

This PR implements upload functionality for ArcAdapter and improves CRF search.

adapter.rs

First, regarding the CRF search feature, when there is no search query (i.e., on the default page), there was a bug where sorting by added date and other sort options did not work correctly, so the default query was removed. Also, the added-date sort order was reversed, so that was fixed as well.

Next, about the upload feature.

First, in order to create cube_amounts, which stores in JSON the count of each part of the robot contained at the end of the ROBOT_CUBES table, we count—by type—the number of all byte sequences (part IDs) excluding the first 4 bytes (total part count) and excluding the last 4 bytes (coordinates) out of each following 8 bytes, and then create a JSON string where the keys are the part IDs converted to decimal and the values are the counts.

After that, we insert Base64-encoded data into cube_data and colour_data in ROBOT_CUBES, and insert the above JSON string into cube_amounts.

Next, regarding ROBOT_METADATA, the thumbnail URL is set to the internal CDN, and the actual thumbnail data is saved under data/robocraft/thumbnails.

The added date and expiration date are converted to match the same format as the other data before insertion, and the other data is inserted as-is.

arc.rs

If the requested file is not present in the ZIP file, it was changed to scan the JPG files existing in data/robocraft/thumbnails.

package_release.py

Made it so that data/robocraft/thumbnails is created.

### Game

Robocraft

### Please confirm

- [x] I am the legal owner or represent the owner of all work submitted
- [x] I consent to my submission being added to this FOSS project
- [x] This PR used LLMs to generate some or all of the code changes

Reviewed-on: https://git.ngram.ca/OpenJam/rc-servers/pulls/62
Co-authored-by: MaxSignal <kastera58@gmail.com>
Co-committed-by: MaxSignal <kastera58@gmail.com>
This commit is contained in:
MaxSignal
2025-12-13 19:23:07 +00:00
committed by NGnius
parent 0f9f67e5a9
commit aedec62b8d
7 changed files with 165 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
cdn/src/robocraft/factory/arc.rs
View File

@@ -8,10 +8,11 @@ static ZIP_FILE: std::sync::Mutex<Option<zip::read::ZipArchive<std::io::BufReade
pub async fn get(cli: Data<crate::cli::CliArgs>, id: Path<u32>) -> HttpResponse {
let id: u32 = *id;
let zip_path = std::path::PathBuf::from(&cli.data_robocraft).join("rc_archive_thumbnails.zip");
try_find_file(zip_path, id).await
let thumb_dir_path = std::path::PathBuf::from(&cli.data_robocraft).join("factorythumbnails");
try_find_file(zip_path, thumb_dir_path, id).await
}
async fn try_find_file(zip_path: std::path::PathBuf, id: u32) -> HttpResponse {
async fn try_find_file(zip_path: std::path::PathBuf, thumb_dir: std::path::PathBuf, id: u32) -> HttpResponse {
let result = tokio::task::spawn_blocking(move || get_file_in_zip(zip_path, id)).await.unwrap();
match result {
Ok(bytes) => {
@@ -20,6 +21,19 @@ async fn try_find_file(zip_path: std::path::PathBuf, id: u32) -> HttpResponse {
.append_header(("Content-Type", "image/jpeg"))
.body(bytes)
},
Err(zip::result::ZipError::FileNotFound) => {
let result = tokio::task::spawn_blocking(move || get_file_in_thumbnails(thumb_dir, id)).await.unwrap();
match result {
Ok(bytes) => {
log::debug!("Found id {} in thumbnails dir", id);
actix_web::HttpResponseBuilder::new(actix_web::http::StatusCode::OK)
.append_header(("Content-Type", "image/jpeg"))
.body(bytes)
},
Err(_) => actix_web::HttpResponseBuilder::new(actix_web::http::StatusCode::NOT_FOUND)
.body("file not found in zip archive or thumbnails dir".to_string()),
}
},
Err(e) => {
log::debug!("Failed to find id {} in factory arc: {}", id, e);
match e {
@@ -35,10 +49,6 @@ async fn try_find_file(zip_path: std::path::PathBuf, id: u32) -> HttpResponse {
actix_web::HttpResponseBuilder::new(actix_web::http::StatusCode::INTERNAL_SERVER_ERROR)
.body(format!("unsupported zip file: {}", e))
},
zip::result::ZipError::FileNotFound => {
actix_web::HttpResponseBuilder::new(actix_web::http::StatusCode::NOT_FOUND)
.body("file not found in zip archive".to_string())
},
zip::result::ZipError::InvalidPassword => {
actix_web::HttpResponseBuilder::new(actix_web::http::StatusCode::INTERNAL_SERVER_ERROR)
.body("invalid zip password".to_string())
@@ -68,6 +78,19 @@ fn get_file_in_zip(zip_path: std::path::PathBuf, id: u32) -> zip::result::ZipRes
}
fn get_file_in_thumbnails(dir: std::path::PathBuf, id: u32) -> std::io::Result<Vec<u8>> {
let prefix = format!("{} - ", id);
for ent in std::fs::read_dir(&dir)? {
let ent = ent?;
let name = ent.file_name().to_string_lossy().into_owned();
if name.starts_with(&prefix) && name.ends_with(".jpg") {
return std::fs::read(ent.path());
}
}
Err(std::io::Error::new(std::io::ErrorKind::NotFound, "thumbnail not found"))
}
fn read_file_with_prefix(prefix: &str, archive: &mut zip::read::ZipArchive<std::io::BufReader<std::fs::File>>) -> zip::result::ZipResult<Vec<u8>> {
let index = if let Some((index, _)) = archive.file_names().enumerate().find(|(_, name)| name.starts_with(prefix)) {
index

2
rc_core/src/factory/adapter_enum.rs
View File

@@ -34,7 +34,7 @@ impl oj_rc_factory::VehicleFactoryAdapter for Factory {
impl Factory {
pub async fn from_config(conf: &crate::persist::FactoryConfig) -> Result<Self, Box<dyn std::error::Error + 'static>> {
Ok(match &conf.adapter {
crate::persist::AdapterSettings::Arc(x) => Self::Arc(oj_rc_factory::arc::ArcAdapter::init(&x.uri, x.show_expired, x.override_cdn.clone(), x.spoof_username).await?),
crate::persist::AdapterSettings::Arc(x) => Self::Arc(oj_rc_factory::arc::ArcAdapter::init(&x.uri, x.show_expired, x.cdn.clone(), x.override_cdn, x.spoof_username).await?),
crate::persist::AdapterSettings::None => Self::None,
})
}

2
rc_core/src/persist/settings.rs
View File

@@ -124,7 +124,7 @@ fn default_server_conf() -> ServerSettings {
}
}
fn default_cdn_root_url() -> String {
pub fn default_cdn_root_url() -> String {
"http://127.0.0.1:8010".to_owned()
}

8
rc_core/src/persist/vehicle_factory.rs
View File

@@ -32,12 +32,18 @@ pub struct ArcFactorySettings {
#[serde(default = "default_true")]
pub show_expired: bool,
/// should probably end with /roboshop/arc/Live/
#[serde(default = "default_arc_live_url")]
pub cdn: String,
#[serde(default)]
pub override_cdn: Option<String>,
pub override_cdn: bool,
#[serde(default = "default_true")]
pub spoof_username: bool,
}
fn default_arc_live_url() -> String {
format!("{}/roboshop/arc/Live/", super::settings::default_cdn_root_url())
}
fn default_true() -> bool {
true
}

2
rc_factory/Cargo.toml
View File

@@ -15,3 +15,5 @@ chrono.workspace = true
log.workspace = true
base64.workspace = true
hex.workspace = true
serde_json = "1.0"
tokio = "1.48.0"

197
rc_factory/src/arc/adapter.rs
View File

@@ -1,21 +1,26 @@
use sea_orm::{ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder};
use sea_orm::{ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder, ActiveModelTrait, Set, TransactionTrait};
use base64::Engine;
use std::collections::HashMap;
use std::path::Path;
pub struct ArcAdapter {
orm: sea_orm::DatabaseConnection,
ignore_expiry: bool,
cdn: Option<String>,
cdn: String,
override_cdn: bool,
spoof_users: bool,
}
impl ArcAdapter {
pub async fn init(uri: &str, show_expired: bool, override_cdn: Option<String>, username_spoofing: bool) -> Result<Self, sea_orm::DbErr>{
pub async fn init(uri: &str, show_expired: bool, cdn: String, override_cdn: bool, username_spoofing: bool) -> Result<Self, sea_orm::DbErr>{
log::debug!("Connecting to Archive of RoboCraft (ARC) vehicle factory database URI: {}", uri);
let db = sea_orm::Database::connect(uri).await?;
let good_cdn = override_cdn.map(|s| if s.ends_with("/") { s } else { format!("{}/", s) });
let good_cdn = if cdn.ends_with('/') { cdn } else { format!("{}/", cdn) };
let adapter = Self {
orm: db,
ignore_expiry: show_expired,
cdn: good_cdn,
override_cdn: override_cdn,
spoof_users: username_spoofing,
};
// do query to ensure database is ok
@@ -30,8 +35,8 @@ impl ArcAdapter {
}
fn thumbnail_url(&self, meta: String, id: u32) -> String {
if let Some(cdn) = &self.cdn {
format!("{}{}", cdn, id)
if self.override_cdn {
format!("{}{}", &self.cdn, id)
} else {
meta
}
@@ -85,72 +90,67 @@ impl crate::VehicleFactoryAdapter for ArcAdapter {
async fn list(&self, query: libfj::robocraft::ListQuery) -> Result<Vec<crate::VehicleQueryInfo>, Box<dyn std::error::Error>> {
log::debug!("Search vehicles with query {:?}", query);
let query_params = if query.default_page {
log::debug!("Default vehicle list query");
self.default_query()
} else {
let mut query_builder = super::entities::robot_metadata::Entity::find();
match query.order {
libfj::robocraft::FactoryOrderType::Suggested => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::RentCount); },
libfj::robocraft::FactoryOrderType::CombatRating => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::CombatRating); },
libfj::robocraft::FactoryOrderType::CosmeticRating => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::CosmeticRating); },
libfj::robocraft::FactoryOrderType::Added => { query_builder = query_builder.order_by_asc(super::entities::robot_metadata::Column::AddedDate); },
libfj::robocraft::FactoryOrderType::CPU => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::Cpu); },
libfj::robocraft::FactoryOrderType::MostBought => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::BuyCount); },
}
if !query.text_filter.is_empty() {
let query_text = format!("%{}%", query.text_filter.replace('%', ""));
if query.player_filter {
query_builder = query_builder.filter(
sea_orm::sea_query::Condition::any()
.add(super::entities::robot_metadata::Column::AddedBy.like(query_text.clone()))
.add(super::entities::robot_metadata::Column::AddedByDisplayName.like(query_text))
);
} else {
match query.text_search_field {
libfj::robocraft::FactoryTextSearchField::All => {
query_builder = query_builder.filter(
sea_orm::sea_query::Condition::any()
.add(super::entities::robot_metadata::Column::AddedBy.like(query_text.clone()))
.add(super::entities::robot_metadata::Column::AddedByDisplayName.like(query_text.clone()))
.add(super::entities::robot_metadata::Column::Name.like(query_text.clone()))
.add(super::entities::robot_metadata::Column::Description.like(query_text))
);
},
libfj::robocraft::FactoryTextSearchField::Name => {
query_builder = query_builder.filter(
sea_orm::sea_query::Condition::any()
.add(super::entities::robot_metadata::Column::Name.like(query_text.clone()))
);
},
libfj::robocraft::FactoryTextSearchField::Player => {
query_builder = query_builder.filter(
sea_orm::sea_query::Condition::any()
.add(super::entities::robot_metadata::Column::AddedBy.like(query_text.clone()))
.add(super::entities::robot_metadata::Column::AddedByDisplayName.like(query_text))
);
},
}
let mut query_builder = super::entities::robot_metadata::Entity::find();
match query.order {
libfj::robocraft::FactoryOrderType::Suggested => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::RentCount); },
libfj::robocraft::FactoryOrderType::CombatRating => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::CombatRating); },
libfj::robocraft::FactoryOrderType::CosmeticRating => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::CosmeticRating); },
libfj::robocraft::FactoryOrderType::Added => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::AddedDate); },
libfj::robocraft::FactoryOrderType::CPU => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::Cpu); },
libfj::robocraft::FactoryOrderType::MostBought => { query_builder = query_builder.order_by_desc(super::entities::robot_metadata::Column::BuyCount); },
}
if !query.text_filter.is_empty() {
let query_text = format!("%{}%", query.text_filter.replace('%', ""));
if query.player_filter {
query_builder = query_builder.filter(
sea_orm::sea_query::Condition::any()
.add(super::entities::robot_metadata::Column::AddedBy.like(query_text.clone()))
.add(super::entities::robot_metadata::Column::AddedByDisplayName.like(query_text))
);
} else {
match query.text_search_field {
libfj::robocraft::FactoryTextSearchField::All => {
query_builder = query_builder.filter(
sea_orm::sea_query::Condition::any()
.add(super::entities::robot_metadata::Column::AddedBy.like(query_text.clone()))
.add(super::entities::robot_metadata::Column::AddedByDisplayName.like(query_text.clone()))
.add(super::entities::robot_metadata::Column::Name.like(query_text.clone()))
.add(super::entities::robot_metadata::Column::Description.like(query_text))
);
},
libfj::robocraft::FactoryTextSearchField::Name => {
query_builder = query_builder.filter(
sea_orm::sea_query::Condition::any()
.add(super::entities::robot_metadata::Column::Name.like(query_text.clone()))
);
},
libfj::robocraft::FactoryTextSearchField::Player => {
query_builder = query_builder.filter(
sea_orm::sea_query::Condition::any()
.add(super::entities::robot_metadata::Column::AddedBy.like(query_text.clone()))
.add(super::entities::robot_metadata::Column::AddedByDisplayName.like(query_text))
);
},
}
}
// movement filters not supported
// weapon filters not supported
if query.minimum_cpu > 0 {
query_builder = query_builder.filter(super::entities::robot_metadata::Column::Cpu.gte(query.minimum_cpu as u32));
}
if query.maximum_cpu < usize::MAX {
query_builder = query_builder.filter(super::entities::robot_metadata::Column::Cpu.lte(query.maximum_cpu as u32));
}
if query.buyable {
query_builder = query_builder.filter(super::entities::robot_metadata::Column::Buyable.ne(0));
}
if !self.ignore_expiry {
let now_str = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true).trim_end_matches('Z').to_owned();
log::debug!("Expiry must exceed `{}`", now_str);
query_builder = query_builder.filter(super::entities::robot_metadata::Column::ExpiryDate.gte(now_str))
}
query_builder
};
}
// movement filters not supported
// weapon filters not supported
if query.minimum_cpu > 0 {
query_builder = query_builder.filter(super::entities::robot_metadata::Column::Cpu.gte(query.minimum_cpu as u32));
}
if query.maximum_cpu < usize::MAX {
query_builder = query_builder.filter(super::entities::robot_metadata::Column::Cpu.lte(query.maximum_cpu as u32));
}
if query.buyable {
query_builder = query_builder.filter(super::entities::robot_metadata::Column::Buyable.ne(0));
}
if !self.ignore_expiry {
let now_str = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true).trim_end_matches('Z').to_owned();
log::debug!("Expiry must exceed `{}`", now_str);
query_builder = query_builder.filter(super::entities::robot_metadata::Column::ExpiryDate.gte(now_str))
}
let query_params = query_builder;
// FIXME add support for query.prepend_featured_bot
let metadata_pages = query_params.paginate(&self.orm, query.page_size as u64);
@@ -187,8 +187,59 @@ impl crate::VehicleFactoryAdapter for ArcAdapter {
Ok(infos)
}
async fn upload(&self, _vehicle: crate::VehicleUploadInfo) -> Result<bool, Box<dyn std::error::Error>> {
log::info!("Arc adapter does not support uploading factory vehicles");
Ok(false)
async fn upload(&self, vehicle: crate::VehicleUploadInfo) -> Result<bool, Box<dyn std::error::Error>> {
let transaction = self.orm.begin().await?;
let cube_amounts = {
let mut counts: HashMap<u32, u32> = HashMap::new();
let data = &vehicle.cube_data[4..];
for chunk in data.chunks_exact(8) {
let id_bytes: [u8; 4] = chunk[0..4].try_into().unwrap();
let part_id = u32::from_le_bytes(id_bytes);
*counts.entry(part_id).or_insert(0) += 1;
}
let mut str_map: HashMap<String, u32> = HashMap::new();
for (k, v) in counts {
str_map.insert(k.to_string(), v);
}
serde_json::to_string(&str_map).unwrap_or_else(|_| "{}".to_string())
};
let cubes = super::entities::robot_cubes::ActiveModel {
cube_data: Set(base64::prelude::BASE64_STANDARD.encode(&vehicle.cube_data)),
colour_data: Set(base64::prelude::BASE64_STANDARD.encode(&vehicle.colour_data)),
cube_amounts: Set(cube_amounts),
..Default::default()
}.insert(&transaction).await?;
let now = chrono::Utc::now();
let meta = super::entities::robot_metadata::ActiveModel {
name: Set(vehicle.name),
description: Set(vehicle.description),
thumbnail: Set(format!("{}{}", &self.cdn, cubes.id)),
added_by: Set(vehicle.added_by),
added_by_display_name: Set(vehicle.added_by_display_name),
added_date: Set(now.format("%Y-%m-%dT%H:%M:%S").to_string()),
expiry_date: Set((now + chrono::Duration::days(365 * 2)).format("%Y-%m-%dT%H:%M:%S").to_string()),
cpu: Set(vehicle.cpu),
total_robot_ranking: Set(vehicle.total_robot_ranking as i32),
rent_count: Set(0),
buy_count: Set(0),
buyable: Set(1),
featured: Set(0),
combat_rating: Set(3.0),
cosmetic_rating: Set(3.0),
..Default::default()
}.insert(&transaction).await?;
transaction.commit().await?;
let entry_name = format!("{} - {}.jpg", meta.id, meta.name);
let thumbnail = vehicle.thumbnail;
tokio::task::spawn_blocking(move || {
std::fs::write(Path::new("../data/robocraft/factorythumbnails").join(&entry_name), &thumbnail).ok();
});
Ok(true)
}
}

1
utils/package_release.py
View File

@@ -58,6 +58,7 @@ BLANK_FOLDERS_TO_CREATE = [
"data/robocraft/campaigndata",
"data/robocraft/clanavatar",
"data/robocraft/customavatars",
"data/robocraft/factorythumbnails",
]
def add_folder_to_zip(archive: zipfile.ZipFile, root_dir: str):