From 08fac7af039d8b9850b649fcfff228320b75a3d8 Mon Sep 17 00:00:00 2001 From: red binder Date: Mon, 20 Apr 2026 14:06:04 +0200 Subject: [PATCH] Even more work on datastore --- ...b44a630eb066846c777c462b8c9341e26b696.json | 38 --- ...d3ffbd9f8d5304f6aa6d785dbddbb1626c0e9.json | 28 --- ...077f9b06ffe11cab2693e4e365c265f8e2418.json | 125 ---------- ...88d2d26f7644e85bfd65423e6e32c469e3695.json | 16 -- ...b8158ea907d36588e1e59f6cbe752d89361bb.json | 22 -- ...f7803ee88233c53efaa2df9966a6c72c8c763.json | 23 -- ...a4beaf37e879c28c81c22abeddb3d030535df.json | 28 --- ...8be8946430d93821dbccb6dc9324ef0544c17.json | 29 --- ...1d9f4ef96adbdab7ca7d008acf596af91fc2b.json | 112 --------- ...e1efdbf3cef31e4b6cea4adbd51ee0218f5aa.json | 40 ---- ...fe18e500d243f1b55fd18e7d96d281605bcf9.json | 14 -- Cargo.lock | 1 + rnex-core/Cargo.toml | 1 + rnex-core/src/nex/datastore.rs | 226 ++++++++++++++++-- rnex-core/src/nex/s3presigner.rs | 44 ++++ rnex-core/src/rmc/protocols/datastore.rs | 64 ++++- rnex-core/src/rmc/protocols/ranking.rs | 9 +- rnex-core/src/rmc/structures/mod.rs | 1 + rnex-core/src/rmc/structures/qbuffer.rs | 2 +- rnex-core/src/rmc/structures/resultsrange.rs | 8 + 20 files changed, 327 insertions(+), 504 deletions(-) delete mode 100644 .sqlx/query-039d134f0d1b681b55b1394a710b44a630eb066846c777c462b8c9341e26b696.json delete mode 100644 .sqlx/query-164708b549c483a041d2e54065ed3ffbd9f8d5304f6aa6d785dbddbb1626c0e9.json delete mode 100644 .sqlx/query-18fb671d9d97353420446413b74077f9b06ffe11cab2693e4e365c265f8e2418.json delete mode 100644 .sqlx/query-29e18f241dc860dca12c03cabb788d2d26f7644e85bfd65423e6e32c469e3695.json delete mode 100644 .sqlx/query-2ff34379bbc32276c3b78ef1283b8158ea907d36588e1e59f6cbe752d89361bb.json delete mode 100644 .sqlx/query-48d94383f1a261f19060f5c75d9f7803ee88233c53efaa2df9966a6c72c8c763.json delete mode 100644 .sqlx/query-60d8ebcf914d887439096df400fa4beaf37e879c28c81c22abeddb3d030535df.json delete mode 100644 .sqlx/query-8909f81fbf26a9b559d26e138568be8946430d93821dbccb6dc9324ef0544c17.json delete mode 100644 .sqlx/query-a5e86d7e66d0e16e5f35df883e11d9f4ef96adbdab7ca7d008acf596af91fc2b.json delete mode 100644 .sqlx/query-c2a0ca72fdcb060a7fb1c670b25e1efdbf3cef31e4b6cea4adbd51ee0218f5aa.json delete mode 100644 .sqlx/query-e28d8776cc49b55fe76cf33ac12fe18e500d243f1b55fd18e7d96d281605bcf9.json create mode 100644 rnex-core/src/rmc/structures/resultsrange.rs diff --git a/.sqlx/query-039d134f0d1b681b55b1394a710b44a630eb066846c777c462b8c9341e26b696.json b/.sqlx/query-039d134f0d1b681b55b1394a710b44a630eb066846c777c462b8c9341e26b696.json deleted file mode 100644 index f91446a..0000000 --- a/.sqlx/query-039d134f0d1b681b55b1394a710b44a630eb066846c777c462b8c9341e26b696.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO datastore.objects (\n owner, size, name, data_type, meta_binary,\n permission, permission_recipients,\n delete_permission, delete_permission_recipients,\n flag, period, refer_data_id, tags,\n persistence_slot_id, extra_data, creation_date, update_date\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17\n ) RETURNING data_id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "data_id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int4", - "Int4", - "Text", - "Int4", - "Bytea", - "Int4", - "Int4Array", - "Int4", - "Int4Array", - "Int4", - "Int4", - "Int8", - "TextArray", - "Int4", - "TextArray", - "Timestamp", - "Timestamp" - ] - }, - "nullable": [ - false - ] - }, - "hash": "039d134f0d1b681b55b1394a710b44a630eb066846c777c462b8c9341e26b696" -} diff --git a/.sqlx/query-164708b549c483a041d2e54065ed3ffbd9f8d5304f6aa6d785dbddbb1626c0e9.json b/.sqlx/query-164708b549c483a041d2e54065ed3ffbd9f8d5304f6aa6d785dbddbb1626c0e9.json deleted file mode 100644 index abf5c03..0000000 --- a/.sqlx/query-164708b549c483a041d2e54065ed3ffbd9f8d5304f6aa6d785dbddbb1626c0e9.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT owner, under_review FROM datastore.objects WHERE data_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "owner", - "type_info": "Int4" - }, - { - "ordinal": 1, - "name": "under_review", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - true, - false - ] - }, - "hash": "164708b549c483a041d2e54065ed3ffbd9f8d5304f6aa6d785dbddbb1626c0e9" -} diff --git a/.sqlx/query-18fb671d9d97353420446413b74077f9b06ffe11cab2693e4e365c265f8e2418.json b/.sqlx/query-18fb671d9d97353420446413b74077f9b06ffe11cab2693e4e365c265f8e2418.json deleted file mode 100644 index 7471873..0000000 --- a/.sqlx/query-18fb671d9d97353420446413b74077f9b06ffe11cab2693e4e365c265f8e2418.json +++ /dev/null @@ -1,125 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT data_id, owner, size, name, data_type, meta_binary,\n permission, permission_recipients, delete_permission, delete_permission_recipients,\n period, refer_data_id, flag, tags, creation_date, update_date,\n access_password, under_review\n FROM datastore.objects\n WHERE owner = $1 AND persistence_slot_id = $2\n AND upload_completed = TRUE AND deleted = FALSE", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "data_id", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "owner", - "type_info": "Int4" - }, - { - "ordinal": 2, - "name": "size", - "type_info": "Int4" - }, - { - "ordinal": 3, - "name": "name", - "type_info": "Text" - }, - { - "ordinal": 4, - "name": "data_type", - "type_info": "Int4" - }, - { - "ordinal": 5, - "name": "meta_binary", - "type_info": "Bytea" - }, - { - "ordinal": 6, - "name": "permission", - "type_info": "Int4" - }, - { - "ordinal": 7, - "name": "permission_recipients", - "type_info": "Int4Array" - }, - { - "ordinal": 8, - "name": "delete_permission", - "type_info": "Int4" - }, - { - "ordinal": 9, - "name": "delete_permission_recipients", - "type_info": "Int4Array" - }, - { - "ordinal": 10, - "name": "period", - "type_info": "Int4" - }, - { - "ordinal": 11, - "name": "refer_data_id", - "type_info": "Int8" - }, - { - "ordinal": 12, - "name": "flag", - "type_info": "Int4" - }, - { - "ordinal": 13, - "name": "tags", - "type_info": "TextArray" - }, - { - "ordinal": 14, - "name": "creation_date", - "type_info": "Timestamp" - }, - { - "ordinal": 15, - "name": "update_date", - "type_info": "Timestamp" - }, - { - "ordinal": 16, - "name": "access_password", - "type_info": "Int8" - }, - { - "ordinal": 17, - "name": "under_review", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Int4", - "Int4" - ] - }, - "nullable": [ - false, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - false, - false - ] - }, - "hash": "18fb671d9d97353420446413b74077f9b06ffe11cab2693e4e365c265f8e2418" -} diff --git a/.sqlx/query-29e18f241dc860dca12c03cabb788d2d26f7644e85bfd65423e6e32c469e3695.json b/.sqlx/query-29e18f241dc860dca12c03cabb788d2d26f7644e85bfd65423e6e32c469e3695.json deleted file mode 100644 index 5411a38..0000000 --- a/.sqlx/query-29e18f241dc860dca12c03cabb788d2d26f7644e85bfd65423e6e32c469e3695.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO datastore.object_custom_rankings (data_id, application_id, value)\n VALUES ($1, $2, $3)\n ON CONFLICT (data_id, application_id)\n DO UPDATE SET value = datastore.object_custom_rankings.value + EXCLUDED.value\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int8", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "29e18f241dc860dca12c03cabb788d2d26f7644e85bfd65423e6e32c469e3695" -} diff --git a/.sqlx/query-2ff34379bbc32276c3b78ef1283b8158ea907d36588e1e59f6cbe752d89361bb.json b/.sqlx/query-2ff34379bbc32276c3b78ef1283b8158ea907d36588e1e59f6cbe752d89361bb.json deleted file mode 100644 index 83246ca..0000000 --- a/.sqlx/query-2ff34379bbc32276c3b78ef1283b8158ea907d36588e1e59f6cbe752d89361bb.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM datastore.objects WHERE data_id = $1)", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - null - ] - }, - "hash": "2ff34379bbc32276c3b78ef1283b8158ea907d36588e1e59f6cbe752d89361bb" -} diff --git a/.sqlx/query-48d94383f1a261f19060f5c75d9f7803ee88233c53efaa2df9966a6c72c8c763.json b/.sqlx/query-48d94383f1a261f19060f5c75d9f7803ee88233c53efaa2df9966a6c72c8c763.json deleted file mode 100644 index 7a1cc6f..0000000 --- a/.sqlx/query-48d94383f1a261f19060f5c75d9f7803ee88233c53efaa2df9966a6c72c8c763.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT buffer\n FROM datastore.buffer_queues\n WHERE data_id = $1 AND slot = $2\n ORDER BY creation_date ASC\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "buffer", - "type_info": "Bytea" - } - ], - "parameters": { - "Left": [ - "Int8", - "Int4" - ] - }, - "nullable": [ - false - ] - }, - "hash": "48d94383f1a261f19060f5c75d9f7803ee88233c53efaa2df9966a6c72c8c763" -} diff --git a/.sqlx/query-60d8ebcf914d887439096df400fa4beaf37e879c28c81c22abeddb3d030535df.json b/.sqlx/query-60d8ebcf914d887439096df400fa4beaf37e879c28c81c22abeddb3d030535df.json deleted file mode 100644 index 4f07b49..0000000 --- a/.sqlx/query-60d8ebcf914d887439096df400fa4beaf37e879c28c81c22abeddb3d030535df.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT under_review, access_password\n FROM datastore.objects\n WHERE data_id = $1 AND upload_completed = TRUE AND deleted = FALSE\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "under_review", - "type_info": "Bool" - }, - { - "ordinal": 1, - "name": "access_password", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - false, - false - ] - }, - "hash": "60d8ebcf914d887439096df400fa4beaf37e879c28c81c22abeddb3d030535df" -} diff --git a/.sqlx/query-8909f81fbf26a9b559d26e138568be8946430d93821dbccb6dc9324ef0544c17.json b/.sqlx/query-8909f81fbf26a9b559d26e138568be8946430d93821dbccb6dc9324ef0544c17.json deleted file mode 100644 index 867701f..0000000 --- a/.sqlx/query-8909f81fbf26a9b559d26e138568be8946430d93821dbccb6dc9324ef0544c17.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n rankings.data_id,\n rankings.value\n FROM datastore.object_custom_rankings rankings\n JOIN UNNEST($1::bigint[]) WITH ORDINALITY AS rows(data_id, ord)\n ON rankings.data_id = rows.data_id\n AND rankings.application_id = $2\n ORDER BY rows.ord\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "data_id", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "value", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8Array", - "Int8" - ] - }, - "nullable": [ - false, - true - ] - }, - "hash": "8909f81fbf26a9b559d26e138568be8946430d93821dbccb6dc9324ef0544c17" -} diff --git a/.sqlx/query-a5e86d7e66d0e16e5f35df883e11d9f4ef96adbdab7ca7d008acf596af91fc2b.json b/.sqlx/query-a5e86d7e66d0e16e5f35df883e11d9f4ef96adbdab7ca7d008acf596af91fc2b.json deleted file mode 100644 index 539fdeb..0000000 --- a/.sqlx/query-a5e86d7e66d0e16e5f35df883e11d9f4ef96adbdab7ca7d008acf596af91fc2b.json +++ /dev/null @@ -1,112 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT data_id, owner, size, name, data_type, meta_binary,\n permission, permission_recipients, delete_permission, delete_permission_recipients,\n period, refer_data_id, flag, tags, creation_date, update_date\n FROM datastore.objects WHERE data_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "data_id", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "owner", - "type_info": "Int4" - }, - { - "ordinal": 2, - "name": "size", - "type_info": "Int4" - }, - { - "ordinal": 3, - "name": "name", - "type_info": "Text" - }, - { - "ordinal": 4, - "name": "data_type", - "type_info": "Int4" - }, - { - "ordinal": 5, - "name": "meta_binary", - "type_info": "Bytea" - }, - { - "ordinal": 6, - "name": "permission", - "type_info": "Int4" - }, - { - "ordinal": 7, - "name": "permission_recipients", - "type_info": "Int4Array" - }, - { - "ordinal": 8, - "name": "delete_permission", - "type_info": "Int4" - }, - { - "ordinal": 9, - "name": "delete_permission_recipients", - "type_info": "Int4Array" - }, - { - "ordinal": 10, - "name": "period", - "type_info": "Int4" - }, - { - "ordinal": 11, - "name": "refer_data_id", - "type_info": "Int8" - }, - { - "ordinal": 12, - "name": "flag", - "type_info": "Int4" - }, - { - "ordinal": 13, - "name": "tags", - "type_info": "TextArray" - }, - { - "ordinal": 14, - "name": "creation_date", - "type_info": "Timestamp" - }, - { - "ordinal": 15, - "name": "update_date", - "type_info": "Timestamp" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - false, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true - ] - }, - "hash": "a5e86d7e66d0e16e5f35df883e11d9f4ef96adbdab7ca7d008acf596af91fc2b" -} diff --git a/.sqlx/query-c2a0ca72fdcb060a7fb1c670b25e1efdbf3cef31e4b6cea4adbd51ee0218f5aa.json b/.sqlx/query-c2a0ca72fdcb060a7fb1c670b25e1efdbf3cef31e4b6cea4adbd51ee0218f5aa.json deleted file mode 100644 index 095ca6f..0000000 --- a/.sqlx/query-c2a0ca72fdcb060a7fb1c670b25e1efdbf3cef31e4b6cea4adbd51ee0218f5aa.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT slot, total_value, count, initial_value\n FROM datastore.object_ratings\n WHERE data_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "slot", - "type_info": "Int2" - }, - { - "ordinal": 1, - "name": "total_value", - "type_info": "Int8" - }, - { - "ordinal": 2, - "name": "count", - "type_info": "Int4" - }, - { - "ordinal": 3, - "name": "initial_value", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - false, - true, - false, - true - ] - }, - "hash": "c2a0ca72fdcb060a7fb1c670b25e1efdbf3cef31e4b6cea4adbd51ee0218f5aa" -} diff --git a/.sqlx/query-e28d8776cc49b55fe76cf33ac12fe18e500d243f1b55fd18e7d96d281605bcf9.json b/.sqlx/query-e28d8776cc49b55fe76cf33ac12fe18e500d243f1b55fd18e7d96d281605bcf9.json deleted file mode 100644 index 552f92a..0000000 --- a/.sqlx/query-e28d8776cc49b55fe76cf33ac12fe18e500d243f1b55fd18e7d96d281605bcf9.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE datastore.objects SET upload_completed = true WHERE data_id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "e28d8776cc49b55fe76cf33ac12fe18e500d243f1b55fd18e7d96d281605bcf9" -} diff --git a/Cargo.lock b/Cargo.lock index 8fd84b8..bf83871 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2225,6 +2225,7 @@ dependencies = [ "tokio", "typenum", "ureq", + "urlencoding", "v-byte-helpers", ] diff --git a/rnex-core/Cargo.toml b/rnex-core/Cargo.toml index 646b65e..620357b 100644 --- a/rnex-core/Cargo.toml +++ b/rnex-core/Cargo.toml @@ -33,6 +33,7 @@ aws-sdk-s3 = { version = "1.129.0", optional = true } aws-config = { version = "1.8.15", optional = true } base64 = "0.22.1" sha2 = "0.10.9" +urlencoding = "2.1.3" [dev-dependencies] # criterion = "0.7.0" diff --git a/rnex-core/src/nex/datastore.rs b/rnex-core/src/nex/datastore.rs index 19db2b4..498a75d 100644 --- a/rnex-core/src/nex/datastore.rs +++ b/rnex-core/src/nex/datastore.rs @@ -10,7 +10,7 @@ use rnex_core::nex::remote_console::RemoteConsole; use rnex_core::nex::s3presigner::S3Presigner; use rnex_core::rmc::response::ErrorCode; use rnex_core::rmc::protocols::secure::{Secure, RawSecure, RawSecureInfo, RemoteSecure}; -use rnex_core::rmc::protocols::datastore::{BufferQueueParam, CompletePostParam, DataStoreCustomRankingResult, DataStoreGetCustomRankingByDataIDParam, GetMetaInfo, GetMetaParam, KeyValue, Permission, PersistenceTarget, RateCustomRankingParam, RatingInfo, RatingInfoWithSlot}; +use rnex_core::rmc::protocols::datastore::{BufferQueueParam, CompletePostParam, DataStoreCustomRankingResult, DataStoreGetCustomRankingByDataIDParam, DataStorePrepareGetParam, DataStoreReqGetInfo, DataStoreSearchParam, GetMetaInfo, GetMetaParam, KeyValue, Permission, PersistenceTarget, RateCustomRankingParam, RatingInfo, RatingInfoWithSlot}; use rnex_core::rmc::protocols::datastore::{DataStore, RawDataStore, RawDataStoreInfo, RemoteDataStore, PreparePostParam, ReqPostInfo}; use crate::nex::user::User; use rnex_core::executables::common::{RNEX_DATASTORE_S3_BUCKET, RNEX_DATASTORE_S3_ENDPOINT, get_db}; @@ -336,6 +336,7 @@ async fn get_custom_rankings_by_data_ids( if let Ok(meta) = get_object_info_by_data_id(data_id, 0).await { results.push(DataStoreCustomRankingResult { + order: 0, score, meta_info: meta, }); @@ -347,6 +348,70 @@ async fn get_custom_rankings_by_data_ids( results } +async fn get_user_course_object_ids(owner_pid: u32) -> Result, ErrorCode> { + let rows = sqlx::query!( + r#" + SELECT data_id + FROM datastore.objects + WHERE owner = $1 AND data_type > 2 AND data_type < 50 + "#, + owner_pid as i64 + ) + .fetch_all(get_db()) + .await + .map_err(|e| { + log::error!("error fetching course IDs for PID {}: {:?}", owner_pid, e); + ErrorCode::DataStore_SystemFileError + })?; + + let mut valid_ids = Vec::new(); + for row in rows { + let data_id = row.data_id as u64; + // always check avail + if check_object_availability(data_id, 0).await.is_ok() { + valid_ids.push(data_id); + } + } + + Ok(valid_ids) +} + +fn get_blacklist_1() -> Vec { + vec![ + "けされ", "消され", "削除され", "リセットされ", "BANされ", "ＢＡＮされ", + "キミのコース", "君のコース", "きみのコース", "い い ね", "遊びます", "地震", + "震災", "被災", "津波", "バンされ", "い~ね", "震度", "じしん", "banされ", + "くわしくは", "詳しくは", "ちんちん", "ち0こ", "bicth", "い.い．ね", + "ナイ～ス", "い&い", "い-いね", "いぃね", "nigger", "ngger", "star if u", + "Star if u", "Star if you", "star if you", "PENlS", "マンコ", "butthole", + "LILI", "vagina", "vagyna", "うんち", "うんこ", "ウンコ", "Ｉｉｎｅ", + "EENE", "まんこ", "ウンチ", "niglet", "nigglet", "please like", "きんたま", + "Butthole", "llね", "iいね", "give a star", "ちんぽ", "亀頭", "penis", + "ｳﾝｺ", "plz more stars", "star plz", "い()ね", "PLEASE star", "Bitte Sterne", + ].into_iter().map(String::from).collect() +} + +fn get_blacklist_2() -> Vec { + vec![ + "ゼロから", "０から", "0から", "い　　い　　ね", "いい", "東日本", "大震", + ].into_iter().map(String::from).collect() +} + +fn get_blacklist_3() -> Vec { + vec![ + "いいね", "下さい", "ください", "押して", "おして", "返す", "かえす", + "星", "してくれ", "するよ", "☆くれたら", "☆あげます", "★くれたら", + "★あげます", "しね", "ころす", "ころされた", "アナル", "ファック", + "キンタマ", "○ね", "キチガイ", "うんこ", "KITIGAI", "金玉", "おっぱい", + "☆おす", "☆押す", "★おす", "★押す", "いいする", "いいよ", "イイネ", + "ケツ", "うんち", "かくせいざい", "覚せい剤", "シャブ", "きんたま", + "ちんちん", "おしっこ", "ちんぽこ", "ころして", "グッド", "グット", + "レ●プ", "バーカ", "きちがい", "ちんげ", "マンコ", "まんこ", "チンポ", + "クズ", "ウンコ", "ナイスおねがいします", "penis", "イイね", "☆よろ", + "ナイス!して", "ま/んこ", "まん/こ", + ].into_iter().map(String::from).collect() +} + impl DataStore for User { async fn get_meta(&self, mut metaparam: GetMetaParam) -> Result { let mut meta_info = if metaparam.dataid != 0 { @@ -429,6 +494,9 @@ impl DataStore for User { } async fn complete_post_object(&self, completeparam: CompletePostParam) -> Result<(), ErrorCode> { + log::info!("Data ID: {:?}", completeparam.dataid); + log::info!("Success: {:?}", completeparam.success); + let record = sqlx::query!( r#"SELECT owner, under_review FROM datastore.objects WHERE data_id = $1"#, completeparam.dataid as i64 @@ -469,10 +537,10 @@ impl DataStore for User { } async fn rate_custom_ranking(&self, rankingparam: Vec) -> Result<(), ErrorCode> { - for param in rankingparam { + for abcparam in rankingparam { let exists = sqlx::query_scalar!( r#"SELECT EXISTS(SELECT 1 FROM datastore.objects WHERE data_id = $1)"#, - param.dataid as i64 + abcparam.dataid as i64 ) .fetch_one(get_db()) .await @@ -489,9 +557,9 @@ impl DataStore for User { ON CONFLICT (data_id, application_id) DO UPDATE SET value = datastore.object_custom_rankings.value + EXCLUDED.value "#, - param.dataid as i64, - param.appid as i32, - param.score as i32 + abcparam.dataid as i64, + abcparam.appid as i32, + abcparam.score as i32 ) .execute(get_db()) .await @@ -542,27 +610,30 @@ impl DataStore for User { async fn get_custom_ranking_by_data_id( &self, - param: DataStoreGetCustomRankingByDataIDParam + custom_ranking_param: DataStoreGetCustomRankingByDataIDParam ) -> Result<(Vec, Vec), ErrorCode> { + println!("appid: {:?}", custom_ranking_param.application_id); + println!("dataid list: {:?}", custom_ranking_param.data_id_list); + println!("result option: {:?}", custom_ranking_param.result_option); - let mut ranking_results = get_custom_rankings_by_data_ids(param.application_id, param.data_id_list).await; + let mut ranking_results = get_custom_rankings_by_data_ids(custom_ranking_param.application_id, custom_ranking_param.data_id_list).await; let mut q_results = Vec::with_capacity(ranking_results.len()); for result in &mut ranking_results { - if (param.result_option & 0x01) == 0 { + if (custom_ranking_param.result_option & 0x01) == 0 { result.meta_info.tags = Vec::new(); } - if (param.result_option & 0x02) == 0 { + if (custom_ranking_param.result_option & 0x02) == 0 { result.meta_info.ratings = Vec::new(); } - if (param.result_option & 0x04) == 0 { + if (custom_ranking_param.result_option & 0x04) == 0 { result.meta_info.meta_binary = QBuffer(Vec::new()); } - if (param.result_option & 0x20) == 0 { + if (custom_ranking_param.result_option & 0x20) == 0 { result.score = 0; } @@ -572,11 +643,138 @@ impl DataStore for User { Ok((ranking_results, q_results)) } - async fn get_buffer_queue(&self, param: BufferQueueParam) -> Result, ErrorCode> { + async fn get_buffer_queue(&self, bufferparam: BufferQueueParam) -> Result, ErrorCode> { // log::info!("GetBufferQueue: dataid={}, slot={}", param.dataid, param.slot); - let buffers = get_buffer_queues_by_data_id_and_slot(param.dataid, param.slot).await?; + let buffers = get_buffer_queues_by_data_id_and_slot(bufferparam.dataid, bufferparam.slot).await?; Ok(buffers) } + + async fn prepare_get_object(&self, prepare_get_param: DataStorePrepareGetParam) -> Result { + let meta_info = if prepare_get_param.dataid != 0 { + get_object_info_by_data_id(prepare_get_param.dataid, prepare_get_param.access_password).await? + } else { + get_object_info_by_persistence_target(prepare_get_param.persistence_target, prepare_get_param.access_password).await? + }; + + verify_object_permission(meta_info.owner, self.pid, &meta_info.permission)?; + + let presigner = S3Presigner::new( + &format!("https://{}", *RNEX_DATASTORE_S3_ENDPOINT), + format!("{}", *RNEX_DATASTORE_S3_BUCKET) + ).await; + + let key = format!("data/{}.bin", meta_info.dataid); + let download_url = presigner.generate_presigned_get(&key); + + Ok(DataStoreReqGetInfo { + url: download_url, + request_headers: vec![], + size: meta_info.size, + root_ca_cert: vec![], + dataid: meta_info.dataid, + }) + } + + async fn followings_latest_course_search_object( + &self, + course_search_param: DataStoreSearchParam, + _extra_data: Vec + ) -> Result, ErrorCode> { + let mut all_results = Vec::new(); + + for &owner_pid in &course_search_param.owner_ids { + let course_ids = get_user_course_object_ids(owner_pid).await?; + + if course_ids.is_empty() { + continue; + } + + let mut results = get_custom_rankings_by_data_ids(0, course_ids).await; + + // Flag 0x1: Return Tags + // Flag 0x2: Return Ratings + // Flag 0x4: Return MetaBinary + // Flag 0x20: Return Score + for res in &mut results { + if course_search_param.result_option & 0x1 == 0 { + res.meta_info.tags = Vec::new(); + } + if course_search_param.result_option & 0x2 == 0 { + res.meta_info.ratings = Vec::new(); + } + if course_search_param.result_option & 0x4 == 0 { + res.meta_info.meta_binary = rnex_core::rmc::structures::qbuffer::QBuffer(Vec::new()); + } + if course_search_param.result_option & 0x20 == 0 { + res.score = 0; + } + } + + all_results.extend(results); + } + + // note: we assume the client sorts the data lol + + Ok(all_results) + } + + async fn get_application_config_string(&self, application_id: u32) -> Result, ErrorCode> { + let config = match application_id { + 128 => get_blacklist_1(), + 129 => get_blacklist_2(), + 130 => get_blacklist_3(), + _ => { + log::warn!("unsupported application_id in GetApplicationConfigString: {}", application_id); + Vec::new() + } + }; + + Ok(config) + } + + async fn get_metas_multiple_param( + &self, + params: Vec + ) -> Result<(Vec, Vec), ErrorCode> { + let mut metas = Vec::with_capacity(params.len()); + let mut results = Vec::with_capacity(params.len()); + + for param in params { + let info_result = if param.dataid != 0 { + get_object_info_by_data_id(param.dataid, param.access_password).await + } else { + get_object_info_by_persistence_target(param.persistence_target, param.access_password).await + }; + + match info_result { + Ok(mut meta) => { + if let Err(e) = verify_object_permission(meta.owner, self.pid, &meta.permission) { + metas.push(GetMetaInfo::default()); + results.push(QResult::error(e)); + } else { + if param.result_option & 0x1 == 0 { + meta.tags = Vec::new(); + } + if param.result_option & 0x2 == 0 { + meta.ratings = Vec::new(); + } + if param.result_option & 0x4 == 0 { + meta.meta_binary = rnex_core::rmc::structures::qbuffer::QBuffer(Vec::new()); + } + + metas.push(meta); + results.push(QResult::success(ErrorCode::Core_Unknown)); + } + } + Err(e) => { + metas.push(GetMetaInfo::default()); + results.push(QResult::error(e)); + } + } + } + + Ok((metas, results)) + } } \ No newline at end of file diff --git a/rnex-core/src/nex/s3presigner.rs b/rnex-core/src/nex/s3presigner.rs index 169293b..afe61d4 100644 --- a/rnex-core/src/nex/s3presigner.rs +++ b/rnex-core/src/nex/s3presigner.rs @@ -56,6 +56,50 @@ impl S3Presigner { (url, fields) } + pub fn generate_presigned_get(&self, key: &str) -> String { + let access_key = std::env::var("AWS_ACCESS_KEY_ID").expect("Missing Access Key"); + let secret_key = std::env::var("AWS_SECRET_ACCESS_KEY").expect("Missing Secret Key"); + let region = "us-east-1"; + let date_short = Utc::now().format("%Y%m%d").to_string(); + let date_full = Utc::now().format("%Y%m%dT%H%M%SZ").to_string(); + + let credential_scope = format!("{}/{}/s3/aws4_request", date_short, region); + + let query_string = format!( + "X-Amz-Algorithm=AWS4-HMAC-SHA256&\ + X-Amz-Credential={}%2F{}&\ + X-Amz-Date={}&\ + X-Amz-Expires=900&\ + X-Amz-SignedHeaders=host", + access_key, + urlencoding::encode(&credential_scope), + date_full + ); + + let canonical_request = format!( + "GET\n/{}/{}\n{}\nhost:{}\n\nhost\nUNSIGNED-PAYLOAD", + self.bucket, key, query_string, *RNEX_DATASTORE_S3_ENDPOINT + ); + + let hashed_request = hex::encode(Sha256::digest(canonical_request.as_bytes())); + + let string_to_sign = format!( + "AWS4-HMAC-SHA256\n{}\n{}\n{}", + date_full, credential_scope, hashed_request + ); + + let k_date = self.hmac_sha256(format!("AWS4{}", secret_key).as_bytes(), &date_short); + let k_region = self.hmac_sha256(&k_date, region); + let k_service = self.hmac_sha256(&k_region, "s3"); + let k_signing = self.hmac_sha256(&k_service, "aws4_request"); + let signature = hex::encode(self.hmac_sha256(&k_signing, &string_to_sign)); + + format!( + "https://{}/{}/{}?{}&X-Amz-Signature={}", + *RNEX_DATASTORE_S3_ENDPOINT, self.bucket, key, query_string, signature + ) + } + fn calculate_signature(&self, secret: &str, date: &str, region: &str, policy: &str) -> String { let k_date = self.hmac_sha256(format!("AWS4{}", secret).as_bytes(), date); let k_region = self.hmac_sha256(&k_date, region); diff --git a/rnex-core/src/rmc/protocols/datastore.rs b/rnex-core/src/rmc/protocols/datastore.rs index 00e522f..0c68fc5 100644 --- a/rnex-core/src/rmc/protocols/datastore.rs +++ b/rnex-core/src/rmc/protocols/datastore.rs @@ -4,15 +4,16 @@ use rnex_core::rmc::response::ErrorCode; use rnex_core::rmc::structures::qresult::QResult; use rnex_core::kerberos::KerberosDateTime; use rnex_core::PID; +use rnex_core::rmc::structures::resultsrange::ResultsRange; -#[derive(RmcSerialize, Clone, Debug)] +#[derive(RmcSerialize, Clone, Debug, Default)] #[rmc_struct(0)] pub struct PersistenceTarget { pub owner: PID, pub persistence_slot_id: u16, } -#[derive(RmcSerialize, Clone, Debug)] +#[derive(RmcSerialize, Clone, Debug, Default)] #[rmc_struct(0)] pub struct Permission { pub permission: u8, @@ -34,7 +35,7 @@ pub struct RatingInfo { pub initial_value: i64, } -#[derive(RmcSerialize, Clone)] +#[derive(RmcSerialize, Clone, Default)] #[rmc_struct(0)] pub struct GetMetaParam { pub dataid: u64, @@ -43,7 +44,7 @@ pub struct GetMetaParam { pub access_password: u64, } -#[derive(RmcSerialize, Clone)] +#[derive(RmcSerialize, Clone, Default)] #[rmc_struct(0)] pub struct GetMetaInfo { pub dataid: u64, @@ -165,14 +166,59 @@ pub struct DataStoreGetCustomRankingByDataIDParam { #[derive(RmcSerialize, Clone)] #[rmc_struct(0)] pub struct DataStoreCustomRankingResult { + pub order: u32, pub score: u32, pub meta_info: GetMetaInfo, } +#[derive(RmcSerialize, Clone)] +#[rmc_struct(0)] +pub struct DataStorePrepareGetParam { + pub dataid: u64, + pub lockid: u32, + pub persistence_target: PersistenceTarget, + pub access_password: u64, + pub extra_data: Vec, +} + +#[derive(RmcSerialize, Clone)] +#[rmc_struct(0)] +pub struct DataStoreReqGetInfo { + pub url: String, + pub request_headers: Vec, + pub size: u32, + pub root_ca_cert: Vec, + pub dataid: u64, +} + +#[derive(RmcSerialize, Clone)] +#[rmc_struct(1)] +pub struct DataStoreSearchParam { + pub search_target: u8, + pub owner_ids: Vec, + pub owner_type: u8, + pub destination_ids: Vec, + pub data_type: u16, + pub created_after: KerberosDateTime, + pub created_before: KerberosDateTime, + pub updated_after: KerberosDateTime, + pub updated_before: KerberosDateTime, + pub refer_dat_id: u32, + pub tags: Vec, + pub result_order_column: u8, + pub result_order: u8, + pub result_range: ResultsRange, + pub result_option: u8, + pub minimal_rating_frequency: u32, + pub use_cache: bool, +} + #[rmc_proto(115)] pub trait DataStore{ #[method_id(8)] async fn get_meta(&self, metaparam: GetMetaParam) -> Result; + #[method_id(36)] + async fn get_metas_multiple_param(&self, params: Vec) -> Result<(Vec, Vec), ErrorCode>; #[method_id(24)] async fn prepare_post_object(&self, postparam: PreparePostParam) -> Result; #[method_id(26)] @@ -182,7 +228,13 @@ pub trait DataStore{ #[method_id(61)] async fn get_application_config(&self, appid: u32) -> Result, ErrorCode>; #[method_id(50)] - async fn get_custom_ranking_by_data_id(&self, param: DataStoreGetCustomRankingByDataIDParam) -> Result<(Vec, Vec), ErrorCode>; + async fn get_custom_ranking_by_data_id(&self, custom_ranking_param: DataStoreGetCustomRankingByDataIDParam) -> Result<(Vec, Vec), ErrorCode>; #[method_id(54)] - async fn get_buffer_queue(&self, param: BufferQueueParam) -> Result, ErrorCode>; + async fn get_buffer_queue(&self, bufferparam: BufferQueueParam) -> Result, ErrorCode>; + #[method_id(25)] + async fn prepare_get_object(&self, prepare_get_param: DataStorePrepareGetParam) -> Result; + #[method_id(65)] + async fn followings_latest_course_search_object(&self, course_search_param: DataStoreSearchParam, extra_data: Vec) -> Result, ErrorCode>; + #[method_id(74)] + async fn get_application_config_string(&self, application_id: u32) -> Result, ErrorCode>; } \ No newline at end of file diff --git a/rnex-core/src/rmc/protocols/ranking.rs b/rnex-core/src/rmc/protocols/ranking.rs index b9dba06..53ec9af 100644 --- a/rnex-core/src/rmc/protocols/ranking.rs +++ b/rnex-core/src/rmc/protocols/ranking.rs @@ -2,17 +2,10 @@ use macros::{rmc_struct, rmc_proto, RmcSerialize, method_id}; use rnex_core::kerberos::KerberosDateTime; use rnex_core::rmc::structures::qbuffer::QBuffer; - +use rnex_core::rmc::structures::resultsrange::ResultsRange; use rnex_core::rmc::response::ErrorCode; use rnex_core::rmc::structures::ranking::UploadCompetitionData; -#[derive(RmcSerialize, Debug, Default, Clone)] -#[rmc_struct(0)] -pub struct ResultsRange{ - pub offset: u32, - pub size: u32 -} - #[derive(RmcSerialize, Debug, Default, Clone)] #[rmc_struct(1)] pub struct CompetitionRankingGetParam { diff --git a/rnex-core/src/rmc/structures/mod.rs b/rnex-core/src/rmc/structures/mod.rs index 996cc4b..459aeac 100644 --- a/rnex-core/src/rmc/structures/mod.rs +++ b/rnex-core/src/rmc/structures/mod.rs @@ -37,6 +37,7 @@ pub mod ranking; pub mod rmc_struct; pub mod string; pub mod variant; +pub mod resultsrange; pub trait RmcSerialize { fn serialize(&self, writer: &mut impl Write) -> Result<()>; diff --git a/rnex-core/src/rmc/structures/qbuffer.rs b/rnex-core/src/rmc/structures/qbuffer.rs index 2f7b23a..7f2aa7b 100644 --- a/rnex-core/src/rmc/structures/qbuffer.rs +++ b/rnex-core/src/rmc/structures/qbuffer.rs @@ -3,7 +3,7 @@ use bytemuck::bytes_of; use v_byte_helpers::{IS_BIG_ENDIAN, ReadExtensions}; use crate::rmc::structures::{Result, RmcSerialize}; -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct QBuffer(pub Vec); impl RmcSerialize for QBuffer{ diff --git a/rnex-core/src/rmc/structures/resultsrange.rs b/rnex-core/src/rmc/structures/resultsrange.rs new file mode 100644 index 0000000..ba62e49 --- /dev/null +++ b/rnex-core/src/rmc/structures/resultsrange.rs @@ -0,0 +1,8 @@ +use macros::RmcSerialize; + +#[derive(RmcSerialize, Debug, Default, Clone)] +#[rmc_struct(0)] +pub struct ResultsRange{ + pub offset: u32, + pub size: u32 +} \ No newline at end of file