spacebar/rust-nex
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages 50 Wiki Activity Actions

Even more work on datastore

Browse source
This commit is contained in:
red binder 2026-04-20 14:06:04 +02:00
commit 08fac7af03
20 changed files with 327 additions and 504 deletions
Download patch file Download diff file Expand all files Collapse all files

38
.sqlx/query-039d134f0d1b681b55b1394a710b44a630eb066846c777c462b8c9341e26b696.json generated
View file

@ -1,38 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO datastore.objects (\n owner, size, name, data_type, meta_binary,\n permission, permission_recipients,\n delete_permission, delete_permission_recipients,\n flag, period, refer_data_id, tags,\n persistence_slot_id, extra_data, creation_date, update_date\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17\n ) RETURNING data_id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "data_id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int4",
"Int4",
"Text",
"Int4",
"Bytea",
"Int4",
"Int4Array",
"Int4",
"Int4Array",
"Int4",
"Int4",
"Int8",
"TextArray",
"Int4",
"TextArray",
"Timestamp",
"Timestamp"
]
},
"nullable": [
false
]
},
"hash": "039d134f0d1b681b55b1394a710b44a630eb066846c777c462b8c9341e26b696"
}

28
.sqlx/query-164708b549c483a041d2e54065ed3ffbd9f8d5304f6aa6d785dbddbb1626c0e9.json generated
View file

@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT owner, under_review FROM datastore.objects WHERE data_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "owner",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "under_review",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
true,
false
]
},
"hash": "164708b549c483a041d2e54065ed3ffbd9f8d5304f6aa6d785dbddbb1626c0e9"
}

125
.sqlx/query-18fb671d9d97353420446413b74077f9b06ffe11cab2693e4e365c265f8e2418.json generated
View file

@ -1,125 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT data_id, owner, size, name, data_type, meta_binary,\n permission, permission_recipients, delete_permission, delete_permission_recipients,\n period, refer_data_id, flag, tags, creation_date, update_date,\n access_password, under_review\n FROM datastore.objects\n WHERE owner = $1 AND persistence_slot_id = $2\n AND upload_completed = TRUE AND deleted = FALSE",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "data_id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "owner",
"type_info": "Int4"
},
{
"ordinal": 2,
"name": "size",
"type_info": "Int4"
},
{
"ordinal": 3,
"name": "name",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "data_type",
"type_info": "Int4"
},
{
"ordinal": 5,
"name": "meta_binary",
"type_info": "Bytea"
},
{
"ordinal": 6,
"name": "permission",
"type_info": "Int4"
},
{
"ordinal": 7,
"name": "permission_recipients",
"type_info": "Int4Array"
},
{
"ordinal": 8,
"name": "delete_permission",
"type_info": "Int4"
},
{
"ordinal": 9,
"name": "delete_permission_recipients",
"type_info": "Int4Array"
},
{
"ordinal": 10,
"name": "period",
"type_info": "Int4"
},
{
"ordinal": 11,
"name": "refer_data_id",
"type_info": "Int8"
},
{
"ordinal": 12,
"name": "flag",
"type_info": "Int4"
},
{
"ordinal": 13,
"name": "tags",
"type_info": "TextArray"
},
{
"ordinal": 14,
"name": "creation_date",
"type_info": "Timestamp"
},
{
"ordinal": 15,
"name": "update_date",
"type_info": "Timestamp"
},
{
"ordinal": 16,
"name": "access_password",
"type_info": "Int8"
},
{
"ordinal": 17,
"name": "under_review",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Int4",
"Int4"
]
},
"nullable": [
false,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
false,
false
]
},
"hash": "18fb671d9d97353420446413b74077f9b06ffe11cab2693e4e365c265f8e2418"
}

16
.sqlx/query-29e18f241dc860dca12c03cabb788d2d26f7644e85bfd65423e6e32c469e3695.json generated
View file

@ -1,16 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO datastore.object_custom_rankings (data_id, application_id, value)\n VALUES ($1, $2, $3)\n ON CONFLICT (data_id, application_id)\n DO UPDATE SET value = datastore.object_custom_rankings.value + EXCLUDED.value\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8",
"Int8",
"Int8"
]
},
"nullable": []
},
"hash": "29e18f241dc860dca12c03cabb788d2d26f7644e85bfd65423e6e32c469e3695"
}

22
.sqlx/query-2ff34379bbc32276c3b78ef1283b8158ea907d36588e1e59f6cbe752d89361bb.json generated
View file

@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM datastore.objects WHERE data_id = $1)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
null
]
},
"hash": "2ff34379bbc32276c3b78ef1283b8158ea907d36588e1e59f6cbe752d89361bb"
}

23
.sqlx/query-48d94383f1a261f19060f5c75d9f7803ee88233c53efaa2df9966a6c72c8c763.json generated
View file

@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT buffer\n FROM datastore.buffer_queues\n WHERE data_id = $1 AND slot = $2\n ORDER BY creation_date ASC\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "buffer",
"type_info": "Bytea"
}
],
"parameters": {
"Left": [
"Int8",
"Int4"
]
},
"nullable": [
false
]
},
"hash": "48d94383f1a261f19060f5c75d9f7803ee88233c53efaa2df9966a6c72c8c763"
}

28
.sqlx/query-60d8ebcf914d887439096df400fa4beaf37e879c28c81c22abeddb3d030535df.json generated
View file

@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT under_review, access_password\n FROM datastore.objects\n WHERE data_id = $1 AND upload_completed = TRUE AND deleted = FALSE\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "under_review",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "access_password",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false,
false
]
},
"hash": "60d8ebcf914d887439096df400fa4beaf37e879c28c81c22abeddb3d030535df"
}

29
.sqlx/query-8909f81fbf26a9b559d26e138568be8946430d93821dbccb6dc9324ef0544c17.json generated
View file

@ -1,29 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n rankings.data_id,\n rankings.value\n FROM datastore.object_custom_rankings rankings\n JOIN UNNEST($1::bigint[]) WITH ORDINALITY AS rows(data_id, ord)\n ON rankings.data_id = rows.data_id\n AND rankings.application_id = $2\n ORDER BY rows.ord\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "data_id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "value",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8Array",
"Int8"
]
},
"nullable": [
false,
true
]
},
"hash": "8909f81fbf26a9b559d26e138568be8946430d93821dbccb6dc9324ef0544c17"
}

112
.sqlx/query-a5e86d7e66d0e16e5f35df883e11d9f4ef96adbdab7ca7d008acf596af91fc2b.json generated
View file

@ -1,112 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT data_id, owner, size, name, data_type, meta_binary,\n permission, permission_recipients, delete_permission, delete_permission_recipients,\n period, refer_data_id, flag, tags, creation_date, update_date\n FROM datastore.objects WHERE data_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "data_id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "owner",
"type_info": "Int4"
},
{
"ordinal": 2,
"name": "size",
"type_info": "Int4"
},
{
"ordinal": 3,
"name": "name",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "data_type",
"type_info": "Int4"
},
{
"ordinal": 5,
"name": "meta_binary",
"type_info": "Bytea"
},
{
"ordinal": 6,
"name": "permission",
"type_info": "Int4"
},
{
"ordinal": 7,
"name": "permission_recipients",
"type_info": "Int4Array"
},
{
"ordinal": 8,
"name": "delete_permission",
"type_info": "Int4"
},
{
"ordinal": 9,
"name": "delete_permission_recipients",
"type_info": "Int4Array"
},
{
"ordinal": 10,
"name": "period",
"type_info": "Int4"
},
{
"ordinal": 11,
"name": "refer_data_id",
"type_info": "Int8"
},
{
"ordinal": 12,
"name": "flag",
"type_info": "Int4"
},
{
"ordinal": 13,
"name": "tags",
"type_info": "TextArray"
},
{
"ordinal": 14,
"name": "creation_date",
"type_info": "Timestamp"
},
{
"ordinal": 15,
"name": "update_date",
"type_info": "Timestamp"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true
]
},
"hash": "a5e86d7e66d0e16e5f35df883e11d9f4ef96adbdab7ca7d008acf596af91fc2b"
}

40
.sqlx/query-c2a0ca72fdcb060a7fb1c670b25e1efdbf3cef31e4b6cea4adbd51ee0218f5aa.json generated
View file

@ -1,40 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT slot, total_value, count, initial_value\n FROM datastore.object_ratings\n WHERE data_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "slot",
"type_info": "Int2"
},
{
"ordinal": 1,
"name": "total_value",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "count",
"type_info": "Int4"
},
{
"ordinal": 3,
"name": "initial_value",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false,
true,
false,
true
]
},
"hash": "c2a0ca72fdcb060a7fb1c670b25e1efdbf3cef31e4b6cea4adbd51ee0218f5aa"
}

14
.sqlx/query-e28d8776cc49b55fe76cf33ac12fe18e500d243f1b55fd18e7d96d281605bcf9.json generated
View file

@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE datastore.objects SET upload_completed = true WHERE data_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "e28d8776cc49b55fe76cf33ac12fe18e500d243f1b55fd18e7d96d281605bcf9"
}

1
Cargo.lock generated
View file

@ -2225,6 +2225,7 @@ dependencies = [
"tokio",
"typenum",
"ureq",
"urlencoding",
"v-byte-helpers",
]

1
rnex-core/Cargo.toml
View file

@ -33,6 +33,7 @@ aws-sdk-s3 = { version = "1.129.0", optional = true }
aws-config = { version = "1.8.15", optional = true }
base64 = "0.22.1"
sha2 = "0.10.9"
urlencoding = "2.1.3"
[dev-dependencies]
# criterion = "0.7.0"

226
rnex-core/src/nex/datastore.rs
View file

@ -10,7 +10,7 @@ use rnex_core::nex::remote_console::RemoteConsole;
use rnex_core::nex::s3presigner::S3Presigner;
use rnex_core::rmc::response::ErrorCode;
use rnex_core::rmc::protocols::secure::{Secure, RawSecure, RawSecureInfo, RemoteSecure};
use rnex_core::rmc::protocols::datastore::{BufferQueueParam, CompletePostParam, DataStoreCustomRankingResult, DataStoreGetCustomRankingByDataIDParam, GetMetaInfo, GetMetaParam, KeyValue, Permission, PersistenceTarget, RateCustomRankingParam, RatingInfo, RatingInfoWithSlot};
use rnex_core::rmc::protocols::datastore::{BufferQueueParam, CompletePostParam, DataStoreCustomRankingResult, DataStoreGetCustomRankingByDataIDParam, DataStorePrepareGetParam, DataStoreReqGetInfo, DataStoreSearchParam, GetMetaInfo, GetMetaParam, KeyValue, Permission, PersistenceTarget, RateCustomRankingParam, RatingInfo, RatingInfoWithSlot};
use rnex_core::rmc::protocols::datastore::{DataStore, RawDataStore, RawDataStoreInfo, RemoteDataStore, PreparePostParam, ReqPostInfo};
use crate::nex::user::User;
use rnex_core::executables::common::{RNEX_DATASTORE_S3_BUCKET, RNEX_DATASTORE_S3_ENDPOINT, get_db};
@ -336,6 +336,7 @@ async fn get_custom_rankings_by_data_ids(
if let Ok(meta) = get_object_info_by_data_id(data_id, 0).await {
results.push(DataStoreCustomRankingResult {
order: 0,
score,
meta_info: meta,
});
@ -347,6 +348,70 @@ async fn get_custom_rankings_by_data_ids(
results
}
async fn get_user_course_object_ids(owner_pid: u32) -> Result<Vec<u64>, ErrorCode> {
let rows = sqlx::query!(
r#"
SELECT data_id
FROM datastore.objects
WHERE owner = $1 AND data_type > 2 AND data_type < 50
"#,
owner_pid as i64
)
.fetch_all(get_db())
.await
.map_err(|e| {
log::error!("error fetching course IDs for PID {}: {:?}", owner_pid, e);
ErrorCode::DataStore_SystemFileError
})?;
let mut valid_ids = Vec::new();
for row in rows {
let data_id = row.data_id as u64;
// always check avail
if check_object_availability(data_id, 0).await.is_ok() {
valid_ids.push(data_id);
}
}
Ok(valid_ids)
}
fn get_blacklist_1() -> Vec<String> {
vec![
"けされ", "消され", "削除され", "リセットされ", "BANされ", "ＢＡＮされ",
"キミのコース", "君のコース", "きみのコース", "い い ね", "遊びます", "地震",
"震災", "被災", "津波", "バンされ", "い~ね", "震度", "じしん", "banされ",
"くわしくは", "詳しくは", "ちんちん", "ち0こ", "bicth", "い.い．ね",
"ナイ～ス", "い&い", "い-いね", "いぃね", "nigger", "ngger", "star if u",
"Star if u", "Star if you", "star if you", "PENlS", "マンコ", "butthole",
"LILI", "vagina", "vagyna", "うんち", "うんこ", "ウンコ", "Ｉｉｎｅ",
"EENE", "まんこ", "ウンチ", "niglet", "nigglet", "please like", "きんたま",
"Butthole", "llね", "iいね", "give a star", "ちんぽ", "亀頭", "penis",
"ｳﾝｺ", "plz more stars", "star plz", "い()ね", "PLEASE star", "Bitte Sterne",
].into_iter().map(String::from).collect()
}
fn get_blacklist_2() -> Vec<String> {
vec![
"ゼロから", "０から", "0から", "い　　い　　ね", "いい", "東日本", "大震",
].into_iter().map(String::from).collect()
}
fn get_blacklist_3() -> Vec<String> {
vec![
"いいね", "下さい", "ください", "押して", "おして", "返す", "かえす",
"星", "してくれ", "するよ", "☆くれたら", "☆あげます", "★くれたら",
"★あげます", "しね", "ころす", "ころされた", "アナル", "ファック",
"キンタマ", "○ね", "キチガイ", "うんこ", "KITIGAI", "金玉", "おっぱい",
"☆おす", "☆押す", "★おす", "★押す", "いいする", "いいよ", "イイネ",
"ケツ", "うんち", "かくせいざい", "覚せい剤", "シャブ", "きんたま",
"ちんちん", "おしっこ", "ちんぽこ", "ころして", "グッド", "グット",
"レ●プ", "バーカ", "きちがい", "ちんげ", "マンコ", "まんこ", "チンポ",
"クズ", "ウンコ", "ナイスおねがいします", "penis", "イイね", "☆よろ",
"ナイス!して", "ま/んこ", "まん/こ",
].into_iter().map(String::from).collect()
}
impl DataStore for User {
async fn get_meta(&self, mut metaparam: GetMetaParam) -> Result<GetMetaInfo, ErrorCode> {
let mut meta_info = if metaparam.dataid != 0 {
@ -429,6 +494,9 @@ impl DataStore for User {
}
async fn complete_post_object(&self, completeparam: CompletePostParam) -> Result<(), ErrorCode> {
log::info!("Data ID: {:?}", completeparam.dataid);
log::info!("Success: {:?}", completeparam.success);
let record = sqlx::query!(
r#"SELECT owner, under_review FROM datastore.objects WHERE data_id = $1"#,
completeparam.dataid as i64
@ -469,10 +537,10 @@ impl DataStore for User {
}
async fn rate_custom_ranking(&self, rankingparam: Vec<RateCustomRankingParam>) -> Result<(), ErrorCode> {
for param in rankingparam {
for abcparam in rankingparam {
let exists = sqlx::query_scalar!(
r#"SELECT EXISTS(SELECT 1 FROM datastore.objects WHERE data_id = $1)"#,
param.dataid as i64
abcparam.dataid as i64
)
.fetch_one(get_db())
.await
@ -489,9 +557,9 @@ impl DataStore for User {
ON CONFLICT (data_id, application_id)
DO UPDATE SET value = datastore.object_custom_rankings.value + EXCLUDED.value
"#,
param.dataid as i64,
param.appid as i32,
param.score as i32
abcparam.dataid as i64,
abcparam.appid as i32,
abcparam.score as i32
)
.execute(get_db())
.await
@ -542,27 +610,30 @@ impl DataStore for User {
async fn get_custom_ranking_by_data_id(
&self,
param: DataStoreGetCustomRankingByDataIDParam
custom_ranking_param: DataStoreGetCustomRankingByDataIDParam
) -> Result<(Vec<DataStoreCustomRankingResult>, Vec<QResult>), ErrorCode> {
println!("appid: {:?}", custom_ranking_param.application_id);
println!("dataid list: {:?}", custom_ranking_param.data_id_list);
println!("result option: {:?}", custom_ranking_param.result_option);
let mut ranking_results = get_custom_rankings_by_data_ids(param.application_id, param.data_id_list).await;
let mut ranking_results = get_custom_rankings_by_data_ids(custom_ranking_param.application_id, custom_ranking_param.data_id_list).await;
let mut q_results = Vec::with_capacity(ranking_results.len());
for result in &mut ranking_results {
if (param.result_option & 0x01) == 0 {
if (custom_ranking_param.result_option & 0x01) == 0 {
result.meta_info.tags = Vec::new();
}
if (param.result_option & 0x02) == 0 {
if (custom_ranking_param.result_option & 0x02) == 0 {
result.meta_info.ratings = Vec::new();
}
if (param.result_option & 0x04) == 0 {
if (custom_ranking_param.result_option & 0x04) == 0 {
result.meta_info.meta_binary = QBuffer(Vec::new());
}
if (param.result_option & 0x20) == 0 {
if (custom_ranking_param.result_option & 0x20) == 0 {
result.score = 0;
}
@ -572,11 +643,138 @@ impl DataStore for User {
Ok((ranking_results, q_results))
}
async fn get_buffer_queue(&self, param: BufferQueueParam) -> Result<Vec<QBuffer>, ErrorCode> {
async fn get_buffer_queue(&self, bufferparam: BufferQueueParam) -> Result<Vec<QBuffer>, ErrorCode> {
// log::info!("GetBufferQueue: dataid={}, slot={}", param.dataid, param.slot);
let buffers = get_buffer_queues_by_data_id_and_slot(param.dataid, param.slot).await?;
let buffers = get_buffer_queues_by_data_id_and_slot(bufferparam.dataid, bufferparam.slot).await?;
Ok(buffers)
}
async fn prepare_get_object(&self, prepare_get_param: DataStorePrepareGetParam) -> Result<DataStoreReqGetInfo, ErrorCode> {
let meta_info = if prepare_get_param.dataid != 0 {
get_object_info_by_data_id(prepare_get_param.dataid, prepare_get_param.access_password).await?
} else {
get_object_info_by_persistence_target(prepare_get_param.persistence_target, prepare_get_param.access_password).await?
};
verify_object_permission(meta_info.owner, self.pid, &meta_info.permission)?;
let presigner = S3Presigner::new(
&format!("https://{}", *RNEX_DATASTORE_S3_ENDPOINT),
format!("{}", *RNEX_DATASTORE_S3_BUCKET)
).await;
let key = format!("data/{}.bin", meta_info.dataid);
let download_url = presigner.generate_presigned_get(&key);
Ok(DataStoreReqGetInfo {
url: download_url,
request_headers: vec![],
size: meta_info.size,
root_ca_cert: vec![],
dataid: meta_info.dataid,
})
}
async fn followings_latest_course_search_object(
&self,
course_search_param: DataStoreSearchParam,
_extra_data: Vec<String>
) -> Result<Vec<DataStoreCustomRankingResult>, ErrorCode> {
let mut all_results = Vec::new();
for &owner_pid in &course_search_param.owner_ids {
let course_ids = get_user_course_object_ids(owner_pid).await?;
if course_ids.is_empty() {
continue;
}
let mut results = get_custom_rankings_by_data_ids(0, course_ids).await;
// Flag 0x1: Return Tags
// Flag 0x2: Return Ratings
// Flag 0x4: Return MetaBinary
// Flag 0x20: Return Score
for res in &mut results {
if course_search_param.result_option & 0x1 == 0 {
res.meta_info.tags = Vec::new();
}
if course_search_param.result_option & 0x2 == 0 {
res.meta_info.ratings = Vec::new();
}
if course_search_param.result_option & 0x4 == 0 {
res.meta_info.meta_binary = rnex_core::rmc::structures::qbuffer::QBuffer(Vec::new());
}
if course_search_param.result_option & 0x20 == 0 {
res.score = 0;
}
}
all_results.extend(results);
}
// note: we assume the client sorts the data lol
Ok(all_results)
}
async fn get_application_config_string(&self, application_id: u32) -> Result<Vec<String>, ErrorCode> {
let config = match application_id {
128 => get_blacklist_1(),
129 => get_blacklist_2(),
130 => get_blacklist_3(),
_ => {
log::warn!("unsupported application_id in GetApplicationConfigString: {}", application_id);
Vec::new()
}
};
Ok(config)
}
async fn get_metas_multiple_param(
&self,
params: Vec<GetMetaParam>
) -> Result<(Vec<GetMetaInfo>, Vec<QResult>), ErrorCode> {
let mut metas = Vec::with_capacity(params.len());
let mut results = Vec::with_capacity(params.len());
for param in params {
let info_result = if param.dataid != 0 {
get_object_info_by_data_id(param.dataid, param.access_password).await
} else {
get_object_info_by_persistence_target(param.persistence_target, param.access_password).await
};
match info_result {
Ok(mut meta) => {
if let Err(e) = verify_object_permission(meta.owner, self.pid, &meta.permission) {
metas.push(GetMetaInfo::default());
results.push(QResult::error(e));
} else {
if param.result_option & 0x1 == 0 {
meta.tags = Vec::new();
}
if param.result_option & 0x2 == 0 {
meta.ratings = Vec::new();
}
if param.result_option & 0x4 == 0 {
meta.meta_binary = rnex_core::rmc::structures::qbuffer::QBuffer(Vec::new());
}
metas.push(meta);
results.push(QResult::success(ErrorCode::Core_Unknown));
}
}
Err(e) => {
metas.push(GetMetaInfo::default());
results.push(QResult::error(e));
}
}
}
Ok((metas, results))
}
}

44
rnex-core/src/nex/s3presigner.rs
View file

@ -56,6 +56,50 @@ impl S3Presigner {
(url, fields)
}
pub fn generate_presigned_get(&self, key: &str) -> String {
let access_key = std::env::var("AWS_ACCESS_KEY_ID").expect("Missing Access Key");
let secret_key = std::env::var("AWS_SECRET_ACCESS_KEY").expect("Missing Secret Key");
let region = "us-east-1";
let date_short = Utc::now().format("%Y%m%d").to_string();
let date_full = Utc::now().format("%Y%m%dT%H%M%SZ").to_string();
let credential_scope = format!("{}/{}/s3/aws4_request", date_short, region);
let query_string = format!(
"X-Amz-Algorithm=AWS4-HMAC-SHA256&\
X-Amz-Credential={}%2F{}&\
X-Amz-Date={}&\
X-Amz-Expires=900&\
X-Amz-SignedHeaders=host",
access_key,
urlencoding::encode(&credential_scope),
date_full
);
let canonical_request = format!(
"GET\n/{}/{}\n{}\nhost:{}\n\nhost\nUNSIGNED-PAYLOAD",
self.bucket, key, query_string, *RNEX_DATASTORE_S3_ENDPOINT
);
let hashed_request = hex::encode(Sha256::digest(canonical_request.as_bytes()));
let string_to_sign = format!(
"AWS4-HMAC-SHA256\n{}\n{}\n{}",
date_full, credential_scope, hashed_request
);
let k_date = self.hmac_sha256(format!("AWS4{}", secret_key).as_bytes(), &date_short);
let k_region = self.hmac_sha256(&k_date, region);
let k_service = self.hmac_sha256(&k_region, "s3");
let k_signing = self.hmac_sha256(&k_service, "aws4_request");
let signature = hex::encode(self.hmac_sha256(&k_signing, &string_to_sign));
format!(
"https://{}/{}/{}?{}&X-Amz-Signature={}",
*RNEX_DATASTORE_S3_ENDPOINT, self.bucket, key, query_string, signature
)
}
fn calculate_signature(&self, secret: &str, date: &str, region: &str, policy: &str) -> String {
let k_date = self.hmac_sha256(format!("AWS4{}", secret).as_bytes(), date);
let k_region = self.hmac_sha256(&k_date, region);

64
rnex-core/src/rmc/protocols/datastore.rs
View file

@ -4,15 +4,16 @@ use rnex_core::rmc::response::ErrorCode;
use rnex_core::rmc::structures::qresult::QResult;
use rnex_core::kerberos::KerberosDateTime;
use rnex_core::PID;
use rnex_core::rmc::structures::resultsrange::ResultsRange;
#[derive(RmcSerialize, Clone, Debug)]
#[derive(RmcSerialize, Clone, Debug, Default)]
#[rmc_struct(0)]
pub struct PersistenceTarget {
pub owner: PID,
pub persistence_slot_id: u16,
}
#[derive(RmcSerialize, Clone, Debug)]
#[derive(RmcSerialize, Clone, Debug, Default)]
#[rmc_struct(0)]
pub struct Permission {
pub permission: u8,
@ -34,7 +35,7 @@ pub struct RatingInfo {
pub initial_value: i64,
}
#[derive(RmcSerialize, Clone)]
#[derive(RmcSerialize, Clone, Default)]
#[rmc_struct(0)]
pub struct GetMetaParam {
pub dataid: u64,
@ -43,7 +44,7 @@ pub struct GetMetaParam {
pub access_password: u64,
}
#[derive(RmcSerialize, Clone)]
#[derive(RmcSerialize, Clone, Default)]
#[rmc_struct(0)]
pub struct GetMetaInfo {
pub dataid: u64,
@ -165,14 +166,59 @@ pub struct DataStoreGetCustomRankingByDataIDParam {
#[derive(RmcSerialize, Clone)]
#[rmc_struct(0)]
pub struct DataStoreCustomRankingResult {
pub order: u32,
pub score: u32,
pub meta_info: GetMetaInfo,
}
#[derive(RmcSerialize, Clone)]
#[rmc_struct(0)]
pub struct DataStorePrepareGetParam {
pub dataid: u64,
pub lockid: u32,
pub persistence_target: PersistenceTarget,
pub access_password: u64,
pub extra_data: Vec<String>,
}
#[derive(RmcSerialize, Clone)]
#[rmc_struct(0)]
pub struct DataStoreReqGetInfo {
pub url: String,
pub request_headers: Vec<KeyValue>,
pub size: u32,
pub root_ca_cert: Vec<u8>,
pub dataid: u64,
}
#[derive(RmcSerialize, Clone)]
#[rmc_struct(1)]
pub struct DataStoreSearchParam {
pub search_target: u8,
pub owner_ids: Vec<PID>,
pub owner_type: u8,
pub destination_ids: Vec<u64>,
pub data_type: u16,
pub created_after: KerberosDateTime,
pub created_before: KerberosDateTime,
pub updated_after: KerberosDateTime,
pub updated_before: KerberosDateTime,
pub refer_dat_id: u32,
pub tags: Vec<String>,
pub result_order_column: u8,
pub result_order: u8,
pub result_range: ResultsRange,
pub result_option: u8,
pub minimal_rating_frequency: u32,
pub use_cache: bool,
}
#[rmc_proto(115)]
pub trait DataStore{
#[method_id(8)]
async fn get_meta(&self, metaparam: GetMetaParam) -> Result<GetMetaInfo, ErrorCode>;
#[method_id(36)]
async fn get_metas_multiple_param(&self, params: Vec<GetMetaParam>) -> Result<(Vec<GetMetaInfo>, Vec<QResult>), ErrorCode>;
#[method_id(24)]
async fn prepare_post_object(&self, postparam: PreparePostParam) -> Result<ReqPostInfo, ErrorCode>;
#[method_id(26)]
@ -182,7 +228,13 @@ pub trait DataStore{
#[method_id(61)]
async fn get_application_config(&self, appid: u32) -> Result<Vec<i32>, ErrorCode>;
#[method_id(50)]
async fn get_custom_ranking_by_data_id(&self, param: DataStoreGetCustomRankingByDataIDParam) -> Result<(Vec<DataStoreCustomRankingResult>, Vec<QResult>), ErrorCode>;
async fn get_custom_ranking_by_data_id(&self, custom_ranking_param: DataStoreGetCustomRankingByDataIDParam) -> Result<(Vec<DataStoreCustomRankingResult>, Vec<QResult>), ErrorCode>;
#[method_id(54)]
async fn get_buffer_queue(&self, param: BufferQueueParam) -> Result<Vec<QBuffer>, ErrorCode>;
async fn get_buffer_queue(&self, bufferparam: BufferQueueParam) -> Result<Vec<QBuffer>, ErrorCode>;
#[method_id(25)]
async fn prepare_get_object(&self, prepare_get_param: DataStorePrepareGetParam) -> Result<DataStoreReqGetInfo, ErrorCode>;
#[method_id(65)]
async fn followings_latest_course_search_object(&self, course_search_param: DataStoreSearchParam, extra_data: Vec<String>) -> Result<Vec<DataStoreCustomRankingResult>, ErrorCode>;
#[method_id(74)]
async fn get_application_config_string(&self, application_id: u32) -> Result<Vec<String>, ErrorCode>;
}

9
rnex-core/src/rmc/protocols/ranking.rs
View file

@ -2,17 +2,10 @@ use macros::{rmc_struct, rmc_proto, RmcSerialize, method_id};
use rnex_core::kerberos::KerberosDateTime;
use rnex_core::rmc::structures::qbuffer::QBuffer;
use rnex_core::rmc::structures::resultsrange::ResultsRange;
use rnex_core::rmc::response::ErrorCode;
use rnex_core::rmc::structures::ranking::UploadCompetitionData;
#[derive(RmcSerialize, Debug, Default, Clone)]
#[rmc_struct(0)]
pub struct ResultsRange{
pub offset: u32,
pub size: u32
}
#[derive(RmcSerialize, Debug, Default, Clone)]
#[rmc_struct(1)]
pub struct CompetitionRankingGetParam {

1
rnex-core/src/rmc/structures/mod.rs
View file

@ -37,6 +37,7 @@ pub mod ranking;
pub mod rmc_struct;
pub mod string;
pub mod variant;
pub mod resultsrange;
pub trait RmcSerialize {
fn serialize(&self, writer: &mut impl Write) -> Result<()>;

2
rnex-core/src/rmc/structures/qbuffer.rs
View file

@ -3,7 +3,7 @@ use bytemuck::bytes_of;
use v_byte_helpers::{IS_BIG_ENDIAN, ReadExtensions};
use crate::rmc::structures::{Result, RmcSerialize};
#[derive(Clone, Debug)]
#[derive(Clone, Debug, Default)]
pub struct QBuffer(pub Vec<u8>);
impl RmcSerialize for QBuffer{

8
rnex-core/src/rmc/structures/resultsrange.rs Normal file
View file

@ -0,0 +1,8 @@
use macros::RmcSerialize;
#[derive(RmcSerialize, Debug, Default, Clone)]
#[rmc_struct(0)]
pub struct ResultsRange{
pub offset: u32,
pub size: u32
}