From 47a6e51a2a060460791faad7dfb9e0af18731126 Mon Sep 17 00:00:00 2001 From: hole-thu Date: Sun, 10 Apr 2022 15:34:10 +0800 Subject: [PATCH] feat: configurable auto block --- README.md | 2 +- src/api/attention.rs | 2 +- src/api/comment.rs | 18 ++++++---- src/api/mod.rs | 12 +++++-- src/api/operation.rs | 48 +++++++++++++++++++------ src/api/post.rs | 84 +++++++++++++++++++++----------------------- src/api/search.rs | 2 +- src/cache.rs | 55 +++++++++++++++++++++++++++-- src/main.rs | 7 ++-- src/rds_models.rs | 66 +++++++++++++++++++--------------- 10 files changed, 195 insertions(+), 101 deletions(-) diff --git a/README.md b/README.md index 5525302..935f93f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# hole-backend-rust v1.1.0 +# hole-backend-rust v1.2.0 ## 部署 diff --git a/src/api/attention.rs b/src/api/attention.rs index 5fc8cfb..7ef166a 100644 --- a/src/api/attention.rs +++ b/src/api/attention.rs @@ -67,7 +67,7 @@ pub async fn get_attention(user: CurrentUser, db: Db, rconn: RdsConn) -> JsonAPI let mut ids = Attention::init(&user.namehash, &rconn).all().await?; ids.sort_by_key(|x| -x); let ps = Post::get_multi(&db, &rconn, &ids).await?; - let ps_data = ps2outputs(&ps, &user, &db, &rconn).await; + let ps_data = ps2outputs(&ps, &user, &db, &rconn).await?; code0!(ps_data) } diff --git a/src/api/comment.rs b/src/api/comment.rs index cd1cc9c..a5210ff 100644 --- a/src/api/comment.rs +++ b/src/api/comment.rs @@ -1,4 +1,5 @@ use crate::api::{APIError, CurrentUser, JsonAPI, PolicyError::*, UGC}; +use crate::cache::BlockDictCache; use crate::db_conn::Db; use crate::libs::diesel_logger::LoggingConnection; use crate::models::*; @@ -41,6 +42,7 @@ pub async fn c2output<'r>( p: &'r Post, cs: &Vec, user: &CurrentUser, + cached_block_dict: &HashMap, rconn: &RdsConn, ) -> Vec { let mut hash2id = HashMap::<&String, i32>::from([(&p.author_hash, 0)]); @@ -56,10 +58,7 @@ pub async fn c2output<'r>( if c.is_deleted { None } else { - let is_blocked = - BlockedUsers::check_blocked(rconn, user.id, &user.namehash, &c.author_hash) - .await - .unwrap_or_default(); + let is_blocked = cached_block_dict[&c.author_hash]; let can_view = user.is_admin || (!is_blocked && user.id.is_some() || user.namehash.eq(&c.author_hash)); Some(CommentOutput { @@ -72,7 +71,10 @@ pub async fn c2output<'r>( create_time: c.create_time.timestamp(), is_blocked: is_blocked, blocked_count: if user.is_admin { - BlockCounter::get_count(rconn, &c.author_hash).await.ok() + BlockCounter::get_count(rconn, &c.author_hash) + .await + .ok() + .flatten() } else { None }, @@ -94,7 +96,11 @@ pub async fn get_comment(pid: i32, user: CurrentUser, db: Db, rconn: RdsConn) -> return Err(APIError::PcError(IsDeleted)); } let cs = p.get_comments(&db, &rconn).await?; - let data = c2output(&p, &cs, &user, &rconn).await; + let hash_list = cs.iter().map(|c| &c.author_hash).collect(); + let cached_block_dict = BlockDictCache::init(&user.namehash, p.id, &rconn) + .get_or_create(&user, &hash_list) + .await?; + let data = c2output(&p, &cs, &user, &cached_block_dict, &rconn).await; Ok(json!({ "code": 0, diff --git a/src/api/mod.rs b/src/api/mod.rs index 498da64..727f156 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -3,7 +3,7 @@ use crate::libs::diesel_logger::LoggingConnection; use crate::models::*; use crate::random_hasher::RandomHasher; use crate::rds_conn::RdsConn; -use crate::rds_models::BannedUsers; +use crate::rds_models::*; use crate::schema; use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; use rocket::http::Status; @@ -52,10 +52,11 @@ pub fn catch_403_error() -> &'static str { } pub struct CurrentUser { - id: Option, // tmp user has no id, only for block + pub id: Option, // tmp user has no id, only for block namehash: String, is_admin: bool, custom_title: String, + pub auto_block_rank: u8, } #[rocket::async_trait] @@ -91,7 +92,12 @@ impl<'r> FromRequest<'r> for CurrentUser { } else { Outcome::Success(CurrentUser { id: id, - custom_title: format!("title todo: {}", &nh), + custom_title: CustomTitle::get(&rconn, &nh) + .await + .ok() + .flatten() + .unwrap_or_default(), + auto_block_rank: AutoBlockRank::get(&rconn, &nh).await.unwrap_or(2), namehash: nh, is_admin: is_admin, }) diff --git a/src/api/operation.rs b/src/api/operation.rs index 8db9281..ab6c593 100644 --- a/src/api/operation.rs +++ b/src/api/operation.rs @@ -1,4 +1,5 @@ -use crate::api::{CurrentUser, JsonAPI, PolicyError::*, UGC}; +use crate::api::{APIError, CurrentUser, JsonAPI, PolicyError::*, UGC}; +use crate::cache::*; use crate::db_conn::Db; use crate::libs::diesel_logger::LoggingConnection; use crate::models::*; @@ -72,7 +73,6 @@ pub async fn delete(di: Form, user: CurrentUser, db: Db, rconn: Rds .create(&rconn) .await?; BannedUsers::add(&rconn, &author_hash).await?; - DangerousUser::add(&rconn, &author_hash).await?; } } @@ -128,28 +128,39 @@ pub async fn block(bi: Form, user: CurrentUser, db: Db, rconn: RdsCo let mut blk = BlockedUsers::init(user.id.ok_or_else(|| NotAllowed)?, &rconn); + let pid; let nh_to_block = match bi.content_type.as_str() { - "post" => Post::get(&db, &rconn, bi.id).await?.author_hash, - "comment" => Comment::get(&db, bi.id).await?.author_hash, - _ => Err(NotAllowed)?, + "post" => { + let p = Post::get(&db, &rconn, bi.id).await?; + pid = p.id; + p.author_hash + } + "comment" => { + let c = Comment::get(&db, bi.id).await?; + pid = c.post_id; + c.author_hash + } + _ => return Err(APIError::PcError(NotAllowed)), }; if nh_to_block.eq(&user.namehash) { Err(NotAllowed)?; } - blk.add(&nh_to_block).await?; - let curr = BlockCounter::count_incr(&rconn, &nh_to_block).await?; + let curr = if blk.add(&nh_to_block).await? > 0 { + BlockCounter::count_incr(&rconn, &nh_to_block).await? + } else { + 114514 + }; - if curr >= BLOCK_THRESHOLD || user.is_admin { - DangerousUser::add(&rconn, &nh_to_block).await?; - } + BlockDictCache::init(&user.namehash, pid, &rconn) + .clear() + .await?; Ok(json!({ "code": 0, "data": { "curr": curr, - "threshold": BLOCK_THRESHOLD, }, })) } @@ -168,3 +179,18 @@ pub async fn set_title(ti: Form, user: CurrentUser, rconn: RdsConn) Err(TitleUsed)? } } + +#[derive(FromForm)] +pub struct AutoBlockInput { + rank: u8, +} + +#[post("/auto_block", data = "")] +pub async fn set_auto_block( + ai: Form, + user: CurrentUser, + rconn: RdsConn, +) -> JsonAPI { + AutoBlockRank::set(&rconn, &user.namehash, ai.rank).await?; + code0!() +} diff --git a/src/api/post.rs b/src/api/post.rs index b7015bd..82fd159 100644 --- a/src/api/post.rs +++ b/src/api/post.rs @@ -1,6 +1,7 @@ use crate::api::comment::{c2output, CommentOutput}; use crate::api::vote::get_poll_dict; -use crate::api::{CurrentUser, JsonAPI, PolicyError::*, UGC}; +use crate::api::{CurrentUser, JsonAPI, PolicyError::*, API, UGC}; +use crate::cache::*; use crate::db_conn::Db; use crate::libs::diesel_logger::LoggingConnection; use crate::models::*; @@ -9,7 +10,7 @@ use crate::rds_models::*; use crate::schema; use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; use rocket::form::Form; -use rocket::futures::future; +use rocket::futures::future::{self, OptionFuture}; use rocket::serde::{ json::{json, Value}, Serialize, @@ -62,47 +63,48 @@ pub struct CwInput { cw: String, } -async fn p2output(p: &Post, user: &CurrentUser, db: &Db, rconn: &RdsConn) -> PostOutput { - let is_blocked = BlockedUsers::check_blocked(rconn, user.id, &user.namehash, &p.author_hash) - .await - .unwrap_or_default(); +async fn p2output(p: &Post, user: &CurrentUser, db: &Db, rconn: &RdsConn) -> API { + let comments: Option> = if p.n_comments < 5 { + Some(p.get_comments(db, rconn).await?) + } else { + None + }; + let hash_list = comments + .iter() + .flatten() + .map(|c| &c.author_hash) + .chain(std::iter::once(&p.author_hash)) + .collect(); + //dbg!(&hash_list); + let cached_block_dict = BlockDictCache::init(&user.namehash, p.id, rconn) + .get_or_create(&user, &hash_list) + .await?; + let is_blocked = cached_block_dict[&p.author_hash]; let can_view = user.is_admin || (!is_blocked && user.id.is_some() || user.namehash.eq(&p.author_hash)); - PostOutput { + Ok(PostOutput { pid: p.id, - text: (if can_view { &p.content } else { "" }).to_string(), - cw: (!p.cw.is_empty()).then(|| p.cw.to_string()), + text: can_view.then(|| p.content.clone()).unwrap_or_default(), + cw: (!p.cw.is_empty()).then(|| p.cw.clone()), n_attentions: p.n_attentions, n_comments: p.n_comments, create_time: p.create_time.timestamp(), last_comment_time: p.last_comment_time.timestamp(), allow_search: p.allow_search, - author_title: (!p.author_title.is_empty()).then(|| p.author_title.to_string()), + author_title: (!p.author_title.is_empty()).then(|| p.author_title.clone()), is_tmp: p.is_tmp, is_reported: user.is_admin.then(|| p.is_reported), - comments: if p.n_comments > 50 { - None - } else { - // 单个洞还有查询评论的接口，这里挂了不用报错 - Some( - c2output( - p, - &p.get_comments(db, rconn).await.unwrap_or(vec![]), - user, - rconn, - ) - .await, - ) - }, + comments: OptionFuture::from( + comments + .map(|cs| async move { c2output(p, &cs, user, &cached_block_dict, rconn).await }), + ) + .await, can_del: p.check_permission(user, "wd").is_ok(), - attention: Attention::init(&user.namehash, &rconn) - .has(p.id) - .await - .unwrap_or_default(), + attention: Attention::init(&user.namehash, &rconn).has(p.id).await?, hot_score: user.is_admin.then(|| p.hot_score), is_blocked: is_blocked, blocked_count: if user.is_admin { - BlockCounter::get_count(rconn, &p.author_hash).await.ok() + BlockCounter::get_count(rconn, &p.author_hash).await? } else { None }, @@ -116,7 +118,7 @@ async fn p2output(p: &Post, user: &CurrentUser, db: &Db, rconn: &RdsConn) -> Pos likenum: p.n_attentions, reply: p.n_comments, blocked: is_blocked, - } + }) } pub async fn ps2outputs( @@ -124,10 +126,10 @@ pub async fn ps2outputs( user: &CurrentUser, db: &Db, rconn: &RdsConn, -) -> Vec { - future::join_all( +) -> API> { + future::try_join_all( ps.iter() - .map(|p| async { p2output(p, &user, &db, &rconn).await }), + .map(|p| async { Ok(p2output(p, &user, &db, &rconn).await?) }), ) .await } @@ -137,7 +139,7 @@ pub async fn get_one(pid: i32, user: CurrentUser, db: Db, rconn: RdsConn) -> Jso let p = Post::get(&db, &rconn, pid).await?; p.check_permission(&user, "ro")?; Ok(json!({ - "data": p2output(&p, &user,&db, &rconn).await, + "data": p2output(&p, &user,&db, &rconn).await?, "code": 0, })) } @@ -155,11 +157,12 @@ pub async fn get_list( let page_size = 25; let start = (page - 1) * page_size; let ps = Post::gets_by_page(&db, &rconn, order_mode, start.into(), page_size.into()).await?; - let ps_data = ps2outputs(&ps, &user, &db, &rconn).await; + let ps_data = ps2outputs(&ps, &user, &db, &rconn).await?; Ok(json!({ "data": ps_data, "count": ps_data.len(), - "custom_title": CustomTitle::get(&rconn, &user.namehash).await?, + "custom_title": user.custom_title, + "auto_block_rank": user.auto_block_rank, "code": 0 })) } @@ -177,12 +180,7 @@ pub async fn publish_post( content: poi.text.to_string(), cw: poi.cw.to_string(), author_hash: user.namehash.to_string(), - author_title: (if poi.use_title.is_some() { - CustomTitle::get(&rconn, &user.namehash).await? - } else { - None - }) - .unwrap_or_default(), + author_title: poi.use_title.map(|_| user.custom_title).unwrap_or_default(), is_tmp: user.id.is_none(), n_attentions: 1, allow_search: poi.allow_search.is_some(), @@ -213,7 +211,7 @@ pub async fn edit_cw(cwi: Form, user: CurrentUser, db: Db, rconn: RdsCo pub async fn get_multi(pids: Vec, user: CurrentUser, db: Db, rconn: RdsConn) -> JsonAPI { user.id.ok_or_else(|| YouAreTmp)?; let ps = Post::get_multi(&db, &rconn, &pids).await?; - let ps_data = ps2outputs(&ps, &user, &db, &rconn).await; + let ps_data = ps2outputs(&ps, &user, &db, &rconn).await?; Ok(json!({ "code": 0, diff --git a/src/api/search.rs b/src/api/search.rs index 361558f..7e95faa 100644 --- a/src/api/search.rs +++ b/src/api/search.rs @@ -36,7 +36,7 @@ pub async fn search( ) .await? }; - let ps_data = ps2outputs(&ps, &user, &db, &rconn).await; + let ps_data = ps2outputs(&ps, &user, &db, &rconn).await?; Ok(json!({ "data": ps_data, "count": ps_data.len(), diff --git a/src/cache.rs b/src/cache.rs index 2a5e518..1056e72 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -1,10 +1,13 @@ +use crate::api::CurrentUser; use crate::models::{Comment, Post, User}; use crate::rds_conn::RdsConn; -use crate::rds_models::init; +use crate::rds_models::{init, BlockedUsers}; use rand::Rng; -use redis::AsyncCommands; +use redis::{AsyncCommands, RedisError, RedisResult}; use rocket::serde::json::serde_json; // can use rocket::serde::json::to_string in master version +use rocket::futures::future; +use std::collections::HashMap; const INSTANCE_EXPIRE_TIME: usize = 60 * 60; @@ -321,3 +324,51 @@ impl UserCache { } } } + +pub struct BlockDictCache { + key: String, + rconn: RdsConn, +} + +impl BlockDictCache { + // namehash, pid + init!(&str, i32, "hole_v2:cache:block_dict:{}:{}"); + + pub async fn get_or_create( + &mut self, + user: &CurrentUser, + hash_list: &Vec<&String>, + ) -> RedisResult> { + let mut block_dict = self + .rconn + .hgetall::<&String, HashMap>(&self.key) + .await?; + + //dbg!(&self.key, &block_dict); + + let missing: Vec<(String, bool)> = + future::try_join_all(hash_list.iter().filter_map(|hash| { + (!block_dict.contains_key(&hash.to_string())).then(|| async { + Ok::<(String, bool), RedisError>(( + hash.to_string(), + BlockedUsers::check_if_block(&self.rconn, user, hash).await?, + )) + }) + })) + .await?; + + if !missing.is_empty() { + self.rconn.hset_multiple(&self.key, &missing).await?; + self.rconn.expire(&self.key, INSTANCE_EXPIRE_TIME).await?; + block_dict.extend(missing.into_iter()); + } + + //dbg!(&block_dict); + + Ok(block_dict) + } + + pub async fn clear(&mut self) -> RedisResult<()> { + self.rconn.del(&self.key).await + } +} diff --git a/src/main.rs b/src/main.rs index b2273ed..649e231 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,6 +26,7 @@ use db_conn::{establish_connection, Conn, Db}; use diesel::Connection; use random_hasher::RandomHasher; use rds_conn::{init_rds_client, RdsConn}; +use rds_models::clear_outdate_redis_data; use std::env; use tokio::time::{sleep, Duration}; @@ -68,6 +69,7 @@ async fn main() -> Result<(), rocket::Error> { api::operation::report, api::operation::set_title, api::operation::block, + api::operation::set_auto_block, api::vote::vote, api::upload::ipfs_upload, ], @@ -105,8 +107,3 @@ fn init_database() { let conn = Conn::establish(&database_url).unwrap(); embedded_migrations::run(&conn).unwrap(); } - -async fn clear_outdate_redis_data(rconn: &RdsConn) { - rds_models::BannedUsers::clear(&rconn).await.unwrap(); - rds_models::CustomTitle::clear(&rconn).await.unwrap(); -} diff --git a/src/rds_models.rs b/src/rds_models.rs index 6fd1cbf..0c2dc85 100644 --- a/src/rds_models.rs +++ b/src/rds_models.rs @@ -1,3 +1,4 @@ +use crate::api::CurrentUser; use crate::rds_conn::RdsConn; use chrono::{offset::Local, DateTime}; use redis::{AsyncCommands, RedisResult}; @@ -40,7 +41,7 @@ macro_rules! has { macro_rules! add { ($vtype:ty) => { - pub async fn add(&mut self, v: $vtype) -> RedisResult<()> { + pub async fn add(&mut self, v: $vtype) -> RedisResult { self.rconn.sadd(&self.key, v).await } }; @@ -49,11 +50,10 @@ macro_rules! add { const KEY_SYSTEMLOG: &str = "hole_v2:systemlog_list"; const KEY_BANNED_USERS: &str = "hole_v2:banned_user_hash_list"; const KEY_BLOCKED_COUNTER: &str = "hole_v2:blocked_counter"; -const KEY_DANGEROUS_USERS: &str = "hole_thu:dangerous_users"; //兼容一下旧版 const KEY_CUSTOM_TITLE: &str = "hole_v2:title"; +const KEY_AUTO_BLOCK_RANK: &str = "hole_v2:auto_block_rank"; // rank * 5: 自动过滤的拉黑数阈值 const SYSTEMLOG_MAX_LEN: isize = 1000; -pub const BLOCK_THRESHOLD: i32 = 10; pub struct Attention { key: String, @@ -159,47 +159,31 @@ impl BlockedUsers { has!(&str); - pub async fn check_blocked( + pub async fn check_if_block( rconn: &RdsConn, - viewer_id: Option, - viewer_hash: &str, - author_hash: &str, + user: &CurrentUser, + hash: &str, ) -> RedisResult { - Ok(match viewer_id { - Some(id) => Self::init(id, rconn).has(author_hash).await?, + Ok(match user.id { + Some(id) => BlockedUsers::init(id, rconn).has(hash).await?, None => false, - } || (DangerousUser::has(rconn, author_hash).await? - && !DangerousUser::has(rconn, viewer_hash).await?)) + } || BlockCounter::get_count(rconn, hash).await?.unwrap_or(0) + >= i32::from(user.auto_block_rank) * 5) } } pub struct BlockCounter; impl BlockCounter { - pub async fn count_incr(rconn: &RdsConn, namehash: &str) -> RedisResult { + pub async fn count_incr(rconn: &RdsConn, namehash: &str) -> RedisResult { rconn.clone().hincr(KEY_BLOCKED_COUNTER, namehash, 1).await } - pub async fn get_count(rconn: &RdsConn, namehash: &str) -> RedisResult { + pub async fn get_count(rconn: &RdsConn, namehash: &str) -> RedisResult> { rconn.clone().hget(KEY_BLOCKED_COUNTER, namehash).await } } -pub struct DangerousUser; - -impl DangerousUser { - pub async fn add(rconn: &RdsConn, namehash: &str) -> RedisResult<()> { - rconn - .clone() - .sadd::<&str, &str, ()>(KEY_DANGEROUS_USERS, namehash) - .await - } - - pub async fn has(rconn: &RdsConn, namehash: &str) -> RedisResult { - rconn.clone().sismember(KEY_DANGEROUS_USERS, namehash).await - } -} - pub struct CustomTitle; impl CustomTitle { @@ -224,6 +208,26 @@ impl CustomTitle { } } +pub struct AutoBlockRank; + +impl AutoBlockRank { + pub async fn set(rconn: &RdsConn, namehash: &str, rank: u8) -> RedisResult { + rconn + .clone() + .hset(KEY_AUTO_BLOCK_RANK, namehash, rank) + .await + } + + pub async fn get(rconn: &RdsConn, namehash: &str) -> RedisResult { + let rank: Option = rconn.clone().hget(KEY_AUTO_BLOCK_RANK, namehash).await?; + Ok(rank.unwrap_or(2)) + } + + pub async fn clear(rconn: &RdsConn) -> RedisResult<()> { + rconn.clone().del(KEY_AUTO_BLOCK_RANK).await + } +} + pub struct PollOption { key: String, rconn: RdsConn, @@ -259,4 +263,10 @@ impl PollVote { } } +pub async fn clear_outdate_redis_data(rconn: &RdsConn) { + BannedUsers::clear(&rconn).await.unwrap(); + CustomTitle::clear(&rconn).await.unwrap(); + AutoBlockRank::clear(&rconn).await.unwrap(); +} + pub(crate) use init;