feat: configurable auto block

README.md

@@ -1,4 +1,4 @@
-# hole-backend-rust v1.1.0
+# hole-backend-rust v1.2.0
 
 
 ## 部署

src/api/attention.rs

@@ -67,7 +67,7 @@ pub async fn get_attention(user: CurrentUser, db: Db, rconn: RdsConn) -> JsonAPI
     let mut ids = Attention::init(&user.namehash, &rconn).all().await?;
     ids.sort_by_key(|x| -x);
     let ps = Post::get_multi(&db, &rconn, &ids).await?;
-    let ps_data = ps2outputs(&ps, &user, &db, &rconn).await;
+    let ps_data = ps2outputs(&ps, &user, &db, &rconn).await?;
 
     code0!(ps_data)
 }

src/api/comment.rs

@@ -1,4 +1,5 @@
 use crate::api::{APIError, CurrentUser, JsonAPI, PolicyError::*, UGC};
+use crate::cache::BlockDictCache;
 use crate::db_conn::Db;
 use crate::libs::diesel_logger::LoggingConnection;
 use crate::models::*;
@@ -41,6 +42,7 @@ pub async fn c2output<'r>(
     p: &'r Post,
     cs: &Vec<Comment>,
     user: &CurrentUser,
+    cached_block_dict: &HashMap<String, bool>,
     rconn: &RdsConn,
 ) -> Vec<CommentOutput> {
     let mut hash2id = HashMap::<&String, i32>::from([(&p.author_hash, 0)]);
@@ -56,10 +58,7 @@ pub async fn c2output<'r>(
         if c.is_deleted {
             None
         } else {
-            let is_blocked =
+            let is_blocked = cached_block_dict[&c.author_hash];
-                BlockedUsers::check_blocked(rconn, user.id, &user.namehash, &c.author_hash)
-                    .await
-                    .unwrap_or_default();
             let can_view = user.is_admin
                 || (!is_blocked && user.id.is_some() || user.namehash.eq(&c.author_hash));
             Some(CommentOutput {
@@ -72,7 +71,10 @@ pub async fn c2output<'r>(
                 create_time: c.create_time.timestamp(),
                 is_blocked: is_blocked,
                 blocked_count: if user.is_admin {
-                    BlockCounter::get_count(rconn, &c.author_hash).await.ok()
+                    BlockCounter::get_count(rconn, &c.author_hash)
+                        .await
+                        .ok()
+                        .flatten()
                 } else {
                     None
                 },
@@ -94,7 +96,11 @@ pub async fn get_comment(pid: i32, user: CurrentUser, db: Db, rconn: RdsConn) ->
         return Err(APIError::PcError(IsDeleted));
     }
     let cs = p.get_comments(&db, &rconn).await?;
-    let data = c2output(&p, &cs, &user, &rconn).await;
+    let hash_list = cs.iter().map(|c| &c.author_hash).collect();
+    let cached_block_dict = BlockDictCache::init(&user.namehash, p.id, &rconn)
+        .get_or_create(&user, &hash_list)
+        .await?;
+    let data = c2output(&p, &cs, &user, &cached_block_dict, &rconn).await;
 
     Ok(json!({
         "code": 0,

src/api/mod.rs

@@ -3,7 +3,7 @@ use crate::libs::diesel_logger::LoggingConnection;
 use crate::models::*;
 use crate::random_hasher::RandomHasher;
 use crate::rds_conn::RdsConn;
-use crate::rds_models::BannedUsers;
+use crate::rds_models::*;
 use crate::schema;
 use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
 use rocket::http::Status;
@@ -52,10 +52,11 @@ pub fn catch_403_error() -> &'static str {
 }
 
 pub struct CurrentUser {
-    id: Option<i32>, // tmp user has no id, only for block
+    pub id: Option<i32>, // tmp user has no id, only for block
     namehash: String,
     is_admin: bool,
     custom_title: String,
+    pub auto_block_rank: u8,
 }
 
 #[rocket::async_trait]
@@ -91,7 +92,12 @@ impl<'r> FromRequest<'r> for CurrentUser {
                 } else {
                     Outcome::Success(CurrentUser {
                         id: id,
-                        custom_title: format!("title todo: {}", &nh),
+                        custom_title: CustomTitle::get(&rconn, &nh)
+                            .await
+                            .ok()
+                            .flatten()
+                            .unwrap_or_default(),
+                        auto_block_rank: AutoBlockRank::get(&rconn, &nh).await.unwrap_or(2),
                         namehash: nh,
                         is_admin: is_admin,
                     })

src/api/operation.rs

@@ -1,4 +1,5 @@
-use crate::api::{CurrentUser, JsonAPI, PolicyError::*, UGC};
+use crate::api::{APIError, CurrentUser, JsonAPI, PolicyError::*, UGC};
+use crate::cache::*;
 use crate::db_conn::Db;
 use crate::libs::diesel_logger::LoggingConnection;
 use crate::models::*;
@@ -72,7 +73,6 @@ pub async fn delete(di: Form<DeleteInput>, user: CurrentUser, db: Db, rconn: Rds
             .create(&rconn)
             .await?;
             BannedUsers::add(&rconn, &author_hash).await?;
-            DangerousUser::add(&rconn, &author_hash).await?;
         }
     }
 
@@ -128,28 +128,39 @@ pub async fn block(bi: Form<BlockInput>, user: CurrentUser, db: Db, rconn: RdsCo
 
     let mut blk = BlockedUsers::init(user.id.ok_or_else(|| NotAllowed)?, &rconn);
 
+    let pid;
     let nh_to_block = match bi.content_type.as_str() {
-        "post" => Post::get(&db, &rconn, bi.id).await?.author_hash,
+        "post" => {
-        "comment" => Comment::get(&db, bi.id).await?.author_hash,
+            let p = Post::get(&db, &rconn, bi.id).await?;
-        _ => Err(NotAllowed)?,
+            pid = p.id;
+            p.author_hash
+        }
+        "comment" => {
+            let c = Comment::get(&db, bi.id).await?;
+            pid = c.post_id;
+            c.author_hash
+        }
+        _ => return Err(APIError::PcError(NotAllowed)),
     };
 
     if nh_to_block.eq(&user.namehash) {
         Err(NotAllowed)?;
     }
 
-    blk.add(&nh_to_block).await?;
+    let curr = if blk.add(&nh_to_block).await? > 0 {
-    let curr = BlockCounter::count_incr(&rconn, &nh_to_block).await?;
+        BlockCounter::count_incr(&rconn, &nh_to_block).await?
+    } else {
+        114514
+    };
 
-    if curr >= BLOCK_THRESHOLD || user.is_admin {
+    BlockDictCache::init(&user.namehash, pid, &rconn)
-        DangerousUser::add(&rconn, &nh_to_block).await?;
+        .clear()
-    }
+        .await?;
 
     Ok(json!({
         "code": 0,
         "data": {
             "curr": curr,
-            "threshold": BLOCK_THRESHOLD,
         },
     }))
 }
@@ -168,3 +179,18 @@ pub async fn set_title(ti: Form<TitleInput>, user: CurrentUser, rconn: RdsConn)
         Err(TitleUsed)?
     }
 }
+
+#[derive(FromForm)]
+pub struct AutoBlockInput {
+    rank: u8,
+}
+
+#[post("/auto_block", data = "<ai>")]
+pub async fn set_auto_block(
+    ai: Form<AutoBlockInput>,
+    user: CurrentUser,
+    rconn: RdsConn,
+) -> JsonAPI {
+    AutoBlockRank::set(&rconn, &user.namehash, ai.rank).await?;
+    code0!()
+}

src/api/post.rs

@@ -1,6 +1,7 @@
 use crate::api::comment::{c2output, CommentOutput};
 use crate::api::vote::get_poll_dict;
-use crate::api::{CurrentUser, JsonAPI, PolicyError::*, UGC};
+use crate::api::{CurrentUser, JsonAPI, PolicyError::*, API, UGC};
+use crate::cache::*;
 use crate::db_conn::Db;
 use crate::libs::diesel_logger::LoggingConnection;
 use crate::models::*;
@@ -9,7 +10,7 @@ use crate::rds_models::*;
 use crate::schema;
 use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
 use rocket::form::Form;
-use rocket::futures::future;
+use rocket::futures::future::{self, OptionFuture};
 use rocket::serde::{
     json::{json, Value},
     Serialize,
@@ -62,47 +63,48 @@ pub struct CwInput {
     cw: String,
 }
 
-async fn p2output(p: &Post, user: &CurrentUser, db: &Db, rconn: &RdsConn) -> PostOutput {
+async fn p2output(p: &Post, user: &CurrentUser, db: &Db, rconn: &RdsConn) -> API<PostOutput> {
-    let is_blocked = BlockedUsers::check_blocked(rconn, user.id, &user.namehash, &p.author_hash)
+    let comments: Option<Vec<Comment>> = if p.n_comments < 5 {
-        .await
+        Some(p.get_comments(db, rconn).await?)
-        .unwrap_or_default();
+    } else {
+        None
+    };
+    let hash_list = comments
+        .iter()
+        .flatten()
+        .map(|c| &c.author_hash)
+        .chain(std::iter::once(&p.author_hash))
+        .collect();
+    //dbg!(&hash_list);
+    let cached_block_dict = BlockDictCache::init(&user.namehash, p.id, rconn)
+        .get_or_create(&user, &hash_list)
+        .await?;
+    let is_blocked = cached_block_dict[&p.author_hash];
     let can_view =
         user.is_admin || (!is_blocked && user.id.is_some() || user.namehash.eq(&p.author_hash));
-    PostOutput {
+    Ok(PostOutput {
         pid: p.id,
-        text: (if can_view { &p.content } else { "" }).to_string(),
+        text: can_view.then(|| p.content.clone()).unwrap_or_default(),
-        cw: (!p.cw.is_empty()).then(|| p.cw.to_string()),
+        cw: (!p.cw.is_empty()).then(|| p.cw.clone()),
         n_attentions: p.n_attentions,
         n_comments: p.n_comments,
         create_time: p.create_time.timestamp(),
         last_comment_time: p.last_comment_time.timestamp(),
         allow_search: p.allow_search,
-        author_title: (!p.author_title.is_empty()).then(|| p.author_title.to_string()),
+        author_title: (!p.author_title.is_empty()).then(|| p.author_title.clone()),
         is_tmp: p.is_tmp,
         is_reported: user.is_admin.then(|| p.is_reported),
-        comments: if p.n_comments > 50 {
+        comments: OptionFuture::from(
-            None
+            comments
-        } else {
+                .map(|cs| async move { c2output(p, &cs, user, &cached_block_dict, rconn).await }),
-            // 单个洞还有查询评论的接口，这里挂了不用报错
-            Some(
-                c2output(
-                    p,
-                    &p.get_comments(db, rconn).await.unwrap_or(vec![]),
-                    user,
-                    rconn,
         )
         .await,
-            )
-        },
         can_del: p.check_permission(user, "wd").is_ok(),
-        attention: Attention::init(&user.namehash, &rconn)
+        attention: Attention::init(&user.namehash, &rconn).has(p.id).await?,
-            .has(p.id)
-            .await
-            .unwrap_or_default(),
         hot_score: user.is_admin.then(|| p.hot_score),
         is_blocked: is_blocked,
         blocked_count: if user.is_admin {
-            BlockCounter::get_count(rconn, &p.author_hash).await.ok()
+            BlockCounter::get_count(rconn, &p.author_hash).await?
         } else {
             None
         },
@@ -116,7 +118,7 @@ async fn p2output(p: &Post, user: &CurrentUser, db: &Db, rconn: &RdsConn) -> Pos
         likenum: p.n_attentions,
         reply: p.n_comments,
         blocked: is_blocked,
-    }
+    })
 }
 
 pub async fn ps2outputs(
@@ -124,10 +126,10 @@ pub async fn ps2outputs(
     user: &CurrentUser,
     db: &Db,
     rconn: &RdsConn,
-) -> Vec<PostOutput> {
+) -> API<Vec<PostOutput>> {
-    future::join_all(
+    future::try_join_all(
         ps.iter()
-            .map(|p| async { p2output(p, &user, &db, &rconn).await }),
+            .map(|p| async { Ok(p2output(p, &user, &db, &rconn).await?) }),
     )
     .await
 }
@@ -137,7 +139,7 @@ pub async fn get_one(pid: i32, user: CurrentUser, db: Db, rconn: RdsConn) -> Jso
     let p = Post::get(&db, &rconn, pid).await?;
     p.check_permission(&user, "ro")?;
     Ok(json!({
-        "data": p2output(&p, &user,&db, &rconn).await,
+        "data": p2output(&p, &user,&db, &rconn).await?,
         "code": 0,
     }))
 }
@@ -155,11 +157,12 @@ pub async fn get_list(
     let page_size = 25;
     let start = (page - 1) * page_size;
     let ps = Post::gets_by_page(&db, &rconn, order_mode, start.into(), page_size.into()).await?;
-    let ps_data = ps2outputs(&ps, &user, &db, &rconn).await;
+    let ps_data = ps2outputs(&ps, &user, &db, &rconn).await?;
     Ok(json!({
         "data": ps_data,
         "count": ps_data.len(),
-        "custom_title": CustomTitle::get(&rconn, &user.namehash).await?,
+        "custom_title": user.custom_title,
+        "auto_block_rank": user.auto_block_rank,
         "code": 0
     }))
 }
@@ -177,12 +180,7 @@ pub async fn publish_post(
             content: poi.text.to_string(),
             cw: poi.cw.to_string(),
             author_hash: user.namehash.to_string(),
-            author_title: (if poi.use_title.is_some() {
+            author_title: poi.use_title.map(|_| user.custom_title).unwrap_or_default(),
-                CustomTitle::get(&rconn, &user.namehash).await?
-            } else {
-                None
-            })
-            .unwrap_or_default(),
             is_tmp: user.id.is_none(),
             n_attentions: 1,
             allow_search: poi.allow_search.is_some(),
@@ -213,7 +211,7 @@ pub async fn edit_cw(cwi: Form<CwInput>, user: CurrentUser, db: Db, rconn: RdsCo
 pub async fn get_multi(pids: Vec<i32>, user: CurrentUser, db: Db, rconn: RdsConn) -> JsonAPI {
     user.id.ok_or_else(|| YouAreTmp)?;
     let ps = Post::get_multi(&db, &rconn, &pids).await?;
-    let ps_data = ps2outputs(&ps, &user, &db, &rconn).await;
+    let ps_data = ps2outputs(&ps, &user, &db, &rconn).await?;
 
     Ok(json!({
         "code": 0,

src/api/search.rs

@@ -36,7 +36,7 @@ pub async fn search(
         )
         .await?
     };
-    let ps_data = ps2outputs(&ps, &user, &db, &rconn).await;
+    let ps_data = ps2outputs(&ps, &user, &db, &rconn).await?;
     Ok(json!({
         "data": ps_data,
         "count": ps_data.len(),

src/cache.rs

@@ -1,10 +1,13 @@
+use crate::api::CurrentUser;
 use crate::models::{Comment, Post, User};
 use crate::rds_conn::RdsConn;
-use crate::rds_models::init;
+use crate::rds_models::{init, BlockedUsers};
 use rand::Rng;
-use redis::AsyncCommands;
+use redis::{AsyncCommands, RedisError, RedisResult};
 use rocket::serde::json::serde_json;
 // can use rocket::serde::json::to_string in master version
+use rocket::futures::future;
+use std::collections::HashMap;
 
 const INSTANCE_EXPIRE_TIME: usize = 60 * 60;
 
@@ -321,3 +324,51 @@ impl UserCache {
         }
     }
 }
+
+pub struct BlockDictCache {
+    key: String,
+    rconn: RdsConn,
+}
+
+impl BlockDictCache {
+    // namehash, pid
+    init!(&str, i32, "hole_v2:cache:block_dict:{}:{}");
+
+    pub async fn get_or_create(
+        &mut self,
+        user: &CurrentUser,
+        hash_list: &Vec<&String>,
+    ) -> RedisResult<HashMap<String, bool>> {
+        let mut block_dict = self
+            .rconn
+            .hgetall::<&String, HashMap<String, bool>>(&self.key)
+            .await?;
+
+        //dbg!(&self.key, &block_dict);
+
+        let missing: Vec<(String, bool)> =
+            future::try_join_all(hash_list.iter().filter_map(|hash| {
+                (!block_dict.contains_key(&hash.to_string())).then(|| async {
+                    Ok::<(String, bool), RedisError>((
+                        hash.to_string(),
+                        BlockedUsers::check_if_block(&self.rconn, user, hash).await?,
+                    ))
+                })
+            }))
+            .await?;
+
+        if !missing.is_empty() {
+            self.rconn.hset_multiple(&self.key, &missing).await?;
+            self.rconn.expire(&self.key, INSTANCE_EXPIRE_TIME).await?;
+            block_dict.extend(missing.into_iter());
+        }
+
+        //dbg!(&block_dict);
+
+        Ok(block_dict)
+    }
+
+    pub async fn clear(&mut self) -> RedisResult<()> {
+        self.rconn.del(&self.key).await
+    }
+}

src/main.rs

@@ -26,6 +26,7 @@ use db_conn::{establish_connection, Conn, Db};
 use diesel::Connection;
 use random_hasher::RandomHasher;
 use rds_conn::{init_rds_client, RdsConn};
+use rds_models::clear_outdate_redis_data;
 use std::env;
 use tokio::time::{sleep, Duration};
 
@@ -68,6 +69,7 @@ async fn main() -> Result<(), rocket::Error> {
                 api::operation::report,
                 api::operation::set_title,
                 api::operation::block,
+                api::operation::set_auto_block,
                 api::vote::vote,
                 api::upload::ipfs_upload,
             ],
@@ -105,8 +107,3 @@ fn init_database() {
     let conn = Conn::establish(&database_url).unwrap();
     embedded_migrations::run(&conn).unwrap();
 }
-
-async fn clear_outdate_redis_data(rconn: &RdsConn) {
-    rds_models::BannedUsers::clear(&rconn).await.unwrap();
-    rds_models::CustomTitle::clear(&rconn).await.unwrap();
-}

src/rds_models.rs

@@ -1,3 +1,4 @@
+use crate::api::CurrentUser;
 use crate::rds_conn::RdsConn;
 use chrono::{offset::Local, DateTime};
 use redis::{AsyncCommands, RedisResult};
@@ -40,7 +41,7 @@ macro_rules! has {
 
 macro_rules! add {
     ($vtype:ty) => {
-        pub async fn add(&mut self, v: $vtype) -> RedisResult<()> {
+        pub async fn add(&mut self, v: $vtype) -> RedisResult<usize> {
             self.rconn.sadd(&self.key, v).await
         }
     };
@@ -49,11 +50,10 @@ macro_rules! add {
 const KEY_SYSTEMLOG: &str = "hole_v2:systemlog_list";
 const KEY_BANNED_USERS: &str = "hole_v2:banned_user_hash_list";
 const KEY_BLOCKED_COUNTER: &str = "hole_v2:blocked_counter";
-const KEY_DANGEROUS_USERS: &str = "hole_thu:dangerous_users"; //兼容一下旧版
 const KEY_CUSTOM_TITLE: &str = "hole_v2:title";
+const KEY_AUTO_BLOCK_RANK: &str = "hole_v2:auto_block_rank"; // rank * 5: 自动过滤的拉黑数阈值
 
 const SYSTEMLOG_MAX_LEN: isize = 1000;
-pub const BLOCK_THRESHOLD: i32 = 10;
 
 pub struct Attention {
     key: String,
@@ -159,47 +159,31 @@ impl BlockedUsers {
 
     has!(&str);
 
-    pub async fn check_blocked(
+    pub async fn check_if_block(
         rconn: &RdsConn,
-        viewer_id: Option<i32>,
+        user: &CurrentUser,
-        viewer_hash: &str,
+        hash: &str,
-        author_hash: &str,
     ) -> RedisResult<bool> {
-        Ok(match viewer_id {
+        Ok(match user.id {
-            Some(id) => Self::init(id, rconn).has(author_hash).await?,
+            Some(id) => BlockedUsers::init(id, rconn).has(hash).await?,
             None => false,
-        } || (DangerousUser::has(rconn, author_hash).await?
+        } || BlockCounter::get_count(rconn, hash).await?.unwrap_or(0)
-            && !DangerousUser::has(rconn, viewer_hash).await?))
+            >= i32::from(user.auto_block_rank) * 5)
     }
 }
 
 pub struct BlockCounter;
 
 impl BlockCounter {
-    pub async fn count_incr(rconn: &RdsConn, namehash: &str) -> RedisResult<i32> {
+    pub async fn count_incr(rconn: &RdsConn, namehash: &str) -> RedisResult<usize> {
         rconn.clone().hincr(KEY_BLOCKED_COUNTER, namehash, 1).await
     }
 
-    pub async fn get_count(rconn: &RdsConn, namehash: &str) -> RedisResult<i32> {
+    pub async fn get_count(rconn: &RdsConn, namehash: &str) -> RedisResult<Option<i32>> {
         rconn.clone().hget(KEY_BLOCKED_COUNTER, namehash).await
     }
 }
 
-pub struct DangerousUser;
-
-impl DangerousUser {
-    pub async fn add(rconn: &RdsConn, namehash: &str) -> RedisResult<()> {
-        rconn
-            .clone()
-            .sadd::<&str, &str, ()>(KEY_DANGEROUS_USERS, namehash)
-            .await
-    }
-
-    pub async fn has(rconn: &RdsConn, namehash: &str) -> RedisResult<bool> {
-        rconn.clone().sismember(KEY_DANGEROUS_USERS, namehash).await
-    }
-}
-
 pub struct CustomTitle;
 
 impl CustomTitle {
@@ -224,6 +208,26 @@ impl CustomTitle {
     }
 }
 
+pub struct AutoBlockRank;
+
+impl AutoBlockRank {
+    pub async fn set(rconn: &RdsConn, namehash: &str, rank: u8) -> RedisResult<usize> {
+        rconn
+            .clone()
+            .hset(KEY_AUTO_BLOCK_RANK, namehash, rank)
+            .await
+    }
+
+    pub async fn get(rconn: &RdsConn, namehash: &str) -> RedisResult<u8> {
+        let rank: Option<u8> = rconn.clone().hget(KEY_AUTO_BLOCK_RANK, namehash).await?;
+        Ok(rank.unwrap_or(2))
+    }
+
+    pub async fn clear(rconn: &RdsConn) -> RedisResult<()> {
+        rconn.clone().del(KEY_AUTO_BLOCK_RANK).await
+    }
+}
+
 pub struct PollOption {
     key: String,
     rconn: RdsConn,
@@ -259,4 +263,10 @@ impl PollVote {
     }
 }
 
+pub async fn clear_outdate_redis_data(rconn: &RdsConn) {
+    BannedUsers::clear(&rconn).await.unwrap();
+    CustomTitle::clear(&rconn).await.unwrap();
+    AutoBlockRank::clear(&rconn).await.unwrap();
+}
+
 pub(crate) use init;