Browse Source

feat: configurable auto block

master v1.2.0
hole-thu 3 years ago
parent
commit
47a6e51a2a
  1. 2
      README.md
  2. 2
      src/api/attention.rs
  3. 18
      src/api/comment.rs
  4. 12
      src/api/mod.rs
  5. 48
      src/api/operation.rs
  6. 84
      src/api/post.rs
  7. 2
      src/api/search.rs
  8. 55
      src/cache.rs
  9. 7
      src/main.rs
  10. 66
      src/rds_models.rs

2
README.md

@ -1,4 +1,4 @@
# hole-backend-rust v1.1.0
# hole-backend-rust v1.2.0
## 部署

2
src/api/attention.rs

@ -67,7 +67,7 @@ pub async fn get_attention(user: CurrentUser, db: Db, rconn: RdsConn) -> JsonAPI
let mut ids = Attention::init(&user.namehash, &rconn).all().await?;
ids.sort_by_key(|x| -x);
let ps = Post::get_multi(&db, &rconn, &ids).await?;
let ps_data = ps2outputs(&ps, &user, &db, &rconn).await;
let ps_data = ps2outputs(&ps, &user, &db, &rconn).await?;
code0!(ps_data)
}

18
src/api/comment.rs

@ -1,4 +1,5 @@
use crate::api::{APIError, CurrentUser, JsonAPI, PolicyError::*, UGC};
use crate::cache::BlockDictCache;
use crate::db_conn::Db;
use crate::libs::diesel_logger::LoggingConnection;
use crate::models::*;
@ -41,6 +42,7 @@ pub async fn c2output<'r>(
p: &'r Post,
cs: &Vec<Comment>,
user: &CurrentUser,
cached_block_dict: &HashMap<String, bool>,
rconn: &RdsConn,
) -> Vec<CommentOutput> {
let mut hash2id = HashMap::<&String, i32>::from([(&p.author_hash, 0)]);
@ -56,10 +58,7 @@ pub async fn c2output<'r>(
if c.is_deleted {
None
} else {
let is_blocked =
BlockedUsers::check_blocked(rconn, user.id, &user.namehash, &c.author_hash)
.await
.unwrap_or_default();
let is_blocked = cached_block_dict[&c.author_hash];
let can_view = user.is_admin
|| (!is_blocked && user.id.is_some() || user.namehash.eq(&c.author_hash));
Some(CommentOutput {
@ -72,7 +71,10 @@ pub async fn c2output<'r>(
create_time: c.create_time.timestamp(),
is_blocked: is_blocked,
blocked_count: if user.is_admin {
BlockCounter::get_count(rconn, &c.author_hash).await.ok()
BlockCounter::get_count(rconn, &c.author_hash)
.await
.ok()
.flatten()
} else {
None
},
@ -94,7 +96,11 @@ pub async fn get_comment(pid: i32, user: CurrentUser, db: Db, rconn: RdsConn) ->
return Err(APIError::PcError(IsDeleted));
}
let cs = p.get_comments(&db, &rconn).await?;
let data = c2output(&p, &cs, &user, &rconn).await;
let hash_list = cs.iter().map(|c| &c.author_hash).collect();
let cached_block_dict = BlockDictCache::init(&user.namehash, p.id, &rconn)
.get_or_create(&user, &hash_list)
.await?;
let data = c2output(&p, &cs, &user, &cached_block_dict, &rconn).await;
Ok(json!({
"code": 0,

12
src/api/mod.rs

@ -3,7 +3,7 @@ use crate::libs::diesel_logger::LoggingConnection;
use crate::models::*;
use crate::random_hasher::RandomHasher;
use crate::rds_conn::RdsConn;
use crate::rds_models::BannedUsers;
use crate::rds_models::*;
use crate::schema;
use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
use rocket::http::Status;
@ -52,10 +52,11 @@ pub fn catch_403_error() -> &'static str {
}
pub struct CurrentUser {
id: Option<i32>, // tmp user has no id, only for block
pub id: Option<i32>, // tmp user has no id, only for block
namehash: String,
is_admin: bool,
custom_title: String,
pub auto_block_rank: u8,
}
#[rocket::async_trait]
@ -91,7 +92,12 @@ impl<'r> FromRequest<'r> for CurrentUser {
} else {
Outcome::Success(CurrentUser {
id: id,
custom_title: format!("title todo: {}", &nh),
custom_title: CustomTitle::get(&rconn, &nh)
.await
.ok()
.flatten()
.unwrap_or_default(),
auto_block_rank: AutoBlockRank::get(&rconn, &nh).await.unwrap_or(2),
namehash: nh,
is_admin: is_admin,
})

48
src/api/operation.rs

@ -1,4 +1,5 @@
use crate::api::{CurrentUser, JsonAPI, PolicyError::*, UGC};
use crate::api::{APIError, CurrentUser, JsonAPI, PolicyError::*, UGC};
use crate::cache::*;
use crate::db_conn::Db;
use crate::libs::diesel_logger::LoggingConnection;
use crate::models::*;
@ -72,7 +73,6 @@ pub async fn delete(di: Form<DeleteInput>, user: CurrentUser, db: Db, rconn: Rds
.create(&rconn)
.await?;
BannedUsers::add(&rconn, &author_hash).await?;
DangerousUser::add(&rconn, &author_hash).await?;
}
}
@ -128,28 +128,39 @@ pub async fn block(bi: Form<BlockInput>, user: CurrentUser, db: Db, rconn: RdsCo
let mut blk = BlockedUsers::init(user.id.ok_or_else(|| NotAllowed)?, &rconn);
let pid;
let nh_to_block = match bi.content_type.as_str() {
"post" => Post::get(&db, &rconn, bi.id).await?.author_hash,
"comment" => Comment::get(&db, bi.id).await?.author_hash,
_ => Err(NotAllowed)?,
"post" => {
let p = Post::get(&db, &rconn, bi.id).await?;
pid = p.id;
p.author_hash
}
"comment" => {
let c = Comment::get(&db, bi.id).await?;
pid = c.post_id;
c.author_hash
}
_ => return Err(APIError::PcError(NotAllowed)),
};
if nh_to_block.eq(&user.namehash) {
Err(NotAllowed)?;
}
blk.add(&nh_to_block).await?;
let curr = BlockCounter::count_incr(&rconn, &nh_to_block).await?;
let curr = if blk.add(&nh_to_block).await? > 0 {
BlockCounter::count_incr(&rconn, &nh_to_block).await?
} else {
114514
};
if curr >= BLOCK_THRESHOLD || user.is_admin {
DangerousUser::add(&rconn, &nh_to_block).await?;
}
BlockDictCache::init(&user.namehash, pid, &rconn)
.clear()
.await?;
Ok(json!({
"code": 0,
"data": {
"curr": curr,
"threshold": BLOCK_THRESHOLD,
},
}))
}
@ -168,3 +179,18 @@ pub async fn set_title(ti: Form<TitleInput>, user: CurrentUser, rconn: RdsConn)
Err(TitleUsed)?
}
}
#[derive(FromForm)]
pub struct AutoBlockInput {
rank: u8,
}
#[post("/auto_block", data = "<ai>")]
pub async fn set_auto_block(
ai: Form<AutoBlockInput>,
user: CurrentUser,
rconn: RdsConn,
) -> JsonAPI {
AutoBlockRank::set(&rconn, &user.namehash, ai.rank).await?;
code0!()
}

84
src/api/post.rs

@ -1,6 +1,7 @@
use crate::api::comment::{c2output, CommentOutput};
use crate::api::vote::get_poll_dict;
use crate::api::{CurrentUser, JsonAPI, PolicyError::*, UGC};
use crate::api::{CurrentUser, JsonAPI, PolicyError::*, API, UGC};
use crate::cache::*;
use crate::db_conn::Db;
use crate::libs::diesel_logger::LoggingConnection;
use crate::models::*;
@ -9,7 +10,7 @@ use crate::rds_models::*;
use crate::schema;
use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
use rocket::form::Form;
use rocket::futures::future;
use rocket::futures::future::{self, OptionFuture};
use rocket::serde::{
json::{json, Value},
Serialize,
@ -62,47 +63,48 @@ pub struct CwInput {
cw: String,
}
async fn p2output(p: &Post, user: &CurrentUser, db: &Db, rconn: &RdsConn) -> PostOutput {
let is_blocked = BlockedUsers::check_blocked(rconn, user.id, &user.namehash, &p.author_hash)
.await
.unwrap_or_default();
async fn p2output(p: &Post, user: &CurrentUser, db: &Db, rconn: &RdsConn) -> API<PostOutput> {
let comments: Option<Vec<Comment>> = if p.n_comments < 5 {
Some(p.get_comments(db, rconn).await?)
} else {
None
};
let hash_list = comments
.iter()
.flatten()
.map(|c| &c.author_hash)
.chain(std::iter::once(&p.author_hash))
.collect();
//dbg!(&hash_list);
let cached_block_dict = BlockDictCache::init(&user.namehash, p.id, rconn)
.get_or_create(&user, &hash_list)
.await?;
let is_blocked = cached_block_dict[&p.author_hash];
let can_view =
user.is_admin || (!is_blocked && user.id.is_some() || user.namehash.eq(&p.author_hash));
PostOutput {
Ok(PostOutput {
pid: p.id,
text: (if can_view { &p.content } else { "" }).to_string(),
cw: (!p.cw.is_empty()).then(|| p.cw.to_string()),
text: can_view.then(|| p.content.clone()).unwrap_or_default(),
cw: (!p.cw.is_empty()).then(|| p.cw.clone()),
n_attentions: p.n_attentions,
n_comments: p.n_comments,
create_time: p.create_time.timestamp(),
last_comment_time: p.last_comment_time.timestamp(),
allow_search: p.allow_search,
author_title: (!p.author_title.is_empty()).then(|| p.author_title.to_string()),
author_title: (!p.author_title.is_empty()).then(|| p.author_title.clone()),
is_tmp: p.is_tmp,
is_reported: user.is_admin.then(|| p.is_reported),
comments: if p.n_comments > 50 {
None
} else {
// 单个洞还有查询评论的接口,这里挂了不用报错
Some(
c2output(
p,
&p.get_comments(db, rconn).await.unwrap_or(vec![]),
user,
rconn,
)
.await,
)
},
comments: OptionFuture::from(
comments
.map(|cs| async move { c2output(p, &cs, user, &cached_block_dict, rconn).await }),
)
.await,
can_del: p.check_permission(user, "wd").is_ok(),
attention: Attention::init(&user.namehash, &rconn)
.has(p.id)
.await
.unwrap_or_default(),
attention: Attention::init(&user.namehash, &rconn).has(p.id).await?,
hot_score: user.is_admin.then(|| p.hot_score),
is_blocked: is_blocked,
blocked_count: if user.is_admin {
BlockCounter::get_count(rconn, &p.author_hash).await.ok()
BlockCounter::get_count(rconn, &p.author_hash).await?
} else {
None
},
@ -116,7 +118,7 @@ async fn p2output(p: &Post, user: &CurrentUser, db: &Db, rconn: &RdsConn) -> Pos
likenum: p.n_attentions,
reply: p.n_comments,
blocked: is_blocked,
}
})
}
pub async fn ps2outputs(
@ -124,10 +126,10 @@ pub async fn ps2outputs(
user: &CurrentUser,
db: &Db,
rconn: &RdsConn,
) -> Vec<PostOutput> {
future::join_all(
) -> API<Vec<PostOutput>> {
future::try_join_all(
ps.iter()
.map(|p| async { p2output(p, &user, &db, &rconn).await }),
.map(|p| async { Ok(p2output(p, &user, &db, &rconn).await?) }),
)
.await
}
@ -137,7 +139,7 @@ pub async fn get_one(pid: i32, user: CurrentUser, db: Db, rconn: RdsConn) -> Jso
let p = Post::get(&db, &rconn, pid).await?;
p.check_permission(&user, "ro")?;
Ok(json!({
"data": p2output(&p, &user,&db, &rconn).await,
"data": p2output(&p, &user,&db, &rconn).await?,
"code": 0,
}))
}
@ -155,11 +157,12 @@ pub async fn get_list(
let page_size = 25;
let start = (page - 1) * page_size;
let ps = Post::gets_by_page(&db, &rconn, order_mode, start.into(), page_size.into()).await?;
let ps_data = ps2outputs(&ps, &user, &db, &rconn).await;
let ps_data = ps2outputs(&ps, &user, &db, &rconn).await?;
Ok(json!({
"data": ps_data,
"count": ps_data.len(),
"custom_title": CustomTitle::get(&rconn, &user.namehash).await?,
"custom_title": user.custom_title,
"auto_block_rank": user.auto_block_rank,
"code": 0
}))
}
@ -177,12 +180,7 @@ pub async fn publish_post(
content: poi.text.to_string(),
cw: poi.cw.to_string(),
author_hash: user.namehash.to_string(),
author_title: (if poi.use_title.is_some() {
CustomTitle::get(&rconn, &user.namehash).await?
} else {
None
})
.unwrap_or_default(),
author_title: poi.use_title.map(|_| user.custom_title).unwrap_or_default(),
is_tmp: user.id.is_none(),
n_attentions: 1,
allow_search: poi.allow_search.is_some(),
@ -213,7 +211,7 @@ pub async fn edit_cw(cwi: Form<CwInput>, user: CurrentUser, db: Db, rconn: RdsCo
pub async fn get_multi(pids: Vec<i32>, user: CurrentUser, db: Db, rconn: RdsConn) -> JsonAPI {
user.id.ok_or_else(|| YouAreTmp)?;
let ps = Post::get_multi(&db, &rconn, &pids).await?;
let ps_data = ps2outputs(&ps, &user, &db, &rconn).await;
let ps_data = ps2outputs(&ps, &user, &db, &rconn).await?;
Ok(json!({
"code": 0,

2
src/api/search.rs

@ -36,7 +36,7 @@ pub async fn search(
)
.await?
};
let ps_data = ps2outputs(&ps, &user, &db, &rconn).await;
let ps_data = ps2outputs(&ps, &user, &db, &rconn).await?;
Ok(json!({
"data": ps_data,
"count": ps_data.len(),

55
src/cache.rs

@ -1,10 +1,13 @@
use crate::api::CurrentUser;
use crate::models::{Comment, Post, User};
use crate::rds_conn::RdsConn;
use crate::rds_models::init;
use crate::rds_models::{init, BlockedUsers};
use rand::Rng;
use redis::AsyncCommands;
use redis::{AsyncCommands, RedisError, RedisResult};
use rocket::serde::json::serde_json;
// can use rocket::serde::json::to_string in master version
use rocket::futures::future;
use std::collections::HashMap;
const INSTANCE_EXPIRE_TIME: usize = 60 * 60;
@ -321,3 +324,51 @@ impl UserCache {
}
}
}
pub struct BlockDictCache {
key: String,
rconn: RdsConn,
}
impl BlockDictCache {
// namehash, pid
init!(&str, i32, "hole_v2:cache:block_dict:{}:{}");
pub async fn get_or_create(
&mut self,
user: &CurrentUser,
hash_list: &Vec<&String>,
) -> RedisResult<HashMap<String, bool>> {
let mut block_dict = self
.rconn
.hgetall::<&String, HashMap<String, bool>>(&self.key)
.await?;
//dbg!(&self.key, &block_dict);
let missing: Vec<(String, bool)> =
future::try_join_all(hash_list.iter().filter_map(|hash| {
(!block_dict.contains_key(&hash.to_string())).then(|| async {
Ok::<(String, bool), RedisError>((
hash.to_string(),
BlockedUsers::check_if_block(&self.rconn, user, hash).await?,
))
})
}))
.await?;
if !missing.is_empty() {
self.rconn.hset_multiple(&self.key, &missing).await?;
self.rconn.expire(&self.key, INSTANCE_EXPIRE_TIME).await?;
block_dict.extend(missing.into_iter());
}
//dbg!(&block_dict);
Ok(block_dict)
}
pub async fn clear(&mut self) -> RedisResult<()> {
self.rconn.del(&self.key).await
}
}

7
src/main.rs

@ -26,6 +26,7 @@ use db_conn::{establish_connection, Conn, Db};
use diesel::Connection;
use random_hasher::RandomHasher;
use rds_conn::{init_rds_client, RdsConn};
use rds_models::clear_outdate_redis_data;
use std::env;
use tokio::time::{sleep, Duration};
@ -68,6 +69,7 @@ async fn main() -> Result<(), rocket::Error> {
api::operation::report,
api::operation::set_title,
api::operation::block,
api::operation::set_auto_block,
api::vote::vote,
api::upload::ipfs_upload,
],
@ -105,8 +107,3 @@ fn init_database() {
let conn = Conn::establish(&database_url).unwrap();
embedded_migrations::run(&conn).unwrap();
}
async fn clear_outdate_redis_data(rconn: &RdsConn) {
rds_models::BannedUsers::clear(&rconn).await.unwrap();
rds_models::CustomTitle::clear(&rconn).await.unwrap();
}

66
src/rds_models.rs

@ -1,3 +1,4 @@
use crate::api::CurrentUser;
use crate::rds_conn::RdsConn;
use chrono::{offset::Local, DateTime};
use redis::{AsyncCommands, RedisResult};
@ -40,7 +41,7 @@ macro_rules! has {
macro_rules! add {
($vtype:ty) => {
pub async fn add(&mut self, v: $vtype) -> RedisResult<()> {
pub async fn add(&mut self, v: $vtype) -> RedisResult<usize> {
self.rconn.sadd(&self.key, v).await
}
};
@ -49,11 +50,10 @@ macro_rules! add {
const KEY_SYSTEMLOG: &str = "hole_v2:systemlog_list";
const KEY_BANNED_USERS: &str = "hole_v2:banned_user_hash_list";
const KEY_BLOCKED_COUNTER: &str = "hole_v2:blocked_counter";
const KEY_DANGEROUS_USERS: &str = "hole_thu:dangerous_users"; //兼容一下旧版
const KEY_CUSTOM_TITLE: &str = "hole_v2:title";
const KEY_AUTO_BLOCK_RANK: &str = "hole_v2:auto_block_rank"; // rank * 5: 自动过滤的拉黑数阈值
const SYSTEMLOG_MAX_LEN: isize = 1000;
pub const BLOCK_THRESHOLD: i32 = 10;
pub struct Attention {
key: String,
@ -159,47 +159,31 @@ impl BlockedUsers {
has!(&str);
pub async fn check_blocked(
pub async fn check_if_block(
rconn: &RdsConn,
viewer_id: Option<i32>,
viewer_hash: &str,
author_hash: &str,
user: &CurrentUser,
hash: &str,
) -> RedisResult<bool> {
Ok(match viewer_id {
Some(id) => Self::init(id, rconn).has(author_hash).await?,
Ok(match user.id {
Some(id) => BlockedUsers::init(id, rconn).has(hash).await?,
None => false,
} || (DangerousUser::has(rconn, author_hash).await?
&& !DangerousUser::has(rconn, viewer_hash).await?))
} || BlockCounter::get_count(rconn, hash).await?.unwrap_or(0)
>= i32::from(user.auto_block_rank) * 5)
}
}
pub struct BlockCounter;
impl BlockCounter {
pub async fn count_incr(rconn: &RdsConn, namehash: &str) -> RedisResult<i32> {
pub async fn count_incr(rconn: &RdsConn, namehash: &str) -> RedisResult<usize> {
rconn.clone().hincr(KEY_BLOCKED_COUNTER, namehash, 1).await
}
pub async fn get_count(rconn: &RdsConn, namehash: &str) -> RedisResult<i32> {
pub async fn get_count(rconn: &RdsConn, namehash: &str) -> RedisResult<Option<i32>> {
rconn.clone().hget(KEY_BLOCKED_COUNTER, namehash).await
}
}
pub struct DangerousUser;
impl DangerousUser {
pub async fn add(rconn: &RdsConn, namehash: &str) -> RedisResult<()> {
rconn
.clone()
.sadd::<&str, &str, ()>(KEY_DANGEROUS_USERS, namehash)
.await
}
pub async fn has(rconn: &RdsConn, namehash: &str) -> RedisResult<bool> {
rconn.clone().sismember(KEY_DANGEROUS_USERS, namehash).await
}
}
pub struct CustomTitle;
impl CustomTitle {
@ -224,6 +208,26 @@ impl CustomTitle {
}
}
pub struct AutoBlockRank;
impl AutoBlockRank {
pub async fn set(rconn: &RdsConn, namehash: &str, rank: u8) -> RedisResult<usize> {
rconn
.clone()
.hset(KEY_AUTO_BLOCK_RANK, namehash, rank)
.await
}
pub async fn get(rconn: &RdsConn, namehash: &str) -> RedisResult<u8> {
let rank: Option<u8> = rconn.clone().hget(KEY_AUTO_BLOCK_RANK, namehash).await?;
Ok(rank.unwrap_or(2))
}
pub async fn clear(rconn: &RdsConn) -> RedisResult<()> {
rconn.clone().del(KEY_AUTO_BLOCK_RANK).await
}
}
pub struct PollOption {
key: String,
rconn: RdsConn,
@ -259,4 +263,10 @@ impl PollVote {
}
}
pub async fn clear_outdate_redis_data(rconn: &RdsConn) {
BannedUsers::clear(&rconn).await.unwrap();
CustomTitle::clear(&rconn).await.unwrap();
AutoBlockRank::clear(&rconn).await.unwrap();
}
pub(crate) use init;

Loading…
Cancel
Save