From c9659acd6ecaabe0542a3378cbd35d7b9deed33d Mon Sep 17 00:00:00 2001 From: hole-thu Date: Tue, 22 Mar 2022 02:57:22 +0800 Subject: [PATCH] use postgresql --- Cargo.toml | 4 +- README.md | 23 +++++++++ migrations/postgres/.gitkeep | 0 .../down.sql | 6 +++ .../up.sql | 36 +++++++++++++ .../2022-03-21-155550_create_posts/down.sql | 2 + .../2022-03-21-155550_create_posts/up.sql | 23 +++++++++ .../2022-03-21-162914_create_users/down.sql | 2 + .../2022-03-21-162914_create_users/up.sql | 7 +++ .../down.sql | 2 + .../2022-03-21-164718_create_comments/up.sql | 14 ++++++ .../2022-03-11-065048_create_posts/down.sql | 0 .../2022-03-11-065048_create_posts/up.sql | 2 +- .../2022-03-15-061041_create_users/down.sql | 0 .../2022-03-15-061041_create_users/up.sql | 0 .../down.sql | 0 .../2022-03-15-104943_create_comments/up.sql | 0 src/api/operation.rs | 3 +- src/api/post.rs | 3 +- src/db_conn.rs | 4 +- src/models.rs | 16 ++++-- src/schema.rs | 29 +++++------ tools/migdb.py | 50 +++++++++++++++---- 23 files changed, 188 insertions(+), 38 deletions(-) create mode 100644 migrations/postgres/.gitkeep create mode 100644 migrations/postgres/00000000000000_diesel_initial_setup/down.sql create mode 100644 migrations/postgres/00000000000000_diesel_initial_setup/up.sql create mode 100644 migrations/postgres/2022-03-21-155550_create_posts/down.sql create mode 100644 migrations/postgres/2022-03-21-155550_create_posts/up.sql create mode 100644 migrations/postgres/2022-03-21-162914_create_users/down.sql create mode 100644 migrations/postgres/2022-03-21-162914_create_users/up.sql create mode 100644 migrations/postgres/2022-03-21-164718_create_comments/down.sql create mode 100644 migrations/postgres/2022-03-21-164718_create_comments/up.sql rename migrations/{ => sqlite3}/2022-03-11-065048_create_posts/down.sql (100%) rename migrations/{ => sqlite3}/2022-03-11-065048_create_posts/up.sql (94%) rename migrations/{ => sqlite3}/2022-03-15-061041_create_users/down.sql (100%) rename migrations/{ => sqlite3}/2022-03-15-061041_create_users/up.sql (100%) rename migrations/{ => sqlite3}/2022-03-15-104943_create_comments/down.sql (100%) rename migrations/{ => sqlite3}/2022-03-15-104943_create_comments/up.sql (100%) diff --git a/Cargo.toml b/Cargo.toml index 1504024..329403b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ license = "AGPL-3.0" [dependencies] rocket = { version = "0.5.0-rc.1", features = ["json"] } -diesel = { version = "1.4.8", features = ["sqlite", "chrono", "r2d2"] } +diesel = { version = "1.4.8", features = ["postgres", "chrono"] } redis = { version="0.21.5", features = ["aio", "async-std-comp"] } chrono = { version="0.*", features =["serde"] } rand = "0.*" @@ -17,4 +17,4 @@ sha2 = "0.*" [dependencies.rocket_sync_db_pools] version = "0.1.0-rc.1" -features = ["diesel_sqlite_pool"] +features = ["diesel_postgres_pool"] diff --git a/README.md b/README.md index 52d6661..7e7b98a 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,25 @@ # hole-backend-rust + +## 部署 + +### prepare database + +```sql +CREATE USER hole CREATEDB; +ALTER USER hole WITH PASSWORD "hole_pass"; +``` + +``` +$ diesel setup +``` + +```sql +\c hole_v2 +CREATE EXTENSION pg_trgm; +``` + +``` +$ diesel run +$ python3 tools/migdb.py +``` diff --git a/migrations/postgres/.gitkeep b/migrations/postgres/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/migrations/postgres/00000000000000_diesel_initial_setup/down.sql b/migrations/postgres/00000000000000_diesel_initial_setup/down.sql new file mode 100644 index 0000000..a9f5260 --- /dev/null +++ b/migrations/postgres/00000000000000_diesel_initial_setup/down.sql @@ -0,0 +1,6 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + +DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass); +DROP FUNCTION IF EXISTS diesel_set_updated_at(); diff --git a/migrations/postgres/00000000000000_diesel_initial_setup/up.sql b/migrations/postgres/00000000000000_diesel_initial_setup/up.sql new file mode 100644 index 0000000..d68895b --- /dev/null +++ b/migrations/postgres/00000000000000_diesel_initial_setup/up.sql @@ -0,0 +1,36 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + + + + +-- Sets up a trigger for the given table to automatically set a column called +-- `updated_at` whenever the row is modified (unless `updated_at` was included +-- in the modified columns) +-- +-- # Example +-- +-- ```sql +-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW()); +-- +-- SELECT diesel_manage_updated_at('users'); +-- ``` +CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$ +BEGIN + EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s + FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl); +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$ +BEGIN + IF ( + NEW IS DISTINCT FROM OLD AND + NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at + ) THEN + NEW.updated_at := current_timestamp; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; diff --git a/migrations/postgres/2022-03-21-155550_create_posts/down.sql b/migrations/postgres/2022-03-21-155550_create_posts/down.sql new file mode 100644 index 0000000..56ed16e --- /dev/null +++ b/migrations/postgres/2022-03-21-155550_create_posts/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +DROP TABLE posts; diff --git a/migrations/postgres/2022-03-21-155550_create_posts/up.sql b/migrations/postgres/2022-03-21-155550_create_posts/up.sql new file mode 100644 index 0000000..d7d0eba --- /dev/null +++ b/migrations/postgres/2022-03-21-155550_create_posts/up.sql @@ -0,0 +1,23 @@ +-- Your SQL goes here +CREATE TABLE posts ( + id SERIAL PRIMARY KEY, + author_hash VARCHAR NOT NULL, + content TEXT NOT NULL, + cw VARCHAR NOT NULL DEFAULT '', + author_title VARCHAR NOT NULL DEFAULT '', + is_tmp BOOLEAN NOT NULL DEFAULT FALSE, + n_attentions INTEGER NOT NULL DEFAULT 0, + n_comments INTEGER NOT NULL DEFAULT 0, + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + last_comment_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + is_deleted BOOLEAN NOT NULL DEFAULT FALSE, + is_reported BOOLEAN NOT NULL DEFAULT FALSE, + hot_score INTEGER NOT NULL DEFAULT 0, + allow_search BOOLEAN NOT NULL DEFAULT FALSE +); + +CREATE INDEX posts_last_comment_time_idx ON posts (last_comment_time); +CREATE INDEX posts_hot_idx ON posts (hot_score); +CREATE INDEX posts_author_idx ON posts (author_title); +CREATE INDEX posts_cw_idx ON posts (cw); +CREATE INDEX posts_search_text_trgm_idx ON posts USING gin(content gin_trgm_ops); diff --git a/migrations/postgres/2022-03-21-162914_create_users/down.sql b/migrations/postgres/2022-03-21-162914_create_users/down.sql new file mode 100644 index 0000000..dc3714b --- /dev/null +++ b/migrations/postgres/2022-03-21-162914_create_users/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +DROP TABLE users; diff --git a/migrations/postgres/2022-03-21-162914_create_users/up.sql b/migrations/postgres/2022-03-21-162914_create_users/up.sql new file mode 100644 index 0000000..cefd288 --- /dev/null +++ b/migrations/postgres/2022-03-21-162914_create_users/up.sql @@ -0,0 +1,7 @@ +-- Your SQL goes here +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + name VARCHAR NOT NULL UNIQUE, + token VARCHAR NOT NULL UNIQUE, + is_admin BOOLEAN NOT NULL DEFAULT FALSE +); diff --git a/migrations/postgres/2022-03-21-164718_create_comments/down.sql b/migrations/postgres/2022-03-21-164718_create_comments/down.sql new file mode 100644 index 0000000..d0841ff --- /dev/null +++ b/migrations/postgres/2022-03-21-164718_create_comments/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +DROP TABLE comments; diff --git a/migrations/postgres/2022-03-21-164718_create_comments/up.sql b/migrations/postgres/2022-03-21-164718_create_comments/up.sql new file mode 100644 index 0000000..5b1558a --- /dev/null +++ b/migrations/postgres/2022-03-21-164718_create_comments/up.sql @@ -0,0 +1,14 @@ +-- Your SQL goes here +CREATE TABLE comments ( + id SERIAL PRIMARY KEY, + author_hash VARCHAR NOT NULL, + author_title VARCHAR NOT NULL DEFAULT '', + is_tmp BOOLEAN NOT NULL DEFAULT FALSE, + content TEXT NOT NULL, + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + is_deleted BOOLEAN NOT NULL DEFAULT FALSE, + allow_search BOOLEAN NOT NULL DEFAULT FALSE, + post_id INTEGER NOT NULL REFERENCES posts(id) +); +CREATE INDEX comments_postId_idx ON comments (post_id); + diff --git a/migrations/2022-03-11-065048_create_posts/down.sql b/migrations/sqlite3/2022-03-11-065048_create_posts/down.sql similarity index 100% rename from migrations/2022-03-11-065048_create_posts/down.sql rename to migrations/sqlite3/2022-03-11-065048_create_posts/down.sql diff --git a/migrations/2022-03-11-065048_create_posts/up.sql b/migrations/sqlite3/2022-03-11-065048_create_posts/up.sql similarity index 94% rename from migrations/2022-03-11-065048_create_posts/up.sql rename to migrations/sqlite3/2022-03-11-065048_create_posts/up.sql index 91a785a..bb74e0b 100644 --- a/migrations/2022-03-11-065048_create_posts/up.sql +++ b/migrations/sqlite3/2022-03-11-065048_create_posts/up.sql @@ -14,7 +14,7 @@ CREATE TABLE posts ( is_deleted BOOLEAN NOT NULL DEFAULT FALSE, is_reported BOOLEAN NOT NULL DEFAULT FALSE, hot_score INTEGER NOT NULL DEFAULT 0, - allow_search BOOLEAN NOT NULL DEFAULT '' + allow_search BOOLEAN NOT NULL DEFAULT FALSE ); CREATE INDEX posts_last_comment_time_idx ON posts (`last_comment_time`); CREATE INDEX posts_hot_idx ON posts (`hot_score`) diff --git a/migrations/2022-03-15-061041_create_users/down.sql b/migrations/sqlite3/2022-03-15-061041_create_users/down.sql similarity index 100% rename from migrations/2022-03-15-061041_create_users/down.sql rename to migrations/sqlite3/2022-03-15-061041_create_users/down.sql diff --git a/migrations/2022-03-15-061041_create_users/up.sql b/migrations/sqlite3/2022-03-15-061041_create_users/up.sql similarity index 100% rename from migrations/2022-03-15-061041_create_users/up.sql rename to migrations/sqlite3/2022-03-15-061041_create_users/up.sql diff --git a/migrations/2022-03-15-104943_create_comments/down.sql b/migrations/sqlite3/2022-03-15-104943_create_comments/down.sql similarity index 100% rename from migrations/2022-03-15-104943_create_comments/down.sql rename to migrations/sqlite3/2022-03-15-104943_create_comments/down.sql diff --git a/migrations/2022-03-15-104943_create_comments/up.sql b/migrations/sqlite3/2022-03-15-104943_create_comments/up.sql similarity index 100% rename from migrations/2022-03-15-104943_create_comments/up.sql rename to migrations/sqlite3/2022-03-15-104943_create_comments/up.sql diff --git a/src/api/operation.rs b/src/api/operation.rs index 25e978f..c3c7767 100644 --- a/src/api/operation.rs +++ b/src/api/operation.rs @@ -18,11 +18,12 @@ pub async fn delete(di: Form, user: CurrentUser, db: Db) -> API { let c = Comment::get(&db, di.id).await.m()?; c.soft_delete(&user, &db).await?; + let p = Post::get(&db, c.post_id).await.m()?; + p.change_n_comments(&db, -1).await.m()?; } "pid" => { let p = Post::get(&db, di.id).await.m()?; p.soft_delete(&user, &db).await?; - p.change_n_comments(&db, -1).await.m()?; } _ => return Err(APIError::PcError(NotAllowed)), } diff --git a/src/api/post.rs b/src/api/post.rs index 78192b9..f36ba0a 100644 --- a/src/api/post.rs +++ b/src/api/post.rs @@ -140,7 +140,7 @@ pub async fn get_list( #[post("/dopost", data = "")] pub async fn publish_post(poi: Form, user: CurrentUser, db: Db) -> JsonAPI { - let r = Post::create( + let p = Post::create( &db, NewPost { content: poi.text.to_string(), @@ -156,7 +156,6 @@ pub async fn publish_post(poi: Form, user: CurrentUser, db: Db) -> Js .m()?; // TODO: attention Ok(json!({ - "data": r, "code": 0 })) } diff --git a/src/db_conn.rs b/src/db_conn.rs index 2a7ef2f..5eb0e66 100644 --- a/src/db_conn.rs +++ b/src/db_conn.rs @@ -1,7 +1,7 @@ use rocket_sync_db_pools::{database, diesel}; -pub type Conn = diesel::SqliteConnection; +pub type Conn = diesel::pg::PgConnection; -#[database("sqlite_v2")] +#[database("pg_v2")] pub struct Db(Conn); diff --git a/src/models.rs b/src/models.rs index 18c6066..54722cc 100644 --- a/src/models.rs +++ b/src/models.rs @@ -20,6 +20,7 @@ macro_rules! get { macro_rules! get_multi { ($table:ident) => { pub async fn get_multi(db: &Db, ids: Vec) -> QueryResult> { + // can use eq(any()) for postgres db.run(move |c| $table::table.filter($table::id.eq_any(ids)).load(c)) .await } @@ -106,9 +107,9 @@ impl Post { .await } - pub async fn create(db: &Db, new_post: NewPost) -> QueryResult { + pub async fn create(db: &Db, new_post: NewPost) -> QueryResult { // TODO: tags - db.run(move |c| insert_into(posts::table).values(&new_post).execute(c)) + db.run(move |c| insert_into(posts::table).values(&new_post).get_result(c)) .await } @@ -169,6 +170,7 @@ pub struct Comment { pub content: String, pub create_time: NaiveDateTime, pub is_deleted: bool, + pub allow_search: bool, pub post_id: i32, } @@ -187,9 +189,13 @@ impl Comment { set_deleted!(comments); - pub async fn create(db: &Db, new_comment: NewComment) -> QueryResult { - db.run(move |c| insert_into(comments::table).values(&new_comment).execute(c)) - .await + pub async fn create(db: &Db, new_comment: NewComment) -> QueryResult { + db.run(move |c| { + insert_into(comments::table) + .values(&new_comment) + .get_result(c) + }) + .await } pub async fn gets_by_post_id(db: &Db, post_id: i32) -> QueryResult> { diff --git a/src/schema.rs b/src/schema.rs index 7e7c600..0fe625d 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -1,40 +1,41 @@ table! { comments (id) { - id -> Integer, - author_hash -> Text, - author_title -> Text, + id -> Int4, + author_hash -> Varchar, + author_title -> Varchar, is_tmp -> Bool, content -> Text, create_time -> Timestamp, is_deleted -> Bool, - post_id -> Integer, + allow_search -> Bool, + post_id -> Int4, } } table! { posts (id) { - id -> Integer, - author_hash -> Text, + id -> Int4, + author_hash -> Varchar, content -> Text, - cw -> Text, - author_title -> Text, + cw -> Varchar, + author_title -> Varchar, is_tmp -> Bool, - n_attentions -> Integer, - n_comments -> Integer, + n_attentions -> Int4, + n_comments -> Int4, create_time -> Timestamp, last_comment_time -> Timestamp, is_deleted -> Bool, is_reported -> Bool, - hot_score -> Integer, + hot_score -> Int4, allow_search -> Bool, } } table! { users (id) { - id -> Integer, - name -> Text, - token -> Text, + id -> Int4, + name -> Varchar, + token -> Varchar, is_admin -> Bool, } } diff --git a/tools/migdb.py b/tools/migdb.py index 9dd8327..3b69a08 100644 --- a/tools/migdb.py +++ b/tools/migdb.py @@ -1,18 +1,25 @@ import sqlite3 +import psycopg2 from datetime import datetime db_old = sqlite3.connect('hole.db') -db_new = sqlite3.connect('hole_v2.db') +# change hole_pass to your real password +db_new = psycopg2.connect("postgres://hole:hole_pass@localhost/hole_v2") c_old = db_old.cursor() c_new = db_new.cursor() +searchable = {} +post_d = {} + +dt = datetime.now() + def mig_post(): rs = c_old.execute( 'SELECT id, name_hash, content, cw, author_title, ' 'likenum, n_comments, timestamp, comment_timestamp, ' 'deleted, is_reported, hot_score, allow_search ' - 'FROM post WHERE deleted = false' + 'FROM post ORDER BY id' ) for r in rs: @@ -22,12 +29,29 @@ def mig_post(): r[8] = r[8] or r[7] # comment_timestamp r[7] = datetime.fromtimestamp(r[7]) r[8] = datetime.fromtimestamp(r[8]) - r[10] = r[10] or False # comment - r.insert(4, r[2].startswith('[tmp]\n')) + r[9] = bool(r[9]) + r[10] = bool(r[10] or False) # comment + r[12] = bool(r[12]) + searchable[r[0]] = r[12] + r.insert(5, r[2].startswith('[tmp]\n')) + # print(r) + + post_d[r[0]] = r[1:] + + max_id = r[0] + for i in range(1, max_id + 1): + r = post_d.get(i, [ + '', '', '', '', False, 0, 0, dt, dt, True, False, 0, False + ]) + c_new.execute( - 'INSERT OR REPLACE INTO posts VALUES({})'.format(','.join(['?'] * 14)), + ( + 'INSERT INTO posts VALUES({}) ' + 'ON CONFLICT (id) DO NOTHING' + ).format(','.join(["DEFAULT"] + ['%s'] * 13)), r ) + db_new.commit() @@ -35,8 +59,10 @@ def mig_user(): rs = c_old.execute('SELECT name, token FROM user') for r in rs: + # print(r) c_new.execute( - 'INSERT OR REPLACE INTO users(name, token) VALUES(?, ?)', + 'INSERT INTO users(name, token) VALUES(%s, %s) ' + 'ON CONFLICT (name) DO NOTHING', r ) db_new.commit() @@ -57,11 +83,13 @@ def mig_comment(): r = list(r) r[2] = r[2] or '' r[4] = datetime.fromtimestamp(r[4]) - r[5] = r[5] or False - r.insert(2, r[3].startswith('[tmp]\n')) + r[5] = bool(r[5] or False) + r.insert(6, searchable[r[6]]) + r.insert(3, r[3].startswith('[tmp]\n')) + # print(r) c_new.execute( - 'INSERT OR REPLACE INTO comments VALUES({})'.format(','.join(['?'] * 8)), - r + 'INSERT INTO comments VALUES({})'.format(','.join(["DEFAULT"] + ['%s'] * 8)), + r[1:] ) if not r: break @@ -71,8 +99,8 @@ def mig_comment(): if __name__ == '__main__': - mig_post() mig_user() + mig_post() mig_comment() pass