From bf121623ae1b112e93743cdf7a4328b6b34ff12c Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Tue, 6 Jun 2023 16:03:29 +0800 Subject: [PATCH] feat: save snapshot to sqlite db (#2718) * chore: snapshot * chore: impl sqlite snapshot * feat: snapshot config * feat: update patch * ci: fix tauri ci * ci: add cache path * chore: save snapshot * chore: update patch * ci: fix s fmt --- .github/workflows/tauri_ci.yaml | 5 +- frontend/appflowy_tauri/src-tauri/Cargo.toml | 12 +- frontend/rust-lib/Cargo.lock | 31 +-- frontend/rust-lib/Cargo.toml | 10 +- frontend/rust-lib/flowy-core/Cargo.toml | 5 +- .../src/deps_resolve/collab_deps.rs | 184 ++++++++++++++++++ .../flowy-core/src/deps_resolve/mod.rs | 2 + frontend/rust-lib/flowy-core/src/lib.rs | 82 ++++---- .../rust-lib/flowy-database2/src/manager.rs | 13 +- .../flowy-database2/src/services/mod.rs | 1 + .../src/services/snapshot/mod.rs | 1 + .../rust-lib/flowy-document2/src/manager.rs | 2 +- .../flowy-document2/tests/document/util.rs | 2 +- .../down.sql | 2 + .../2023-06-05-135652_collab_snapshot/up.sql | 8 + frontend/rust-lib/flowy-sqlite/src/lib.rs | 28 +-- frontend/rust-lib/flowy-sqlite/src/schema.rs | 12 ++ .../rust-lib/flowy-test/tests/folder/test.rs | 8 +- .../flowy-user/src/services/database.rs | 11 +- .../flowy-user/src/services/user_session.rs | 2 +- 20 files changed, 323 insertions(+), 98 deletions(-) create mode 100644 frontend/rust-lib/flowy-core/src/deps_resolve/collab_deps.rs create mode 100644 frontend/rust-lib/flowy-database2/src/services/snapshot/mod.rs create mode 100644 frontend/rust-lib/flowy-sqlite/migrations/2023-06-05-135652_collab_snapshot/down.sql create mode 100644 frontend/rust-lib/flowy-sqlite/migrations/2023-06-05-135652_collab_snapshot/up.sql diff --git a/.github/workflows/tauri_ci.yaml b/.github/workflows/tauri_ci.yaml index ba92bac1ae..6fb603aea5 100644 --- a/.github/workflows/tauri_ci.yaml +++ b/.github/workflows/tauri_ci.yaml @@ -36,9 +36,10 @@ jobs: - uses: Swatinem/rust-cache@v2 with: - prefix-key: 'ubuntu-latest' + prefix-key: 'ubuntu-latest-tauri' workspaces: | frontend/rust-lib + frontend/appflowy_tauri/src-tauri - name: install dependencies (windows only) if: matrix.platform == 'windows-latest' @@ -51,7 +52,7 @@ jobs: npm install -g pnpm@${{ env.PNPM_VERSION }} - name: install dependencies (ubuntu only) - if: matrix.platform == 'ubuntu-20.04' + if: matrix.platform == 'ubuntu-latest' working-directory: frontend run: | sudo apt-get update diff --git a/frontend/appflowy_tauri/src-tauri/Cargo.toml b/frontend/appflowy_tauri/src-tauri/Cargo.toml index 36be9d1817..82de264d69 100644 --- a/frontend/appflowy_tauri/src-tauri/Cargo.toml +++ b/frontend/appflowy_tauri/src-tauri/Cargo.toml @@ -34,12 +34,12 @@ default = ["custom-protocol"] custom-protocol = ["tauri/custom-protocol"] [patch.crates-io] -collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "cbc2e0" } -collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "cbc2e0" } -collab-persistence = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "cbc2e0" } -collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "cbc2e0" } -collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "cbc2e0" } -appflowy-integrate = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "cbc2e0" } +collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c519e" } +collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c519e" } +collab-persistence = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c519e" } +collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c519e" } +collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c519e" } +appflowy-integrate = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c519e" } #collab = { path = "../../AppFlowy-Collab/collab" } #collab-folder = { path = "../../AppFlowy-Collab/collab-folder" } diff --git a/frontend/rust-lib/Cargo.lock b/frontend/rust-lib/Cargo.lock index 33d0ad9547..879000150c 100644 --- a/frontend/rust-lib/Cargo.lock +++ b/frontend/rust-lib/Cargo.lock @@ -85,7 +85,7 @@ checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" [[package]] name = "appflowy-integrate" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=cbc2e0#cbc2e0acb8420dc997921bb3f56b99f9975c2aab" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c519e#5c519e988e93f6fbc7d98ce4373c5ff30a547fb1" dependencies = [ "anyhow", "collab", @@ -887,7 +887,7 @@ dependencies = [ [[package]] name = "collab" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=cbc2e0#cbc2e0acb8420dc997921bb3f56b99f9975c2aab" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c519e#5c519e988e93f6fbc7d98ce4373c5ff30a547fb1" dependencies = [ "anyhow", "bytes", @@ -905,7 +905,7 @@ dependencies = [ [[package]] name = "collab-client-ws" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=cbc2e0#cbc2e0acb8420dc997921bb3f56b99f9975c2aab" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c519e#5c519e988e93f6fbc7d98ce4373c5ff30a547fb1" dependencies = [ "bytes", "collab-sync", @@ -923,7 +923,7 @@ dependencies = [ [[package]] name = "collab-database" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=cbc2e0#cbc2e0acb8420dc997921bb3f56b99f9975c2aab" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c519e#5c519e988e93f6fbc7d98ce4373c5ff30a547fb1" dependencies = [ "anyhow", "async-trait", @@ -949,7 +949,7 @@ dependencies = [ [[package]] name = "collab-derive" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=cbc2e0#cbc2e0acb8420dc997921bb3f56b99f9975c2aab" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c519e#5c519e988e93f6fbc7d98ce4373c5ff30a547fb1" dependencies = [ "proc-macro2", "quote", @@ -961,7 +961,7 @@ dependencies = [ [[package]] name = "collab-document" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=cbc2e0#cbc2e0acb8420dc997921bb3f56b99f9975c2aab" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c519e#5c519e988e93f6fbc7d98ce4373c5ff30a547fb1" dependencies = [ "anyhow", "collab", @@ -978,7 +978,7 @@ dependencies = [ [[package]] name = "collab-folder" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=cbc2e0#cbc2e0acb8420dc997921bb3f56b99f9975c2aab" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c519e#5c519e988e93f6fbc7d98ce4373c5ff30a547fb1" dependencies = [ "anyhow", "collab", @@ -997,7 +997,7 @@ dependencies = [ [[package]] name = "collab-persistence" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=cbc2e0#cbc2e0acb8420dc997921bb3f56b99f9975c2aab" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c519e#5c519e988e93f6fbc7d98ce4373c5ff30a547fb1" dependencies = [ "bincode", "chrono", @@ -1017,7 +1017,7 @@ dependencies = [ [[package]] name = "collab-plugins" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=cbc2e0#cbc2e0acb8420dc997921bb3f56b99f9975c2aab" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c519e#5c519e988e93f6fbc7d98ce4373c5ff30a547fb1" dependencies = [ "anyhow", "async-trait", @@ -1036,6 +1036,7 @@ dependencies = [ "rusoto_credential", "serde", "serde_json", + "similar 2.2.1", "thiserror", "tokio", "tokio-retry", @@ -1047,7 +1048,7 @@ dependencies = [ [[package]] name = "collab-sync" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=cbc2e0#cbc2e0acb8420dc997921bb3f56b99f9975c2aab" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c519e#5c519e988e93f6fbc7d98ce4373c5ff30a547fb1" dependencies = [ "bytes", "collab", @@ -1568,7 +1569,7 @@ dependencies = [ "quote", "serde", "serde_json", - "similar", + "similar 1.3.0", "syn 1.0.109", "tera", "toml", @@ -1598,6 +1599,7 @@ dependencies = [ "appflowy-integrate", "bytes", "console-subscriber", + "diesel", "flowy-config", "flowy-database2", "flowy-document2", @@ -1619,6 +1621,7 @@ dependencies = [ "serde_repr", "tokio", "tracing", + "uuid", ] [[package]] @@ -4218,6 +4221,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad1d488a557b235fc46dae55512ffbfc429d2482b08b4d9435ab07384ca8aec" +[[package]] +name = "similar" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" + [[package]] name = "siphasher" version = "0.3.10" diff --git a/frontend/rust-lib/Cargo.toml b/frontend/rust-lib/Cargo.toml index ce6e8cb048..dadfa111c6 100644 --- a/frontend/rust-lib/Cargo.toml +++ b/frontend/rust-lib/Cargo.toml @@ -33,11 +33,11 @@ opt-level = 3 incremental = false [patch.crates-io] -collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "cbc2e0" } -collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "cbc2e0" } -collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "cbc2e0" } -collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "cbc2e0" } -appflowy-integrate = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "cbc2e0" } +collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c519e" } +collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c519e" } +collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c519e" } +collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c519e" } +appflowy-integrate = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c519e" } #collab = { path = "../AppFlowy-Collab/collab" } #collab-folder = { path = "../AppFlowy-Collab/collab-folder" } diff --git a/frontend/rust-lib/flowy-core/Cargo.toml b/frontend/rust-lib/flowy-core/Cargo.toml index 1c32d78231..ec24342afc 100644 --- a/frontend/rust-lib/flowy-core/Cargo.toml +++ b/frontend/rust-lib/flowy-core/Cargo.toml @@ -12,13 +12,15 @@ flowy-user = { path = "../flowy-user" } flowy-net = { path = "../flowy-net" } flowy-folder2 = { path = "../flowy-folder2" } flowy-database2 = { path = "../flowy-database2" } -flowy-sqlite = { path = "../flowy-sqlite", optional = true } +flowy-sqlite = { path = "../flowy-sqlite" } flowy-document2 = { path = "../flowy-document2" } flowy-error = { path = "../flowy-error" } flowy-task = { path = "../flowy-task" } flowy-server = { path = "../flowy-server" } flowy-config = { path = "../flowy-config" } appflowy-integrate = { version = "0.1.0" } +diesel = { version = "1.4.8", features = ["sqlite"] } +uuid = { version = "1.3.3", features = ["v4"] } tracing = { version = "0.1", features = ["log"] } futures-core = { version = "0.3", default-features = false } @@ -56,7 +58,6 @@ ts = [ "flowy-config/ts", ] rev-sqlite = [ - "flowy-sqlite", "flowy-user/rev-sqlite", ] openssl_vendored = ["flowy-sqlite/openssl_vendored"] diff --git a/frontend/rust-lib/flowy-core/src/deps_resolve/collab_deps.rs b/frontend/rust-lib/flowy-core/src/deps_resolve/collab_deps.rs new file mode 100644 index 0000000000..3312a32db2 --- /dev/null +++ b/frontend/rust-lib/flowy-core/src/deps_resolve/collab_deps.rs @@ -0,0 +1,184 @@ +use std::sync::Arc; + +use appflowy_integrate::{ + calculate_snapshot_diff, try_encode_snapshot, CollabSnapshot, MutexCollab, PersistenceError, + Snapshot, SnapshotDB, +}; +use diesel::SqliteConnection; + +use flowy_error::FlowyError; +use flowy_sqlite::{ + insert_or_ignore_into, + prelude::*, + schema::{collab_snapshot, collab_snapshot::dsl}, +}; +use flowy_user::services::UserSession; +use lib_infra::util::timestamp; + +pub struct SnapshotDBImpl(pub Arc); + +impl SnapshotDB for SnapshotDBImpl { + fn get_snapshots(&self, _uid: i64, object_id: &str) -> Vec { + self + .0 + .db_pool() + .and_then(|pool| Ok(pool.get()?)) + .and_then(|conn| { + CollabSnapshotTableSql::get_all_snapshots(object_id, &conn) + .and_then(|rows| Ok(rows.into_iter().map(|row| row.into()).collect())) + }) + .unwrap_or_else(|_| vec![]) + } + + fn create_snapshot( + &self, + uid: i64, + object_id: &str, + snapshot: Snapshot, + collab: Arc, + ) -> Result<(), PersistenceError> { + let object_id = object_id.to_string(); + let weak_pool = Arc::downgrade( + &self + .0 + .db_pool() + .map_err(|e| PersistenceError::Internal(Box::new(e)))?, + ); + + let _ = tokio::task::spawn_blocking(move || { + if let Some(pool) = weak_pool.upgrade() { + let conn = pool + .get() + .map_err(|e| PersistenceError::Internal(Box::new(e)))?; + + // Try to acquire a txn lock, if failed, it means there is a txn running, so we just ignore this snapshot + let result = try_encode_snapshot( + &collab + .lock() + .try_transaction() + .map_err(|e| PersistenceError::Internal(Box::new(e)))?, + snapshot, + ); + + match result.and_then(|new_snapshot_data| { + let desc = match CollabSnapshotTableSql::get_latest_snapshot(&object_id, &conn) { + None => Ok("".to_string()), + Some(old_snapshot) => { + calculate_snapshot_diff(uid, &object_id, &old_snapshot.data, &new_snapshot_data) + }, + } + .map_err(|e| PersistenceError::InvalidData(format!("{:?}", e)))?; + + // Save the snapshot to disk + CollabSnapshotTableSql::create( + CollabSnapshotRow { + id: uuid::Uuid::new_v4().to_string(), + object_id: object_id.clone(), + desc, + timestamp: timestamp(), + data: new_snapshot_data, + }, + &conn, + ) + .map_err(|e| PersistenceError::Internal(Box::new(e))) + }) { + Ok(_) => {}, + Err(e) => tracing::error!("create snapshot error: {:?}", e), + } + } + Ok::<(), PersistenceError>(()) + }); + Ok(()) + } +} + +#[derive(PartialEq, Clone, Debug, Queryable, Identifiable, Insertable, Associations)] +#[table_name = "collab_snapshot"] +struct CollabSnapshotRow { + id: String, + object_id: String, + desc: String, + timestamp: i64, + data: Vec, +} + +impl From for CollabSnapshot { + fn from(table: CollabSnapshotRow) -> Self { + Self { + data: table.data, + created_at: table.timestamp, + } + } +} + +struct CollabSnapshotTableSql; +impl CollabSnapshotTableSql { + fn create(row: CollabSnapshotRow, conn: &SqliteConnection) -> Result<(), FlowyError> { + // Batch insert: https://diesel.rs/guides/all-about-inserts.html + let values = ( + dsl::id.eq(row.id), + dsl::object_id.eq(row.object_id), + dsl::desc.eq(row.desc), + dsl::data.eq(row.data), + dsl::timestamp.eq(row.timestamp), + ); + let _ = insert_or_ignore_into(dsl::collab_snapshot) + .values(values) + .execute(conn)?; + Ok(()) + } + + fn get_all_snapshots( + object_id: &str, + conn: &SqliteConnection, + ) -> Result, FlowyError> { + let sql = dsl::collab_snapshot + .filter(dsl::object_id.eq(object_id)) + .into_boxed(); + + let rows = sql + .order(dsl::timestamp.asc()) + .load::(conn)?; + + Ok(rows) + } + + fn get_latest_snapshot(object_id: &str, conn: &SqliteConnection) -> Option { + let sql = dsl::collab_snapshot + .filter(dsl::object_id.eq(object_id)) + .into_boxed(); + + sql + .order(dsl::timestamp.desc()) + .first::(conn) + .ok() + } + + #[allow(dead_code)] + fn delete( + object_id: &str, + snapshot_ids: Option>, + conn: &SqliteConnection, + ) -> Result<(), FlowyError> { + let mut sql = diesel::delete(dsl::collab_snapshot).into_boxed(); + sql = sql.filter(dsl::object_id.eq(object_id)); + + if let Some(snapshot_ids) = snapshot_ids { + tracing::trace!( + "[{}] Delete snapshot: {}:{:?}", + std::any::type_name::(), + object_id, + snapshot_ids + ); + sql = sql.filter(dsl::id.eq_any(snapshot_ids)); + } + + let affected_row = sql.execute(conn)?; + tracing::trace!( + "[{}] Delete {} rows", + std::any::type_name::(), + affected_row + ); + Ok(()) + } +} diff --git a/frontend/rust-lib/flowy-core/src/deps_resolve/mod.rs b/frontend/rust-lib/flowy-core/src/deps_resolve/mod.rs index 7264d60864..0c4e7746c2 100644 --- a/frontend/rust-lib/flowy-core/src/deps_resolve/mod.rs +++ b/frontend/rust-lib/flowy-core/src/deps_resolve/mod.rs @@ -1,7 +1,9 @@ +pub use collab_deps::*; pub use database_deps::*; pub use document2_deps::*; pub use folder2_deps::*; +mod collab_deps; mod document2_deps; mod folder2_deps; mod util; diff --git a/frontend/rust-lib/flowy-core/src/lib.rs b/frontend/rust-lib/flowy-core/src/lib.rs index 69158c66db..3bc2c0ddff 100644 --- a/frontend/rust-lib/flowy-core/src/lib.rs +++ b/frontend/rust-lib/flowy-core/src/lib.rs @@ -10,8 +10,8 @@ use std::{ }; use appflowy_integrate::collab_builder::{AppFlowyCollabBuilder, CloudStorageType}; - use tokio::sync::RwLock; +use tracing::debug; use flowy_database2::DatabaseManager2; use flowy_document2::manager::DocumentManager as DocumentManager2; @@ -27,7 +27,6 @@ use lib_dispatch::runtime::tokio_default_runtime; use lib_infra::future::{to_fut, Fut}; use module::make_plugins; pub use module::*; -use tracing::debug; use crate::deps_resolve::*; use crate::integrate::server::{AppFlowyServerProvider, ServerProviderType}; @@ -141,45 +140,54 @@ impl AppFlowyCore { runtime.spawn(TaskRunner::run(task_dispatcher.clone())); let server_provider = Arc::new(AppFlowyServerProvider::new()); - /// The shared collab builder is used to build the [Collab] instance. The plugins will be loaded - /// on demand based on the [CollabPluginConfig]. - let collab_builder = Arc::new(AppFlowyCollabBuilder::new( - server_provider.provider_type().into(), - )); - let (user_session, folder_manager, server_provider, database_manager, document_manager2) = - runtime.block_on(async { - let user_session = mk_user_session(&config, server_provider.clone()); - let database_manager2 = Database2DepsResolver::resolve( - user_session.clone(), - task_dispatcher.clone(), - collab_builder.clone(), - ) - .await; + let ( + user_session, + folder_manager, + server_provider, + database_manager, + document_manager2, + collab_builder, + ) = runtime.block_on(async { + let user_session = mk_user_session(&config, server_provider.clone()); + /// The shared collab builder is used to build the [Collab] instance. The plugins will be loaded + /// on demand based on the [CollabPluginConfig]. + let collab_builder = Arc::new(AppFlowyCollabBuilder::new( + server_provider.provider_type().into(), + Some(Arc::new(SnapshotDBImpl(user_session.clone()))), + )); - let document_manager2 = Document2DepsResolver::resolve( - user_session.clone(), - &database_manager2, - collab_builder.clone(), - ); + let database_manager2 = Database2DepsResolver::resolve( + user_session.clone(), + task_dispatcher.clone(), + collab_builder.clone(), + ) + .await; - let folder_manager = Folder2DepsResolver::resolve( - user_session.clone(), - &document_manager2, - &database_manager2, - collab_builder.clone(), - server_provider.clone(), - ) - .await; + let document_manager2 = Document2DepsResolver::resolve( + user_session.clone(), + &database_manager2, + collab_builder.clone(), + ); - ( - user_session, - folder_manager, - server_provider, - database_manager2, - document_manager2, - ) - }); + let folder_manager = Folder2DepsResolver::resolve( + user_session.clone(), + &document_manager2, + &database_manager2, + collab_builder.clone(), + server_provider.clone(), + ) + .await; + + ( + user_session, + folder_manager, + server_provider, + database_manager2, + document_manager2, + collab_builder, + ) + }); let user_status_listener = UserStatusCallbackImpl { collab_builder, diff --git a/frontend/rust-lib/flowy-database2/src/manager.rs b/frontend/rust-lib/flowy-database2/src/manager.rs index c8a28dbe91..c6ccc25901 100644 --- a/frontend/rust-lib/flowy-database2/src/manager.rs +++ b/frontend/rust-lib/flowy-database2/src/manager.rs @@ -48,11 +48,12 @@ impl DatabaseManager2 { } pub async fn initialize(&self, user_id: i64) -> FlowyResult<()> { + let config = CollabPersistenceConfig::new().snapshot_per_update(10); let db = self.user.collab_db()?; *self.user_database.lock() = Some(InnerUserDatabase::new( user_id, db, - CollabPersistenceConfig::default(), + config, UserDatabaseCollabBuilderImpl(self.collab_builder.clone()), )); // do nothing @@ -269,16 +270,6 @@ unsafe impl Send for UserDatabase {} struct UserDatabaseCollabBuilderImpl(Arc); impl DatabaseCollabBuilder for UserDatabaseCollabBuilderImpl { - fn build( - &self, - uid: i64, - object_id: &str, - object_name: &str, - db: Arc, - ) -> Arc { - self.0.build(uid, object_id, object_name, db) - } - fn build_with_config( &self, uid: i64, diff --git a/frontend/rust-lib/flowy-database2/src/services/mod.rs b/frontend/rust-lib/flowy-database2/src/services/mod.rs index 5279e8202f..06d4cfc754 100644 --- a/frontend/rust-lib/flowy-database2/src/services/mod.rs +++ b/frontend/rust-lib/flowy-database2/src/services/mod.rs @@ -6,4 +6,5 @@ pub mod filter; pub mod group; pub mod setting; pub mod share; +pub mod snapshot; pub mod sort; diff --git a/frontend/rust-lib/flowy-database2/src/services/snapshot/mod.rs b/frontend/rust-lib/flowy-database2/src/services/snapshot/mod.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/frontend/rust-lib/flowy-database2/src/services/snapshot/mod.rs @@ -0,0 +1 @@ + diff --git a/frontend/rust-lib/flowy-document2/src/manager.rs b/frontend/rust-lib/flowy-document2/src/manager.rs index c4b5958722..518adbc2cf 100644 --- a/frontend/rust-lib/flowy-document2/src/manager.rs +++ b/frontend/rust-lib/flowy-document2/src/manager.rs @@ -49,7 +49,7 @@ impl DocumentManager { let uid = self.user.user_id()?; let db = self.user.collab_db()?; let collab = self.collab_builder.build(uid, &doc_id, "document", db); - let data = data.unwrap_or_else(|| default_document_data()); + let data = data.unwrap_or_else(default_document_data); let document = Arc::new(Document::create_with_data(collab, data)?); Ok(document) } diff --git a/frontend/rust-lib/flowy-document2/tests/document/util.rs b/frontend/rust-lib/flowy-document2/tests/document/util.rs index 5dcb7eb68a..2c8024b4b9 100644 --- a/frontend/rust-lib/flowy-document2/tests/document/util.rs +++ b/frontend/rust-lib/flowy-document2/tests/document/util.rs @@ -50,6 +50,6 @@ pub fn db() -> Arc { } pub fn default_collab_builder() -> Arc { - let builder = AppFlowyCollabBuilder::new(CloudStorageType::Local); + let builder = AppFlowyCollabBuilder::new(CloudStorageType::Local, None); Arc::new(builder) } diff --git a/frontend/rust-lib/flowy-sqlite/migrations/2023-06-05-135652_collab_snapshot/down.sql b/frontend/rust-lib/flowy-sqlite/migrations/2023-06-05-135652_collab_snapshot/down.sql new file mode 100644 index 0000000000..cb1baa942f --- /dev/null +++ b/frontend/rust-lib/flowy-sqlite/migrations/2023-06-05-135652_collab_snapshot/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +DROP TABLE collab_snapshot; \ No newline at end of file diff --git a/frontend/rust-lib/flowy-sqlite/migrations/2023-06-05-135652_collab_snapshot/up.sql b/frontend/rust-lib/flowy-sqlite/migrations/2023-06-05-135652_collab_snapshot/up.sql new file mode 100644 index 0000000000..4686e488dc --- /dev/null +++ b/frontend/rust-lib/flowy-sqlite/migrations/2023-06-05-135652_collab_snapshot/up.sql @@ -0,0 +1,8 @@ +-- Your SQL goes here +CREATE TABLE collab_snapshot ( + id TEXT NOT NULL PRIMARY KEY DEFAULT '', + object_id TEXT NOT NULL DEFAULT '', + desc TEXT NOT NULL DEFAULT '', + timestamp BIGINT NOT NULL DEFAULT 0, + data BLOB NOT NULL DEFAULT (x'') +); \ No newline at end of file diff --git a/frontend/rust-lib/flowy-sqlite/src/lib.rs b/frontend/rust-lib/flowy-sqlite/src/lib.rs index 2d24006f99..7a71b8485d 100644 --- a/frontend/rust-lib/flowy-sqlite/src/lib.rs +++ b/frontend/rust-lib/flowy-sqlite/src/lib.rs @@ -1,31 +1,35 @@ +#[macro_use] +pub extern crate diesel; +#[macro_use] +pub extern crate diesel_derives; +#[macro_use] +extern crate diesel_migrations; + +use std::{fmt::Debug, io, path::Path}; + pub use diesel::*; pub use diesel_derives::*; use diesel_migrations::*; -use std::{fmt::Debug, io, path::Path}; -pub mod kv; -mod sqlite; use crate::sqlite::PoolConfig; pub use crate::sqlite::{ConnectionPool, DBConnection, Database}; +pub mod kv; +mod sqlite; + pub mod schema; #[macro_use] pub mod macros; -#[macro_use] -extern crate diesel; -#[macro_use] -extern crate diesel_derives; -#[macro_use] -extern crate diesel_migrations; - pub type Error = diesel::result::Error; pub mod prelude { - pub use super::UserDatabaseConnection; - pub use crate::*; pub use diesel::SqliteConnection; pub use diesel::{query_dsl::*, BelongingToDsl, ExpressionMethods, RunQueryDsl}; + + pub use crate::*; + + pub use super::UserDatabaseConnection; } embed_migrations!("../flowy-sqlite/migrations/"); diff --git a/frontend/rust-lib/flowy-sqlite/src/schema.rs b/frontend/rust-lib/flowy-sqlite/src/schema.rs index e8b9d2b9bb..48ad76b484 100644 --- a/frontend/rust-lib/flowy-sqlite/src/schema.rs +++ b/frontend/rust-lib/flowy-sqlite/src/schema.rs @@ -1,5 +1,15 @@ // @generated automatically by Diesel CLI. +diesel::table! { + collab_snapshot (id) { + id -> Text, + object_id -> Text, + desc -> Text, + timestamp -> BigInt, + data -> Binary, + } +} + diesel::table! { user_table (id) { id -> Text, @@ -11,3 +21,5 @@ diesel::table! { email -> Text, } } + +diesel::allow_tables_to_appear_in_same_query!(collab_snapshot, user_table,); diff --git a/frontend/rust-lib/flowy-test/tests/folder/test.rs b/frontend/rust-lib/flowy-test/tests/folder/test.rs index 7039b91b12..af15741fba 100644 --- a/frontend/rust-lib/flowy-test/tests/folder/test.rs +++ b/frontend/rust-lib/flowy-test/tests/folder/test.rs @@ -54,7 +54,7 @@ async fn create_view_event_test() { let test = FlowyCoreTest::new_with_user().await; let current_workspace = test.get_current_workspace().await.workspace; let view = test - .create_view(¤t_workspace.id, format!("My first view")) + .create_view(¤t_workspace.id, "My first view".to_string()) .await; assert_eq!(view.parent_view_id, current_workspace.id); assert_eq!(view.name, "My first view"); @@ -66,7 +66,7 @@ async fn delete_view_event_test() { let test = FlowyCoreTest::new_with_user().await; let current_workspace = test.get_current_workspace().await.workspace; let view = test - .create_view(¤t_workspace.id, format!("My first view")) + .create_view(¤t_workspace.id, "My first view".to_string()) .await; test.delete_view(&view.id).await; @@ -89,7 +89,7 @@ async fn put_back_trash_event_test() { let test = FlowyCoreTest::new_with_user().await; let current_workspace = test.get_current_workspace().await.workspace; let view = test - .create_view(¤t_workspace.id, format!("My first view")) + .create_view(¤t_workspace.id, "My first view".to_string()) .await; test.delete_view(&view.id).await; @@ -132,7 +132,7 @@ async fn delete_view_permanently_event_test() { let test = FlowyCoreTest::new_with_user().await; let current_workspace = test.get_current_workspace().await.workspace; let view = test - .create_view(¤t_workspace.id, format!("My first view")) + .create_view(¤t_workspace.id, "My first view".to_string()) .await; let payload = RepeatedViewIdPB { items: vec![view.id.clone()], diff --git a/frontend/rust-lib/flowy-user/src/services/database.rs b/frontend/rust-lib/flowy-user/src/services/database.rs index 23c2042e69..d8264c094a 100644 --- a/frontend/rust-lib/flowy-user/src/services/database.rs +++ b/frontend/rust-lib/flowy-user/src/services/database.rs @@ -50,7 +50,7 @@ impl UserDB { Ok(pool) } - fn open_kv_db_if_need(&self, user_id: i64) -> Result, FlowyError> { + fn open_collab_db_if_need(&self, user_id: i64) -> Result, FlowyError> { if let Some(kv) = COLLAB_DB_MAP.read().get(&user_id) { return Ok(kv.clone()); } @@ -65,8 +65,9 @@ impl UserDB { let mut dir = PathBuf::new(); dir.push(&self.db_dir); dir.push(user_id.to_string()); + dir.push("collab_db"); - tracing::trace!("open kv db {} at path: {:?}", user_id, dir); + tracing::trace!("open collab db {} at path: {:?}", user_id, dir); let db = RocksCollabDB::open(dir).map_err(|err| FlowyError::internal().context(err))?; let db = Arc::new(db); write_guard.insert(user_id.to_owned(), db.clone()); @@ -94,9 +95,9 @@ impl UserDB { Ok(pool) } - pub(crate) fn get_kv_db(&self, user_id: i64) -> Result, FlowyError> { - let kv_db = self.open_kv_db_if_need(user_id)?; - Ok(kv_db) + pub(crate) fn get_collab_db(&self, user_id: i64) -> Result, FlowyError> { + let collab_db = self.open_collab_db_if_need(user_id)?; + Ok(collab_db) } } diff --git a/frontend/rust-lib/flowy-user/src/services/user_session.rs b/frontend/rust-lib/flowy-user/src/services/user_session.rs index 3be9d28c9c..64edd8fc78 100644 --- a/frontend/rust-lib/flowy-user/src/services/user_session.rs +++ b/frontend/rust-lib/flowy-user/src/services/user_session.rs @@ -96,7 +96,7 @@ impl UserSession { pub fn get_collab_db(&self) -> Result, FlowyError> { let user_id = self.get_session()?.user_id; - self.database.get_kv_db(user_id) + self.database.get_collab_db(user_id) } #[tracing::instrument(level = "debug", skip(self, params))]