Moved rocksdb to their own crate

Cargo.lock

@@ -880,7 +880,8 @@ dependencies = [
  "log",
  "nostr-rs-client",
  "nostr-rs-relayer",
- "nostr-rs-storage",
+ "nostr-rs-rocksdb",
+ "nostr-rs-storage-base",
  "nostr-rs-types",
  "serde",
  "serde_json",
@@ -911,7 +912,8 @@ version = "0.1.0"
 dependencies = [
  "futures-util",
  "log",
- "nostr-rs-storage",
+ "nostr-rs-rocksdb",
+ "nostr-rs-storage-base",
  "nostr-rs-types",
  "parking_lot",
  "rand",
@@ -922,14 +924,23 @@ dependencies = [
 ]
 
 [[package]]
-name = "nostr-rs-storage"
+name = "nostr-rs-rocksdb"
 version = "0.1.0"
 dependencies = [
  "chrono",
+ "nostr-rs-storage-base",
+ "nostr-rs-types",
+ "rocksdb",
+ "serde_json",
+]
+
+[[package]]
+name = "nostr-rs-storage-base"
+version = "0.1.0"
+dependencies = [
  "nostr-rs-types",
  "parking_lot",
  "rand",
- "rocksdb",
  "serde_json",
  "thiserror",
  "tokio",

Cargo.toml

@@ -4,12 +4,13 @@ version = "0.1.0"
 edition = "2021"
 
 [workspace]
-members = ["crates/types", "crates/client", "crates/relayer", "crates/storage"]
+members = ["crates/types", "crates/client", "crates/relayer", "crates/storage/base", "crates/storage/rocksdb"]
 
 [dependencies]
 nostr-rs-types = { path = "crates/types" }
 nostr-rs-client = { path = "crates/client" }
-nostr-rs-storage = { path = "crates/storage", features = ["rocksdb"] }
+nostr-rs-rocksdb = { path = "crates/storage/rocksdb" }
+nostr-rs-storage-base = { path = "crates/storage/base" }
 nostr-rs-relayer = { path = "crates/relayer" }
 tokio = { version = "1.26.0", features = ["full"] }
 env_logger = "0.10.0"

crates/relayer/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 
 [dependencies]
 nostr-rs-types = { path = "../types" }
-nostr-rs-storage = { path = "../storage" }
+nostr-rs-storage-base = { path = "../storage/base" }
 futures-util = "0.3.27"
 parking_lot = "0.12.1"
 tokio = { version = "1.26.0", features = ["sync", "macros", "rt", "time"] }
@@ -16,3 +16,6 @@ thiserror = "1.0.39"
 serde_json = "1.0.94"
 rand = "0.8.5"
 log = "0.4.17"
+
+[dev-dependencies]
+nostr-rs-rocksdb = { path = "../storage/rocksdb" }

crates/relayer/src/error.rs

@@ -6,7 +6,7 @@ pub enum Error {
     IdentifierAlreadyUsed(String),
 
     #[error("Internal/DB: {0}")]
-    Db(#[from] nostr_rs_storage::Error),
+    Db(#[from] nostr_rs_storage_base::Error),
 
     #[error("WebSocket error: {0}")]
     WebSocket(#[from] tokio_tungstenite::tungstenite::Error),

crates/relayer/src/relayer.rs

@@ -1,5 +1,5 @@
 use crate::{Connection, Error, Subscription};
-use nostr_rs_storage::Storage;
+use nostr_rs_storage_base::Storage;
 use nostr_rs_types::{
     relayer,
     types::{Event, SubscriptionId},
@@ -229,20 +229,15 @@ impl<T: Storage> Relayer<T> {
 mod test {
     use super::*;
     use crate::get_id;
-    use nostr_rs_storage::RocksDb;
+    use nostr_rs_rocksdb::RocksDb;
     use nostr_rs_types::Request;
-    use std::{
-        fs::File,
-        io::{BufRead, BufReader},
-    };
 
     fn get_db(prefill: bool) -> RocksDb {
-        let db = RocksDb::new(format!("tests/db/{}", get_id())).expect("db");
+        let db = RocksDb::new(format!("/tmp/db/{}", get_id())).expect("db");
         if prefill {
-            let file = File::open("./tests/events.json").expect("file");
-            let events = BufReader::new(file)
+            let events = include_str!("../tests/events.json")
                 .lines()
-                .map(|line| serde_json::from_str(&line.expect("line")).expect("valid"))
+                .map(|line| serde_json::from_str(&line).expect("valid"))
                 .collect::<Vec<Event>>();
 
             for event in events {
@@ -254,7 +249,14 @@ mod test {
 
     #[tokio::test]
     async fn serve_listener_from_local_db() {
-        let request: Request = serde_json::from_str("[\"REQ\",\"1298169700973717\",{\"authors\":[\"39a7d06e824c0c2523bedb93f0cef84245e4401fee03b6257a1c6dfd18b57efb\"],\"since\":1681939304},{\"#p\":[\"39a7d06e824c0c2523bedb93f0cef84245e4401fee03b6257a1c6dfd18b57efb\"],\"kinds\":[1,3,6,7,9735],\"since\":1681939304},{\"#p\":[\"39a7d06e824c0c2523bedb93f0cef84245e4401fee03b6257a1c6dfd18b57efb\"],\"kinds\":[4]},{\"authors\":[\"39a7d06e824c0c2523bedb93f0cef84245e4401fee03b6257a1c6dfd18b57efb\"],\"kinds\":[4]},{\"#e\":[\"2e72250d80e9b3fd30230b3db3ed7d22f15d266ed345c36700b01ec153c9e28a\",\"a5e3369c43daf2675ecbce18831e5f4e07db0d4dde0ef4f5698e645e4c46eed1\"],\"kinds\":[1,6,7,9735]}]").expect("valid object");
+        let request: Request = serde_json::from_str("[
+                \"REQ\",\"1298169700973717\",
+                {\"authors\":[\"39a7d06e824c0c2523bedb93f0cef84245e4401fee03b6257a1c6dfd18b57efb\"],\"since\":1681928304},
+                {\"#p\":[\"39a7d06e824c0c2523bedb93f0cef84245e4401fee03b6257a1c6dfd18b57efb\"],\"kinds\":[1,3,6,7,9735],\"since\":1681928304},
+                {\"#p\":[\"39a7d06e824c0c2523bedb93f0cef84245e4401fee03b6257a1c6dfd18b57efb\"],\"kinds\":[4]},
+                {\"authors\":[\"39a7d06e824c0c2523bedb93f0cef84245e4401fee03b6257a1c6dfd18b57efb\"],\"kinds\":[4]},
+                {\"#e\":[\"2e72250d80e9b3fd30230b3db3ed7d22f15d266ed345c36700b01ec153c9e28a\",\"a5e3369c43daf2675ecbce18831e5f4e07db0d4dde0ef4f5698e645e4c46eed1\"],\"kinds\":[1,6,7,9735]}
+            ]").expect("valid object");
         let (relayer, _) = Relayer::new(Some(get_db(true)));
         let (connection, mut recv) = Connection::new_for_test();
         let _ = relayer.recv_request_from_client(&connection, request);
@@ -269,9 +271,9 @@ mod test {
                 .id
                 .to_string()
         );
-        // ev2
+        // ev3
         assert_eq!(
-            "2e72250d80e9b3fd30230b3db3ed7d22f15d266ed345c36700b01ec153c9e28a",
+            "e862fe23daf52ab09b36a37fa91ca3743e0c323e630e8627891212ca147c2da9",
             recv.try_recv()
                 .expect("valid")
                 .as_event()
@@ -280,9 +282,9 @@ mod test {
                 .id
                 .to_string()
         );
-        // ev3
+        // ev2
         assert_eq!(
-            "e862fe23daf52ab09b36a37fa91ca3743e0c323e630e8627891212ca147c2da9",
+            "2e72250d80e9b3fd30230b3db3ed7d22f15d266ed345c36700b01ec153c9e28a",
             recv.try_recv()
                 .expect("valid")
                 .as_event()

crates/storage/Cargo.toml → crates/storage/base/Cargo.toml

@@ -1,21 +1,16 @@
 [package]
-name = "nostr-rs-storage"
+name = "nostr-rs-storage-base"
 version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-serde_json = "1.0"
-nostr-rs-types = { path = "../types" }
+nostr-rs-types = { path = "../../types" }
 thiserror = "1.0.40"
-rocksdb = { version = "0.20.1", features = [
-    "multi-threaded-cf",
-    "serde",
-    "snappy",
-] }
 rand = "0.8.5"
-chrono = "0.4.26"
 tokio = { version = "1.32.0", features = ["sync"] }
 parking_lot = "0.12.1"
+serde_json = "1.0"
 
 [features]
-rocksdb = []
+default = []
+test = []

crates/storage/src/error.rs → crates/storage/base/src/error.rs

@@ -5,8 +5,8 @@ use std::num::TryFromIntError;
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
     /// Internal database error
-    #[error("RocksDB: {0}")]
-    RocksDb(#[from] rocksdb::Error),
+    #[error("Unknown: {0}")]
+    Unknown(String),
 
     /// Serialization error
     #[error("Serde: {0}")]

crates/storage/base/src/lib.rs

@@ -0,0 +1,73 @@
+//! # Nostr Storage
+//!
+//! This crate will storage events into a database. It will also build index to
+//! find events by their tags, kind and references.
+#![allow(missing_docs, warnings)]
+#![allow(dead_code)]
+mod error;
+mod notification;
+mod storage;
+#[cfg(feature = "test")]
+pub mod test;
+
+pub use crate::{error::Error, notification::Subscription, storage::Storage};
+
+#[macro_export]
+/// This macro creates the
+macro_rules! storage_test {
+    ($type: tt, $init:tt, $destroy:tt) => {
+        #[cfg(test)]
+        mod test {
+            use super::*;
+            use std::sync::atomic::{AtomicUsize, Ordering};
+
+            static LAST_VERSION: AtomicUsize = AtomicUsize::new(0);
+
+            struct DisposableStorage {
+                path: String,
+                storage: $type,
+            }
+
+            impl DisposableStorage {
+                pub fn new() -> Self {
+                    let id = LAST_VERSION.fetch_add(1, Ordering::Relaxed);
+                    let path = format!("/tmp/nostr/test-{}", id);
+                    Self {
+                        storage: $init(&path),
+                        path,
+                    }
+                }
+            }
+
+            impl std::ops::Drop for DisposableStorage {
+                fn drop(&mut self) {
+                    $destroy(&self.path)
+                }
+            }
+
+            nostr_rs_storage_base::storage_test_name! { store_and_get }
+            nostr_rs_storage_base::storage_test_name! { records_are_sorted_by_date_desc }
+            nostr_rs_storage_base::storage_test_name! { filter_by_references }
+            nostr_rs_storage_base::storage_test_name! { filter_by_references_zero_match }
+            nostr_rs_storage_base::storage_test_name! { filter_by_references_and_kind }
+            nostr_rs_storage_base::storage_test_name! { filter_by_authors }
+            nostr_rs_storage_base::storage_test_name! { filter_by_author }
+            nostr_rs_storage_base::storage_test_name! { filter_by_author_and_kinds }
+            nostr_rs_storage_base::storage_test_name! { filter_by_kind }
+            nostr_rs_storage_base::storage_test_name! { get_event_and_related_events }
+            nostr_rs_storage_base::storage_test_name! { get_local_events }
+        }
+    };
+}
+
+#[macro_export]
+/// Creates a
+macro_rules! storage_test_name {
+    ($name:tt) => {
+        #[test]
+        fn $name() {
+            let x = DisposableStorage::new();
+            nostr_rs_storage_base::test::$name(&x.storage);
+        }
+    };
+}

crates/storage/src/lib.rs → crates/storage/base/src/test.rs

@@ -1,87 +1,7 @@
-//! # Nostr Storage
+//! # Nostr Storage - Test suite
 //!
 //! This crate will storage events into a database. It will also build index to
 //! find events by their tags, kind and references.
-//!
-//! Although the trait is generic this implementation uses rocksDB internal,
-//! which is a C++ database open source by Facebook. It is high level enough to
-//! let this crate build the needed indexes to find events quickly.
-#![deny(missing_docs, warnings)]
-mod error;
-mod event_filter;
-mod notification;
-#[cfg(feature = "rocksdb")]
-mod rocksdb;
-mod secondary_index;
-mod storage;
-
-pub use crate::{error::Error, notification::Subscription, storage::Storage};
-
-#[cfg(feature = "rocksdb")]
-pub use crate::rocksdb::RocksDb;
-
-#[macro_export]
-/// This macro creates the
-macro_rules! storage_test {
-    ($type: tt, $init:tt, $destroy:tt) => {
-        #[cfg(test)]
-        mod test {
-            use super::*;
-            use std::sync::atomic::{AtomicUsize, Ordering};
-
-            static LAST_VERSION: AtomicUsize = AtomicUsize::new(0);
-
-            struct DisposableStorage {
-                path: String,
-                storage: $type,
-            }
-
-            impl DisposableStorage {
-                pub fn new() -> Self {
-                    let id = LAST_VERSION.fetch_add(1, Ordering::Relaxed);
-                    let path = format!("/tmp/nostr/test-{}", id);
-                    Self {
-                        storage: $init(&path),
-                        path,
-                    }
-                }
-            }
-
-            impl std::ops::Drop for DisposableStorage {
-                fn drop(&mut self) {
-                    $destroy(&self.path)
-                }
-            }
-
-            $crate::storage_test_name! { store_and_get }
-            $crate::storage_test_name! { records_are_sorted_by_date_desc }
-            $crate::storage_test_name! { filter_by_references }
-            $crate::storage_test_name! { filter_by_references_zero_match }
-            $crate::storage_test_name! { filter_by_references_and_kind }
-            $crate::storage_test_name! { filter_by_authors }
-            $crate::storage_test_name! { filter_by_author }
-            $crate::storage_test_name! { filter_by_author_and_kinds }
-            $crate::storage_test_name! { filter_by_kind }
-            $crate::storage_test_name! { get_event_and_related_events }
-            $crate::storage_test_name! { get_local_events }
-        }
-    };
-}
-
-#[macro_export]
-/// Creates a
-macro_rules! storage_test_name {
-    ($name:tt) => {
-        #[test]
-        fn $name() {
-            let x = DisposableStorage::new();
-            $crate::test::$name(&x.storage);
-        }
-    };
-}
-
-#[cfg(test)]
-mod test {
     use super::*;
     use nostr_rs_types::types::{Addr, Event, Filter, Kind};
     use std::{
@@ -93,10 +13,8 @@ mod test {
     where
         T: Storage,
     {
-        let file = File::open("./tests/events.json").expect("file");
-        let events = BufReader::new(file)
-            .lines()
-            .map(|line| serde_json::from_str(&line.expect("line")).expect("valid"))
+        let events = include_str!("../tests/events.json").lines()
+            .map(|line| serde_json::from_str(&line).expect("valid"))
             .collect::<Vec<Event>>();
 
         for event in events {
@@ -378,4 +296,3 @@ mod test {
             .expect("valid");
         assert_eq!(x.len(), records.len())
     }
-}

crates/storage/rocksdb/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "nostr-rs-rocksdb"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+nostr-rs-storage-base = { path = "../base" }
+nostr-rs-types = { path = "../../types" }
+rocksdb = { version = "0.20.1", features = [
+    "multi-threaded-cf",
+    "serde",
+    "snappy",
+] }
+chrono = "0.4.26"
+serde_json = "1.0"
+
+[dev-dependencies]
+nostr-rs-storage-base = { path = "../base", features = ["test"] }

crates/storage/src/rocksdb/iterator.rs → crates/storage/rocksdb/src/iterator.rs

@@ -1,5 +1,6 @@
 //! Rocks DB implementation of the storage layer
-use crate::{event_filter::EventFilter, Error, RocksDb, Storage};
+use crate::{event_filter::EventFilter, RocksDb};
+use nostr_rs_storage_base::{Error, Storage};
 use nostr_rs_types::types::Event;
 use rocksdb::{BoundColumnFamily, DBIteratorWithThreadMode, DB};
 use std::{collections::VecDeque, sync::Arc};

crates/storage/src/rocksdb/mod.rs → crates/storage/rocksdb/src/lib.rs

@@ -1,6 +1,6 @@
 //! Rocks DB implementation of the storage layer
-use self::iterator::WrapperIterator;
-use crate::{secondary_index::SecondaryIndex, Error, Storage};
+use crate::{secondary_index::SecondaryIndex, iterator::WrapperIterator};
+use nostr_rs_storage_base::{Error, Storage};
 use nostr_rs_types::types::{Event, Filter, Tag};
 use rocksdb::{
     BoundColumnFamily, ColumnFamilyDescriptor, IteratorMode, Options, SliceTransform, WriteBatch,
@@ -8,7 +8,9 @@ use rocksdb::{
 };
 use std::{collections::VecDeque, ops::Deref, path::Path, sync::Arc};
 
+mod event_filter;
 mod iterator;
+mod secondary_index;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
 enum ReferenceType {
@@ -56,7 +58,8 @@ impl RocksDb {
                 ColumnFamilyDescriptor::new(ReferenceType::LocalEvents.as_str(), options.clone()),
                 ColumnFamilyDescriptor::new(ReferenceType::Stream.as_str(), options.clone()),
             ],
-        )?;
+        )
+        .map_err(|e| Error::Unknown(e.to_string()))?;
         Ok(Self { db })
     }
 
@@ -109,7 +112,7 @@ impl Storage for RocksDb {
             &self.reference_to_cf_handle(ReferenceType::LocalEvents)?,
             secondary_index.index_by([]),
             event_id.deref(),
-        )?;
+        ).map_err(|e| Error::Unknown(e.to_string()))?;
         Ok(())
     }
 
@@ -188,7 +191,7 @@ impl Storage for RocksDb {
             }
         }
 
-        self.db.write(buffer).map_err(Error::RocksDb)?;
+        self.db.write(buffer).map_err(|e| Error::Unknown(e.to_string()))?;
 
         Ok(true)
     }
@@ -196,7 +199,8 @@ impl Storage for RocksDb {
     fn get_event<T: AsRef<[u8]>>(&self, id: T) -> Result<Option<Event>, Error> {
         Ok(self
             .db
-            .get_cf(&self.reference_to_cf_handle(ReferenceType::Events)?, id)?
+            .get_cf(&self.reference_to_cf_handle(ReferenceType::Events)?, id)
+            .map_err(|e| Error::Unknown(e.to_string()))?
             .map(|event| serde_json::from_slice(&event))
             .transpose()?)
     }
@@ -283,4 +287,4 @@ fn destroy_instance(path: &str) {
     std::fs::remove_dir_all(path).expect("deleted file");
 }
 
-crate::storage_test!(RocksDb, new_instance, destroy_instance);
+nostr_rs_storage_base::storage_test!(RocksDb, new_instance, destroy_instance);

src/bin/dump.rs

@@ -1,7 +1,8 @@
 use futures::Future;
 use nostr_rs_client::{Error as ClientError, Event, Pool};
-use nostr_rs_storage::{RocksDb, Storage};
+use nostr_rs_rocksdb::RocksDb;
 use nostr_rs_types::{client::Subscribe, Request, Response};
+use nostr_rs_storage_base::Storage;
 use std::pin::Pin;
 use tokio::sync::mpsc;
 

src/main.rs

@@ -1,6 +1,6 @@
 use futures::Future;
 use nostr_rs_client::Event;
-use nostr_rs_storage::RocksDb;
+use nostr_rs_rocksdb::RocksDb;
 use nostr_rs_types::{Request, Response};
 use std::{collections::HashMap, pin::Pin, sync::Arc};
 use tokio::{