1use crate::keri::db::dbing::LMDBer;
2use crate::keri::db::errors::DBError;
3use heed::types::*;
4use heed::Database;
5use serde::{Deserialize, Serialize};
6use std::fmt::Debug;
7use std::marker::PhantomData;
8use std::sync::Arc;
9use thiserror::Error;
10
11#[allow(dead_code)]
12#[derive(Debug, Error)]
13pub enum KomerError {
14 #[error("Database error: {0}")]
15 Database(#[from] heed::Error),
16
17 #[error("LMDB error: {0}")]
18 DBError(#[from] DBError),
19
20 #[error("Serialization error: {0}")]
21 Serialization(String),
22
23 #[error("Deserialization error: {0}")]
24 Deserialization(String),
25
26 #[error("Invalid schema: expected {expected} got {got}")]
27 InvalidSchema { expected: String, got: String },
28
29 #[error("Empty keys")]
30 EmptyKeys,
31}
32
33#[allow(dead_code)]
35#[derive(Debug, Clone, Copy)]
36pub enum SerialKind {
37 Json,
38 MsgPack,
39 Cbor,
40}
41
42pub struct KomerBase<T>
46where
47 T: Serialize + for<'de> Deserialize<'de> + Debug,
48{
49 db: Arc<LMDBer>, pub sdb: Database<Bytes, Bytes>,
54
55 pub kind: SerialKind,
57
58 pub sep: String,
60
61 phantom: PhantomData<T>,
63}
64
65impl<T> KomerBase<T>
66where
67 T: Serialize + for<'de> Deserialize<'de> + Debug,
68{
69 pub fn new(
71 db: Arc<LMDBer>,
72 subkey: &str,
73 kind: SerialKind,
74 dupsort: bool,
75 sep: Option<&str>,
76 ) -> Result<Self, KomerError> {
77 let sdb = db.create_database(Some(subkey), Some(dupsort))?;
78
79 Ok(Self {
80 db,
81 sdb,
82 kind,
83 sep: sep.unwrap_or(".").to_string(),
84 phantom: PhantomData,
85 })
86 }
87
88 pub fn to_key<K: AsRef<[u8]>>(&self, keys: &[K], topive: bool) -> Vec<u8> {
90 let mut result = Vec::new();
93
94 if keys.len() == 1 {
96 let key = keys[0].as_ref();
97 if !topive {
98 return key.to_vec();
99 }
100 result.extend_from_slice(key);
102 result.push(self.sep.as_bytes()[0]);
103 return result;
104 }
105
106 for (i, key) in keys.iter().enumerate() {
108 if i > 0 {
109 result.push(self.sep.as_bytes()[0]);
110 }
111 result.extend_from_slice(key.as_ref());
112 }
113
114 if topive && (!result.is_empty() && result[result.len() - 1] != self.sep.as_bytes()[0]) {
116 result.push(self.sep.as_bytes()[0]);
117 }
118
119 result
120 }
121
122 pub fn to_keys(&self, key: &[u8]) -> Vec<String> {
124 let key_str = String::from_utf8_lossy(key);
125 key_str.split(&self.sep).map(String::from).collect()
126 }
127
128 pub fn serialize(&self, val: &T) -> Result<Vec<u8>, KomerError> {
130 match self.kind {
131 SerialKind::Json => self.serialize_json(val),
132 SerialKind::MsgPack => self.serialize_msgpack(val),
133 SerialKind::Cbor => self.serialize_cbor(val),
134 }
135 }
136
137 pub fn deserialize(&self, val: &[u8]) -> Result<T, KomerError> {
139 match self.kind {
140 SerialKind::Json => self.deserialize_json(val),
141 SerialKind::MsgPack => self.deserialize_msgpack(val),
142 SerialKind::Cbor => self.deserialize_cbor(val),
143 }
144 }
145
146 fn serialize_json(&self, val: &T) -> Result<Vec<u8>, KomerError> {
148 serde_json::to_vec(val)
149 .map_err(|e| KomerError::Serialization(format!("JSON serialization error: {}", e)))
150 }
151
152 fn deserialize_json(&self, val: &[u8]) -> Result<T, KomerError> {
154 serde_json::from_slice(val)
155 .map_err(|e| KomerError::Deserialization(format!("JSON deserialization error: {}", e)))
156 }
157
158 fn serialize_msgpack(&self, val: &T) -> Result<Vec<u8>, KomerError> {
160 rmp_serde::to_vec(val)
161 .map_err(|e| KomerError::Serialization(format!("MsgPack serialization error: {}", e)))
162 }
163
164 fn deserialize_msgpack(&self, val: &[u8]) -> Result<T, KomerError> {
166 rmp_serde::from_slice(val).map_err(|e| {
167 KomerError::Deserialization(format!("MsgPack deserialization error: {}", e))
168 })
169 }
170
171 fn serialize_cbor(&self, val: &T) -> Result<Vec<u8>, KomerError> {
173 serde_cbor::to_vec(val)
174 .map_err(|e| KomerError::Serialization(format!("CBOR serialization error: {}", e)))
175 }
176
177 fn deserialize_cbor(&self, val: &[u8]) -> Result<T, KomerError> {
179 serde_cbor::from_slice(val)
180 .map_err(|e| KomerError::Deserialization(format!("CBOR deserialization error: {}", e)))
181 }
182
183 pub fn get_item_iter<K>(&self, keys: &[K]) -> Result<Vec<(Vec<String>, T)>, KomerError>
185 where
186 K: AsRef<[u8]>,
187 {
188 let key_prefix = self.to_key(keys, false);
190
191 let mut result = Vec::new();
193 self.db
194 .get_top_items_iter(&self.sdb, &key_prefix, |key, value| {
195 let key_strings = self.to_keys(&key);
196 let deserialized = self
197 .deserialize(&value)
198 .map_err(|_| DBError::ValueError("Failed to deserialize value".to_string()))?;
199 result.push((key_strings, deserialized));
200 Ok(true)
201 })?;
202
203 Ok(result)
204 }
205
206 pub fn get_full_item_iter<K>(&self, keys: &[K]) -> Result<Vec<(Vec<String>, T)>, KomerError>
208 where
209 K: AsRef<[u8]>,
210 {
211 self.get_item_iter(keys)
212 }
213
214 pub fn put<K>(&self, keys: &[K], val: &T) -> Result<bool, KomerError>
216 where
217 K: AsRef<[u8]>,
218 {
219 let key = self.to_key(keys, false);
220 if key.is_empty() {
221 Err(KomerError::EmptyKeys)
222 } else {
223 let value = self.serialize(val)?;
224 Ok(self.db.put_val(&self.sdb, &key, &value)?)
225 }
226 }
227
228 pub fn get<K>(&self, keys: &[K]) -> Result<Option<T>, KomerError>
230 where
231 K: AsRef<[u8]>,
232 {
233 let key = self.to_key(keys, false);
234 if let Some(val) = self.db.get_val(&self.sdb, &key)? {
235 Ok(Some(self.deserialize(&val)?))
236 } else {
237 Ok(None)
238 }
239 }
240
241 pub fn rem<K>(&self, keys: &[K]) -> Result<bool, KomerError>
243 where
244 K: AsRef<[u8]>,
245 {
246 let key = self.to_key(keys, false);
247 let existed = self.db.del_val(&self.sdb, &key)?;
248 Ok(existed)
249 }
250
251 pub fn cnt_all(&self) -> Result<usize, KomerError> {
253 Ok(self.db.len(&self.sdb)? as usize)
254 }
255}
256
257#[allow(dead_code)]
259pub struct Komer<T>
260where
261 T: Serialize + for<'de> Deserialize<'de> + Debug,
262{
263 base: KomerBase<T>,
264}
265
266impl<T> Komer<T>
267where
268 T: Serialize + for<'de> Deserialize<'de> + Debug,
269{
270 pub fn new(db: Arc<LMDBer>, subkey: &str, kind: SerialKind) -> Result<Self, KomerError> {
280 let base = KomerBase::new(db, subkey, kind, false, None)?;
281
282 Ok(Self { base })
283 }
284
285 pub fn put<K>(&self, keys: &[K], val: &T) -> Result<bool, KomerError>
295 where
296 K: AsRef<[u8]>,
297 {
298 self.base.put(keys, val)
299 }
300
301 pub fn pin<K>(&self, keys: &[K], val: &T) -> Result<bool, KomerError>
311 where
312 K: AsRef<[u8]>,
313 {
314 let key = self.base.to_key(keys, false);
316
317 let serialized = self.base.serialize(val)?;
319
320 match self.base.db.set_val(&self.base.sdb, &key, &serialized) {
322 Ok(_) => Ok(true),
323 Err(e) => Err(KomerError::DBError(e)),
324 }
325 }
326
327 pub fn get<K>(&self, keys: &[K]) -> Result<Option<T>, KomerError>
336 where
337 K: AsRef<[u8]>,
338 {
339 self.base.get(keys)
340 }
341
342 pub fn get_json<K>(&self, keys: &[K]) -> Result<Option<serde_json::Value>, KomerError>
352 where
353 K: AsRef<[u8]>,
354 {
355 if let Some(val) = self.get(keys)? {
356 match serde_json::to_value(&val) {
357 Ok(json_val) => Ok(Some(json_val)),
358 Err(e) => Err(KomerError::Serialization(e.to_string())),
359 }
360 } else {
361 Ok(None)
362 }
363 }
364
365 pub fn get_item_iter<K>(&self, keys: &[K]) -> Result<Vec<(Vec<String>, T)>, KomerError>
366 where
367 K: AsRef<[u8]>,
368 {
369 self.base.get_item_iter(keys)
370 }
371 pub fn rem<K>(&self, keys: &[K]) -> Result<bool, KomerError>
380 where
381 K: AsRef<[u8]>,
382 {
383 self.base.rem(keys)
384 }
385
386 pub fn trim<K>(&self, keys: &[K]) -> Result<bool, KomerError>
396 where
397 K: AsRef<[u8]>,
398 {
399 let key = self.base.to_key(keys, true);
400 Ok(self.base.db.del_top_val(&self.base.sdb, &key)?)
401 }
402
403 pub fn cnt_all(&self) -> Result<usize, KomerError> {
410 self.base.cnt_all()
411 }
412}
413
414#[cfg(test)]
415mod tests {
416 use super::*;
417 use serde::{Deserialize, Serialize};
418 use std::collections::HashMap;
419 use std::fmt::Debug;
420 use std::sync::Arc;
421
422 #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
423 struct Record {
424 first: String, last: String, street: String, city: String, state: String, zip: u32, }
431
432 #[test]
433 fn test_kom_happy_path() -> Result<(), Box<dyn std::error::Error>> {
434 let jim = Record {
436 first: "Jim".to_string(),
437 last: "Black".to_string(),
438 street: "100 Main Street".to_string(),
439 city: "Riverton".to_string(),
440 state: "UT".to_string(),
441 zip: 84058,
442 };
443
444 let jimser = serde_json::to_string(&jim)?;
445 let jim_deserialized: Record = serde_json::from_str(&jimser)?;
446 assert_eq!(jim_deserialized, jim);
447
448 let lmdber = LMDBer::builder().name("test_db").temp(true).build()?;
450 let db_ref = Arc::new(lmdber);
451
452 let mydb = Komer::<Record>::new(db_ref.clone(), "records.", SerialKind::Json)?;
454
455 let sue = Record {
460 first: "Susan".to_string(),
461 last: "Black".to_string(),
462 street: "100 Main Street".to_string(),
463 city: "Riverton".to_string(),
464 state: "UT".to_string(),
465 zip: 84058,
466 };
467
468 let keys = ["test_key", "0001"];
470 assert_eq!(mydb.base.sep, ".");
471
472 let key = mydb.base.to_key(&keys, false);
474 let str = std::str::from_utf8(&key)?;
475 assert_eq!(key, b"test_key.0001");
476 assert_eq!(
477 mydb.base.to_keys(&key),
478 vec!["test_key".to_string(), "0001".to_string()]
479 );
480
481 mydb.put(&keys, &sue)?;
483 let actual = mydb.get(&keys)?.unwrap();
484
485 assert_eq!(actual.first, "Susan");
486 assert_eq!(actual.last, "Black");
487 assert_eq!(actual.street, "100 Main Street");
488 assert_eq!(actual.city, "Riverton");
489 assert_eq!(actual.state, "UT");
490 assert_eq!(actual.zip, 84058);
491
492 mydb.rem(&keys)?;
494 let actual = mydb.get(&keys)?;
495 assert!(actual.is_none());
496
497 let keys = ["test_key", "0001"];
499 mydb.put(&keys, &sue)?;
500 let actual = mydb.get(&keys)?.unwrap();
501 assert_eq!(actual, sue);
502
503 let kip = Record {
505 first: "Kip".to_string(),
506 last: "Thorne".to_string(),
507 street: "200 Center Street".to_string(),
508 city: "Bluffdale".to_string(),
509 state: "UT".to_string(),
510 zip: 84043,
511 };
512
513 let result = mydb.put(&keys, &kip)?;
514 assert!(!result);
515 let actual = mydb.get(&keys)?.unwrap();
516 assert_eq!(actual, sue);
517
518 let actual_json = mydb.get_json(&keys)?.unwrap();
520 let expected_json = serde_json::to_value(&sue)?;
521 assert_eq!(actual_json, expected_json);
522
523 let result = mydb.pin(&keys, &kip)?;
525 assert!(result);
526 let actual = mydb.get(&keys)?.unwrap();
527 assert_eq!(actual, kip);
528
529 let keys = ["keystr"];
531
532 let bob = Record {
533 first: "Bob".to_string(),
534 last: "Brown".to_string(),
535 street: "100 Center Street".to_string(),
536 city: "Bluffdale".to_string(),
537 state: "UT".to_string(),
538 zip: 84043,
539 };
540
541 mydb.put(&keys, &bob)?;
542 let actual = mydb.get(&keys)?.unwrap();
543
544 assert_eq!(actual.first, "Bob");
545 assert_eq!(actual.last, "Brown");
546 assert_eq!(actual.street, "100 Center Street");
547 assert_eq!(actual.city, "Bluffdale");
548 assert_eq!(actual.state, "UT");
549 assert_eq!(actual.zip, 84043);
550
551 let actual_json = mydb.get_json(&keys)?.unwrap();
553 let expected_json = serde_json::to_value(&bob)?;
554 assert_eq!(actual_json, expected_json);
555
556 let nonexistent_keys = ["bla", "bal"];
558 assert!(mydb.get_json(&nonexistent_keys)?.is_none());
559
560 mydb.rem(&keys)?;
562 let actual = mydb.get(&keys)?;
563 assert!(actual.is_none());
564
565 drop(db_ref);
567
568 Ok(())
570 }
571
572 #[test]
573 fn test_komer_error_handling() -> Result<(), Box<dyn std::error::Error>> {
574 let lmdber = LMDBer::builder().name("test_db").temp(true).build()?;
576 let db_ref = Arc::new(lmdber);
577
578 let mydb = Komer::<Record>::new(db_ref.clone(), "records.", SerialKind::Json)?;
580
581 let empty_keys: [&str; 0] = [];
582 let record = Record {
583 first: "Test".to_string(),
584 last: "User".to_string(),
585 street: "123 Test St".to_string(),
586 city: "Testville".to_string(),
587 state: "TS".to_string(),
588 zip: 12345,
589 };
590
591 let result = mydb.put(&empty_keys, &record);
593 assert!(matches!(result, Err(KomerError::EmptyKeys)));
594
595 Ok(())
596 }
597
598 #[test]
599 fn test_serialization_formats() -> Result<(), Box<dyn std::error::Error>> {
600 let lmdber = LMDBer::builder().name("test_db").temp(true).build()?;
602 let db_ref = Arc::new(lmdber);
603
604 let record = Record {
606 first: "Test".to_string(),
607 last: "User".to_string(),
608 street: "123 Test St".to_string(),
609 city: "Testville".to_string(),
610 state: "TS".to_string(),
611 zip: 12345,
612 };
613
614 for serial_kind in [SerialKind::Json, SerialKind::MsgPack, SerialKind::Cbor] {
616 let mydb = Komer::<Record>::new(
617 db_ref.clone(),
618 &format!("records_{:?}.", serial_kind),
619 serial_kind,
620 )?;
621
622 let keys = ["test_key"];
623 mydb.put(&keys, &record)?;
624 let retrieved = mydb.get(&keys)?.unwrap();
625
626 assert_eq!(retrieved, record);
627 }
628
629 Ok(())
630 }
631
632 #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
633 struct Stuff {
634 a: String,
635 b: String,
636 }
637
638 impl Stuff {
639 pub fn to_map(&self) -> HashMap<String, String> {
641 let mut map = HashMap::new();
642 map.insert("a".to_string(), self.a.clone());
643 map.insert("b".to_string(), self.b.clone());
644 map
645 }
646 }
647
648 #[test]
649 fn test_kom_get_item_iter() -> Result<(), Box<dyn std::error::Error>> {
650 let w = Stuff {
652 a: "Big".to_string(),
653 b: "Blue".to_string(),
654 };
655 let x = Stuff {
656 a: "Tall".to_string(),
657 b: "Red".to_string(),
658 };
659 let y = Stuff {
660 a: "Fat".to_string(),
661 b: "Green".to_string(),
662 };
663 let z = Stuff {
664 a: "Eat".to_string(),
665 b: "White".to_string(),
666 };
667
668 let lmdber = LMDBer::builder().name("test").temp(true).build()?;
670
671 let db_ref = Arc::new(lmdber);
672 assert_eq!(db_ref.name(), "test");
673 assert!(db_ref.opened());
674
675 let mydb = Komer::<Stuff>::new(db_ref.clone(), "recs.", SerialKind::Json)?;
677
678 mydb.put(&["a", "1"], &w)?;
680 mydb.put(&["a", "2"], &x)?;
681 mydb.put(&["a", "3"], &y)?;
682 mydb.put(&["a", "4"], &z)?;
683
684 let items: Vec<(Vec<String>, HashMap<String, String>)> = mydb
686 .get_item_iter(&[] as &[&str])?
687 .into_iter()
688 .map(|(keys, data)| (keys, data.to_map()))
689 .collect();
690
691 assert_eq!(
692 items,
693 vec![
694 (vec!["a".to_string(), "1".to_string()], w.to_map()),
695 (vec!["a".to_string(), "2".to_string()], x.to_map()),
696 (vec!["a".to_string(), "3".to_string()], y.to_map()),
697 (vec!["a".to_string(), "4".to_string()], z.to_map()),
698 ]
699 );
700
701 mydb.put(&["b", "1"], &w)?;
703 mydb.put(&["b", "2"], &x)?;
704 mydb.put(&["bc", "3"], &y)?;
705 mydb.put(&["bc", "4"], &z)?;
706
707 let topkeys = ["b", ""];
709 let items: Vec<(Vec<String>, HashMap<String, String>)> = mydb
710 .get_item_iter(&topkeys)?
711 .into_iter()
712 .map(|(keys, data)| (keys, data.to_map()))
713 .collect();
714
715 assert_eq!(
716 items,
717 vec![
718 (vec!["b".to_string(), "1".to_string()], w.to_map()),
719 (vec!["b".to_string(), "2".to_string()], x.to_map()),
720 ]
721 );
722
723 let items: Vec<(Vec<String>, HashMap<String, String>)> = mydb
725 .get_item_iter(&[] as &[&str])?
726 .into_iter()
727 .map(|(keys, data)| (keys, data.to_map()))
728 .collect();
729
730 assert_eq!(
731 items,
732 vec![
733 (vec!["a".to_string(), "1".to_string()], w.to_map()),
734 (vec!["a".to_string(), "2".to_string()], x.to_map()),
735 (vec!["a".to_string(), "3".to_string()], y.to_map()),
736 (vec!["a".to_string(), "4".to_string()], z.to_map()),
737 (vec!["b".to_string(), "1".to_string()], w.to_map()),
738 (vec!["b".to_string(), "2".to_string()], x.to_map()),
739 (vec!["bc".to_string(), "3".to_string()], y.to_map()),
740 (vec!["bc".to_string(), "4".to_string()], z.to_map()),
741 ]
742 );
743
744 assert_eq!(mydb.cnt_all()?, 8);
746
747 assert!(mydb.trim(&["b", ""])?);
749
750 let items: Vec<(Vec<String>, HashMap<String, String>)> = mydb
751 .get_item_iter(&[] as &[&str])?
752 .into_iter()
753 .map(|(keys, data)| (keys, data.to_map()))
754 .collect();
755
756 assert_eq!(
757 items,
758 vec![
759 (vec!["a".to_string(), "1".to_string()], w.to_map()),
760 (vec!["a".to_string(), "2".to_string()], x.to_map()),
761 (vec!["a".to_string(), "3".to_string()], y.to_map()),
762 (vec!["a".to_string(), "4".to_string()], z.to_map()),
763 (vec!["bc".to_string(), "3".to_string()], y.to_map()),
764 (vec!["bc".to_string(), "4".to_string()], z.to_map()),
765 ]
766 );
767
768 assert!(mydb.trim::<[u8; 0]>(&[])?);
770
771 let items: Vec<(Vec<String>, Stuff)> = mydb.get_item_iter(&[] as &[&str])?;
772
773 assert_eq!(items, vec![]);
774
775 drop(db_ref);
777
778 Ok(())
782 }
783
784 #[test]
785 fn test_kom_put_get() -> Result<(), Box<dyn std::error::Error>> {
786 let lmdber = LMDBer::builder().name("test").temp(true).build()?;
788
789 let db_ref = Arc::new(lmdber);
790
791 let mydb = Komer::<Stuff>::new(db_ref.clone(), "recs.", SerialKind::Json)?;
793
794 let data = Stuff {
795 a: "Test".to_string(),
796 b: "Value".to_string(),
797 };
798
799 assert!(mydb.put(&["test", "key"], &data)?);
801
802 let retrieved = mydb.get(&["test", "key"])?.unwrap();
804 assert_eq!(retrieved, data);
805
806 assert!(mydb.get(&["nonexistent", "key"])?.is_none());
808
809 let new_data = Stuff {
811 a: "Modified".to_string(),
812 b: "Data".to_string(),
813 };
814
815 assert!(!mydb.put(&["test", "key"], &new_data)?);
816
817 let retrieved = mydb.get(&["test", "key"])?.unwrap();
819 assert_eq!(retrieved, data);
820
821 assert!(mydb.pin(&["test", "key"], &new_data)?);
823
824 let retrieved = mydb.get(&["test", "key"])?.unwrap();
826 assert_eq!(retrieved, new_data);
827
828 assert!(mydb.rem(&["test", "key"])?);
830 assert!(mydb.get(&["test", "key"])?.is_none());
831
832 assert!(!mydb.rem(&["test", "key"])?);
834
835 Ok(())
836 }
837
838 #[test]
839 fn test_kom_serialization_formats() -> Result<(), Box<dyn std::error::Error>> {
840 let lmdber = LMDBer::builder().name("test").temp(true).build()?;
842
843 let db_ref = Arc::new(lmdber);
844
845 let test_data = Stuff {
846 a: "Test".to_string(),
847 b: "Value".to_string(),
848 };
849
850 for format in [SerialKind::Json, SerialKind::MsgPack, SerialKind::Cbor].iter() {
852 let db_name = format!("test_{:?}", format);
853 let mydb = Komer::<Stuff>::new(db_ref.clone(), &db_name, *format)?;
854
855 mydb.put(&["test"], &test_data)?;
857 let retrieved = mydb.get(&["test"])?.unwrap();
858
859 assert_eq!(retrieved, test_data);
860 }
861
862 Ok(())
863 }
864
865 #[test]
866 fn test_kom_empty_keys() -> Result<(), Box<dyn std::error::Error>> {
867 let lmdber = LMDBer::builder().name("test").temp(true).build()?;
869
870 let db_ref = Arc::new(lmdber);
871
872 let mydb = Komer::<Stuff>::new(db_ref.clone(), "recs.", SerialKind::Json)?;
874
875 let data = Stuff {
876 a: "Test".to_string(),
877 b: "Value".to_string(),
878 };
879
880 let empty_keys: [&str; 0] = [];
882 let result = mydb.put(&empty_keys, &data);
883
884 assert!(matches!(result, Err(KomerError::EmptyKeys)));
885
886 Ok(())
887 }
888}