Skip to main content

libkeri/keri/db/subing/
dup.rs

1use crate::keri::db::dbing::LMDBer;
2use crate::keri::db::subing::{SuberBase, SuberError, Utf8Codec, ValueCodec};
3use std::sync::Arc;
4
5/// Represents a Duplicate Suber, allowing multiple values (duplicates) at each key.
6///
7/// Do not use if serialized value is greater than 511 bytes.
8/// This is a limitation of dupsort==True sub dbs in LMDB.
9pub struct DupSuber<C: ValueCodec = Utf8Codec> {
10    pub base: SuberBase<C>,
11}
12
13impl<C: ValueCodec> DupSuber<C> {
14    /// Creates a new `DupSuber`.
15    pub fn new(
16        db: Arc<LMDBer>,
17        subkey: &str,
18        sep: Option<u8>,
19        verify: bool,
20    ) -> Result<Self, SuberError> {
21        // Always use dupsort=True for DupSuber
22        let base = SuberBase::new(db, subkey, sep, verify, Some(true))?;
23
24        Ok(Self { base })
25    }
26
27    pub fn is_dupsort(&self) -> bool {
28        self.base.is_dupsort()
29    }
30
31    /// Puts all vals at key made from keys. Does not overwrite. Adds to existing
32    /// dup values at key if any. Duplicate means another entry at the same key
33    /// but the entry is still a unique value. Duplicates are inserted in
34    /// lexicographic order not insertion order. LMDB does not insert a duplicate
35    /// unless it is a unique value for that key.
36    pub fn put<K: AsRef<[u8]>, V: ?Sized + Clone + Into<Vec<u8>>>(
37        &self,
38        keys: &[K],
39        vals: &[&V],
40    ) -> Result<bool, SuberError> {
41        let key = self.base.to_key(keys, false);
42
43        // Serialize all values
44        let serialized_vals: Vec<Vec<u8>> = vals
45            .iter()
46            .map(|v| self.base.ser(*v))
47            .collect::<Result<Vec<Vec<u8>>, _>>()?;
48
49        // Create Vec<&[u8]> by borrowing from serialized_vals
50        let val_slices: Vec<&[u8]> = serialized_vals.iter().map(|v| v.as_slice()).collect();
51
52        // Use the put_vals method for dup databases
53        self.base
54            .db
55            .put_vals(&self.base.sdb, &key, &val_slices)
56            .map_err(SuberError::DBError)
57    }
58
59    /// Add val to vals at key made from keys. Does not overwrite. Adds to existing
60    /// dup values at key if any. Duplicate means another entry at the same key
61    /// but the entry is still a unique value. Duplicates are inserted in
62    /// lexicographic order not insertion order. LMDB does not insert a duplicate
63    /// unless it is a unique value for that key.
64    pub fn add<K: AsRef<[u8]>, V: ?Sized + Clone + Into<Vec<u8>>>(
65        &self,
66        keys: &[K],
67        val: &V,
68    ) -> Result<bool, SuberError> {
69        let key = self.base.to_key(keys, false);
70        let sval = self.base.ser(val)?;
71
72        self.base
73            .db
74            .add_val(&self.base.sdb, &key, &sval)
75            .map_err(SuberError::DBError)
76    }
77
78    /// Pins (sets) vals at key made from keys. Overwrites. Removes all
79    /// pre-existing dup vals and replaces them with vals.
80    pub fn pin<K: AsRef<[u8]>, V: ?Sized + Clone + Into<Vec<u8>>>(
81        &self,
82        keys: &[K],
83        vals: &[&V],
84    ) -> Result<bool, SuberError> {
85        let key = self.base.to_key(keys, false);
86
87        // First delete all values at the key
88        self.base
89            .db
90            .del_vals(&self.base.sdb, &key, None)
91            .map_err(SuberError::DBError)?;
92
93        // If we have no values to add, just return true (successful deletion)
94        if vals.is_empty() {
95            return Ok(true);
96        }
97
98        // Serialize all values
99        let serialized_vals: Vec<Vec<u8>> = vals
100            .iter()
101            .map(|v| self.base.ser(*v))
102            .collect::<Result<Vec<Vec<u8>>, _>>()?;
103
104        // Create Vec<&[u8]> by borrowing from serialized_vals
105        let val_slices: Vec<&[u8]> = serialized_vals.iter().map(|v| v.as_slice()).collect();
106
107        // Add the new values
108        self.base
109            .db
110            .put_vals(&self.base.sdb, &key, &val_slices)
111            .map_err(SuberError::DBError)
112    }
113
114    /// Gets dup vals list at key made from keys.
115    pub fn get<K: AsRef<[u8]>, R: TryFrom<Vec<u8>>>(&self, keys: &[K]) -> Result<Vec<R>, SuberError>
116    where
117        <R as TryFrom<Vec<u8>>>::Error: std::fmt::Debug,
118    {
119        let key = self.base.to_key(keys, false);
120        let mut raw_vals: Vec<Vec<u8>> = Vec::new();
121
122        self.base
123            .db
124            .get_vals_iter(&self.base.sdb, &key, |val| {
125                raw_vals.push(val.to_vec());
126                Ok(true)
127            })
128            .map_err(SuberError::DBError)?;
129
130        raw_vals
131            .iter()
132            .map(|raw_val| self.base.des(raw_val))
133            .collect() // Collects into Result<Vec<R>, SuberError>
134    }
135
136    /// Gets the last duplicate value at key made from keys
137    ///
138    /// # Arguments
139    ///
140    /// * `keys`: An array slice of key parts to be combined to form key
141    ///
142    /// # Returns
143    ///
144    /// `Result<Option<R>, SuberError>` - Deserialized value if found, None if no value at key,
145    /// or an error if deserialization fails
146    pub fn get_last<K: AsRef<[u8]>, R: TryFrom<Vec<u8>>>(
147        &self,
148        keys: &[K],
149    ) -> Result<Option<R>, SuberError>
150    where
151        <R as TryFrom<Vec<u8>>>::Error: std::fmt::Debug,
152    {
153        let key = self.base.to_key(keys, false);
154        let raw_val_opt = self
155            .base
156            .db
157            .get_val_last(&self.base.sdb, &key)
158            .map_err(SuberError::DBError)?;
159
160        match raw_val_opt {
161            Some(val) => self.base.des(&val).map(Some),
162            None => Ok(None),
163        }
164    }
165
166    /// Gets dup vals iterator at key made from keys.
167    /// Duplicates are retrieved in lexicographic order not insertion order.
168    pub fn get_iter<K: AsRef<[u8]>, R: TryFrom<Vec<u8>> + 'static>(
169        &self,
170        keys: &[K],
171    ) -> Result<impl Iterator<Item = Result<R, SuberError>>, SuberError>
172    where
173        <R as TryFrom<Vec<u8>>>::Error: std::fmt::Debug,
174        SuberError: From<<R as TryFrom<Vec<u8>>>::Error>,
175    {
176        let key = self.base.to_key(keys, false);
177        let mut results: Vec<Result<R, SuberError>> = Vec::new();
178
179        self.base
180            .db
181            .get_vals_iter(&self.base.sdb, &key, |val| {
182                let result = self.base.des(val);
183                results.push(result);
184                Ok(true)
185            })
186            .map_err(SuberError::DBError)?;
187
188        Ok(results.into_iter())
189    }
190
191    pub fn get_item_iter<K: AsRef<[u8]>>(
192        &self,
193        keys: &[K],
194        topive: bool,
195    ) -> Result<Vec<(Vec<Vec<u8>>, Vec<u8>)>, SuberError> {
196        Ok(self.base.get_item_iter(keys, topive)?)
197    }
198
199    /// Return count of dup values at key made from keys, zero otherwise
200    pub fn cnt<K: AsRef<[u8]>>(&self, keys: &[K]) -> Result<usize, SuberError> {
201        let key = self.base.to_key(keys, false);
202        self.base
203            .db
204            .cnt_vals(&self.base.sdb, &key)
205            .map_err(SuberError::DBError)
206    }
207
208    /// Removes entry or specific value at keys
209    ///
210    /// If val is provided, remove only that value at the key.
211    /// If val is None, remove all values at the key.
212    pub fn rem<K: AsRef<[u8]>, V: ?Sized + Clone + Into<Vec<u8>>>(
213        &self,
214        keys: &[K],
215        val: Option<&V>,
216    ) -> Result<bool, SuberError> {
217        let key = self.base.to_key(keys, false);
218
219        match val {
220            Some(v) => {
221                let sval = self.base.ser(v)?;
222                self.base
223                    .db
224                    .del_vals(&self.base.sdb, &key, Some(&sval))
225                    .map_err(SuberError::DBError)
226            }
227            None => self
228                .base
229                .db
230                .del_vals(&self.base.sdb, &key, None)
231                .map_err(SuberError::DBError),
232        }
233    }
234}
235
236#[cfg(test)]
237mod tests {
238    use super::*;
239    use crate::keri::db::dbing::keys::on_key;
240
241    #[test]
242    fn test_dup_suber() -> Result<(), SuberError> {
243        // Create a temporary database
244        let db = LMDBer::builder()
245            .temp(true)
246            .name("test")
247            .build()
248            .map_err(SuberError::DBError)?;
249        let db_ref = Arc::new(db);
250
251        // Create a DupSuber
252        let dupber: DupSuber<Utf8Codec> = DupSuber::new(db_ref.clone(), "bags.", None, false)?;
253
254        assert!(dupber.is_dupsort());
255
256        // Define test values
257        let sue = "Hello sailer!";
258        let sal = "Not my type.";
259
260        // Test keys
261        let keys0 = ["test_key", "0001"];
262        let keys1 = ["test_key", "0002"];
263
264        // Test put and get
265        dupber.put(&keys0, &[&sue, &sal])?;
266        let bytes: Vec<Vec<u8>> = dupber.get(&keys0)?;
267        assert_eq!(bytes.len(), 2);
268        // Compare individual strings
269        assert_eq!(String::from_utf8(bytes[0].clone()).unwrap(), sue);
270        assert_eq!(String::from_utf8(bytes[1].clone()).unwrap(), sal);
271        assert_eq!(dupber.cnt(&keys0)?, 2);
272
273        // Test rem
274        dupber.rem(&keys0, None::<&String>)?;
275        let bytes: Vec<Vec<u8>> = dupber.get(&keys0)?;
276        assert!(bytes.is_empty());
277        assert_eq!(dupber.cnt(&keys0)?, 0);
278
279        // Test put and get with different order - should still sort lexicographically
280        dupber.put(&keys0, &[&sal, &sue])?;
281        let bytes: Vec<Vec<u8>> = dupber.get(&keys0)?;
282        assert_eq!(bytes.len(), 2);
283        assert_eq!(String::from_utf8(bytes[0].clone()).unwrap(), sue);
284        assert_eq!(String::from_utf8(bytes[1].clone()).unwrap(), sal);
285
286        // Test getLast
287        let last_bytes: Option<Vec<u8>> = dupber.get_last(&keys0)?;
288        assert!(last_bytes.is_some());
289        assert_eq!(String::from_utf8(last_bytes.unwrap()).unwrap(), sal);
290
291        // Test add
292        let sam = "A real charmer!";
293        let result = dupber.add(&keys0, &sam)?;
294        assert!(result);
295        let bytes: Vec<Vec<u8>> = dupber.get(&keys0)?;
296        assert_eq!(bytes.len(), 3);
297        assert_eq!(String::from_utf8(bytes[0].clone()).unwrap(), sam);
298        assert_eq!(String::from_utf8(bytes[1].clone()).unwrap(), sue);
299        assert_eq!(String::from_utf8(bytes[2].clone()).unwrap(), sal);
300
301        // Test pin (replaces existing values)
302        let zoe = "See ya later.";
303        let zia = "Hey gorgeous!";
304        let result = dupber.pin(&keys0, &[&zoe, &zia])?;
305        assert!(result);
306        let bytes: Vec<Vec<u8>> = dupber.get(&keys0)?;
307        assert_eq!(bytes.len(), 2);
308        assert_eq!(String::from_utf8(bytes[0].clone()).unwrap(), zia);
309        assert_eq!(String::from_utf8(bytes[1].clone()).unwrap(), zoe);
310
311        // Test with more values
312        dupber.put(&keys1, &[&sal, &sue, &sam])?;
313        let bytes: Vec<Vec<u8>> = dupber.get(&keys1)?;
314        assert_eq!(bytes.len(), 3);
315        assert_eq!(String::from_utf8(bytes[0].clone()).unwrap(), sam);
316        assert_eq!(String::from_utf8(bytes[1].clone()).unwrap(), sue);
317        assert_eq!(String::from_utf8(bytes[2].clone()).unwrap(), sal);
318
319        // Test iterator
320        let iter = dupber.get_iter::<_, Vec<u8>>(&keys1)?;
321        let iter_bytes: Vec<Vec<u8>> = iter.collect::<Result<Vec<_>, _>>()?;
322        assert_eq!(iter_bytes.len(), 3);
323        assert_eq!(String::from_utf8(iter_bytes[0].clone()).unwrap(), sam);
324        assert_eq!(String::from_utf8(iter_bytes[1].clone()).unwrap(), sue);
325        assert_eq!(String::from_utf8(iter_bytes[2].clone()).unwrap(), sal);
326
327        // Test get_item_iter with correct parameters
328        let items: Vec<(Vec<Vec<u8>>, Vec<u8>)> = dupber.get_item_iter::<&str>(&[], true)?;
329
330        // Convert the bytes to strings for easier assertions
331        let items_as_strings: Vec<(Vec<String>, String)> = items
332            .into_iter()
333            .map(|(keys_bytes, val_bytes)| {
334                let parsed_keys = keys_bytes
335                    .into_iter()
336                    .map(|k| String::from_utf8(k).unwrap())
337                    .collect();
338                (parsed_keys, String::from_utf8(val_bytes).unwrap())
339            })
340            .collect();
341
342        // Assert that we have the expected items
343        assert_eq!(items_as_strings.len(), 5);
344        assert!(items_as_strings.contains(&(
345            vec!["test_key".to_string(), "0001".to_string()],
346            "Hey gorgeous!".to_string()
347        )));
348        assert!(items_as_strings.contains(&(
349            vec!["test_key".to_string(), "0001".to_string()],
350            "See ya later.".to_string()
351        )));
352        assert!(items_as_strings.contains(&(
353            vec!["test_key".to_string(), "0002".to_string()],
354            "A real charmer!".to_string()
355        )));
356        assert!(items_as_strings.contains(&(
357            vec!["test_key".to_string(), "0002".to_string()],
358            "Hello sailer!".to_string()
359        )));
360        assert!(items_as_strings.contains(&(
361            vec!["test_key".to_string(), "0002".to_string()],
362            "Not my type.".to_string()
363        )));
364
365        // Add test with ("test", "blue") keys as in the Python test
366        assert!(dupber.put(&["test", "blue"], &[&sal, &sue, &sam])?);
367
368        // Test getting items with a prefix - similar to the Python test's usage of topkeys
369        let topkeys = ["test", ""];
370        let prefix_items: Vec<(Vec<Vec<u8>>, Vec<u8>)> = dupber.get_item_iter(&topkeys, true)?;
371
372        // Convert to strings for easier assertions
373        let prefix_items_as_strings: Vec<(Vec<String>, String)> = prefix_items
374            .into_iter()
375            .map(|(keys_bytes, val_bytes)| {
376                let parsed_keys = keys_bytes
377                    .into_iter()
378                    .map(|k| String::from_utf8(k).unwrap())
379                    .collect();
380                (parsed_keys, String::from_utf8(val_bytes).unwrap())
381            })
382            .collect();
383
384        // Assert we have the correct items with the test prefix
385        assert_eq!(prefix_items_as_strings.len(), 3);
386        assert!(prefix_items_as_strings.contains(&(
387            vec!["test".to_string(), "blue".to_string()],
388            "A real charmer!".to_string()
389        )));
390        assert!(prefix_items_as_strings.contains(&(
391            vec!["test".to_string(), "blue".to_string()],
392            "Hello sailer!".to_string()
393        )));
394        assert!(prefix_items_as_strings.contains(&(
395            vec!["test".to_string(), "blue".to_string()],
396            "Not my type.".to_string()
397        )));
398
399        // Test with string key (not tuple)
400        let keys2 = ["keystr"];
401        let bob = "Shove off!";
402        dupber.put(&keys2, &[&bob])?;
403        let bytes: Vec<Vec<u8>> = dupber.get(&keys2)?;
404        assert_eq!(bytes.len(), 1);
405        assert_eq!(String::from_utf8(bytes[0].clone()).unwrap(), bob);
406        assert_eq!(dupber.cnt(&keys2)?, 1);
407
408        // Test removal of string key
409        dupber.rem(&keys2, None::<&String>)?;
410        let bytes: Vec<Vec<u8>> = dupber.get(&keys2)?;
411        assert!(bytes.is_empty());
412        assert_eq!(dupber.cnt(&keys2)?, 0);
413
414        // Test put and pin with string key
415        dupber.put(&keys2, &[&bob])?;
416        let bytes: Vec<Vec<u8>> = dupber.get(&keys2)?;
417        assert_eq!(bytes.len(), 1);
418        assert_eq!(String::from_utf8(bytes[0].clone()).unwrap(), bob);
419
420        let bil = "Go away.";
421        dupber.pin(&keys2, &[&bil])?;
422        let bytes: Vec<Vec<u8>> = dupber.get(&keys2)?;
423        assert_eq!(bytes.len(), 1);
424        assert_eq!(String::from_utf8(bytes[0].clone()).unwrap(), bil);
425
426        // Test add with string key
427        dupber.add(&keys2, &bob)?;
428        let bytes: Vec<Vec<u8>> = dupber.get(&keys2)?;
429        assert_eq!(bytes.len(), 2);
430        assert_eq!(String::from_utf8(bytes[0].clone()).unwrap(), bil);
431        assert_eq!(String::from_utf8(bytes[1].clone()).unwrap(), bob);
432
433        // Test with multiple values per key using on_key
434        let vals1 = ["hi", "me", "my"];
435        for (i, val) in vals1.iter().enumerate() {
436            let key = on_key("bob", i as u64, None);
437            assert!(dupber.put(&[&key], &[val])?);
438        }
439
440        let vals2 = ["bye", "guy", "gal"];
441        for (i, val) in vals2.iter().enumerate() {
442            let key = on_key("bob", i as u64, None);
443            assert!(dupber.put(&[&key], &[val])?);
444        }
445
446        // Test getting items with "bob" prefix
447        let bob_items: Vec<(Vec<Vec<u8>>, Vec<u8>)> = dupber.get_item_iter(&["bob"], true)?;
448
449        // Convert to strings for easier assertions
450        let bob_items_as_strings: Vec<(Vec<String>, String)> = bob_items
451            .into_iter()
452            .map(|(keys_bytes, val_bytes)| {
453                let parsed_keys = keys_bytes
454                    .into_iter()
455                    .map(|k| String::from_utf8(k).unwrap())
456                    .collect();
457                (parsed_keys, String::from_utf8(val_bytes).unwrap())
458            })
459            .collect();
460
461        // Check all "bob" prefix items match expected
462        assert_eq!(bob_items_as_strings.len(), 6);
463        assert!(bob_items_as_strings.contains(&(
464            vec![
465                "bob".to_string(),
466                "00000000000000000000000000000000".to_string()
467            ],
468            "bye".to_string()
469        )));
470        assert!(bob_items_as_strings.contains(&(
471            vec![
472                "bob".to_string(),
473                "00000000000000000000000000000000".to_string()
474            ],
475            "hi".to_string()
476        )));
477        assert!(bob_items_as_strings.contains(&(
478            vec![
479                "bob".to_string(),
480                "00000000000000000000000000000001".to_string()
481            ],
482            "guy".to_string()
483        )));
484        assert!(bob_items_as_strings.contains(&(
485            vec![
486                "bob".to_string(),
487                "00000000000000000000000000000001".to_string()
488            ],
489            "me".to_string()
490        )));
491        assert!(bob_items_as_strings.contains(&(
492            vec![
493                "bob".to_string(),
494                "00000000000000000000000000000002".to_string()
495            ],
496            "gal".to_string()
497        )));
498        assert!(bob_items_as_strings.contains(&(
499            vec![
500                "bob".to_string(),
501                "00000000000000000000000000000002".to_string()
502            ],
503            "my".to_string()
504        )));
505
506        // Test count of values for a specific key
507        let key1 = on_key("bob", 1, None);
508        assert_eq!(dupber.cnt(&[&key1])?, 2);
509
510        // Test get iterator for specific key
511        let key2 = on_key("bob", 2, None);
512        let iter = dupber.get_iter::<_, Vec<u8>>(&[&key2])?;
513        let vals: Vec<Vec<u8>> = iter.collect::<Result<Vec<_>, _>>()?;
514        // Then convert to strings if needed
515        let vals_as_strings: Vec<String> = vals
516            .into_iter()
517            .map(|bytes| String::from_utf8(bytes).unwrap())
518            .collect();
519
520        assert_eq!(vals_as_strings.len(), 2);
521        assert_eq!(vals_as_strings[0], "gal");
522        assert_eq!(vals_as_strings[1], "my");
523
524        // The database should be removed when db goes out of scope
525        Ok(())
526    }
527}