Skip to main content

libkeri/keri/db/dbing/
mod.rs

1pub mod keys;
2
3use crate::keri::core::filing::{BaseFiler, Filer, FilerDefaults};
4use crate::keri::db::dbing::keys::{on_key, split_on_key, suffix, unsuffix};
5use crate::keri::db::errors::DBError;
6use heed::{Database, DatabaseFlags, Env, EnvOpenOptions};
7use std::collections::HashSet;
8use std::fs;
9use std::ops::Bound;
10use std::path::PathBuf;
11use std::sync::Arc;
12
13const MAX_ON: u64 = u64::MAX;
14
15impl LMDBer {
16    pub fn builder() -> LMDBerBuilder {
17        LMDBerBuilder::default()
18    }
19}
20
21pub struct LMDBerBuilder {
22    name: String,
23    base: String,
24    temp: bool,
25    reopen: bool,
26    head_dir: Option<PathBuf>,
27    // other fields...
28}
29
30impl Default for LMDBerBuilder {
31    fn default() -> Self {
32        Self {
33            name: "test".to_string(),
34            base: String::new(),
35            temp: true,
36            reopen: true, // other defaults
37            head_dir: None,
38        }
39    }
40}
41
42impl LMDBerBuilder {
43    pub fn name(mut self, name: impl Into<String>) -> Self {
44        self.name = name.into();
45        self
46    }
47
48    pub fn temp(mut self, temp: bool) -> Self {
49        self.temp = temp;
50        self
51    }
52
53    pub fn reopen(mut self, reopen: bool) -> Self {
54        self.reopen = reopen;
55        self
56    }
57
58    /// Extra directory level between the db root and `name` (keripy `base`).
59    pub fn base(mut self, base: impl Into<String>) -> Self {
60        self.base = base.into();
61        self
62    }
63
64    /// Root directory to create the database under instead of the default
65    /// (`/usr/local/var/keri` or `~/.keri`).
66    pub fn head_dir(mut self, head_dir: impl Into<PathBuf>) -> Self {
67        self.head_dir = Some(head_dir.into());
68        self
69    }
70
71    // other setters
72
73    pub fn build(self) -> Result<LMDBer, DBError> {
74        // Create and return an LMDBer instance
75        LMDBer::new(
76            self.name,
77            self.base,
78            self.temp,
79            self.head_dir,
80            None,        // perm
81            self.reopen, // reopen
82            false,       // clear
83            false,       // reuse
84            false,       // clean
85            false,       // filed
86            false,       // extensioned
87            None,        // mode
88            None,        // fext
89            false,       // readonly
90        )
91    }
92}
93
94// Define our database type
95// Using a type alias for a database that stores bytes as both keys and values
96pub type BytesDatabase = Database<heed::types::Bytes, heed::types::Bytes>;
97
98/// LMDBer is a wrapper around LMDB database providing an interface similar to Filer
99pub struct LMDBer {
100    /// Base Filer instance
101    pub filer: BaseFiler,
102
103    /// LMDB environment
104    pub env: Option<Arc<Env>>,
105
106    /// Whether the database is opened in readonly mode
107    pub readonly: bool,
108
109    /// Version of the database
110    version: Option<String>,
111}
112
113impl LMDBer {
114    pub fn name(&self) -> String {
115        self.filer.name()
116    }
117
118    pub fn base(&self) -> String {
119        self.filer.base()
120    }
121
122    pub fn opened(&self) -> bool {
123        self.filer.opened()
124    }
125
126    pub fn temp(&self) -> bool {
127        self.filer.temp()
128    }
129
130    pub fn env(&self) -> Option<&Arc<Env>> {
131        self.env.as_ref()
132    }
133
134    pub fn path(&self) -> Option<PathBuf> {
135        self.filer.path()
136    }
137}
138
139impl Filer for LMDBer {
140    fn defaults() -> FilerDefaults {
141        let base_defaults = FilerDefaults::default();
142        FilerDefaults {
143            head_dir_path: base_defaults.head_dir_path,
144            tail_dir_path: Self::TAIL_DIR_PATH,
145            clean_tail_dir_path: Self::CLEAN_TAIL_DIR_PATH,
146            alt_head_dir_path: base_defaults.alt_head_dir_path,
147            alt_tail_dir_path: Self::ALT_TAIL_DIR_PATH,
148            alt_clean_tail_dir_path: Self::ALT_CLEAN_TAIL_DIR_PATH,
149            temp_head_dir: base_defaults.temp_head_dir,
150            temp_prefix: Self::TEMP_PREFIX,
151            temp_suffix: base_defaults.temp_suffix,
152            perm: base_defaults.perm,
153            mode: base_defaults.mode,
154            fext: base_defaults.fext,
155        }
156    }
157
158    #[cfg(target_os = "windows")]
159    const TAIL_DIR_PATH: &'static str = "keri\\db";
160    #[cfg(not(target_os = "windows"))]
161    const TAIL_DIR_PATH: &'static str = "keri/db";
162
163    #[cfg(target_os = "windows")]
164    const CLEAN_TAIL_DIR_PATH: &'static str = "keri\\clean\\db";
165    #[cfg(not(target_os = "windows"))]
166    const CLEAN_TAIL_DIR_PATH: &'static str = "keri/clean/db";
167
168    #[cfg(target_os = "windows")]
169    const ALT_TAIL_DIR_PATH: &'static str = ".keri\\db";
170    #[cfg(not(target_os = "windows"))]
171    const ALT_TAIL_DIR_PATH: &'static str = ".keri/db";
172
173    #[cfg(target_os = "windows")]
174    const ALT_CLEAN_TAIL_DIR_PATH: &'static str = ".keri\\clean\\db";
175    #[cfg(not(target_os = "windows"))]
176    const ALT_CLEAN_TAIL_DIR_PATH: &'static str = ".keri/clean/db";
177
178    const TEMP_PREFIX: &'static str = "keri_lmdb_";
179}
180
181impl LMDBer {
182    // Constants specific to LMDBer
183    pub const MAX_NAMED_DBS: u32 = 96;
184    pub const MAP_SIZE: usize = 104857600; // 100MB
185
186    /// Create a new LMDBer instance
187    pub fn new<S1, S2>(
188        name: S1,
189        base: S2,
190        temp: bool,
191        head_dir_path: Option<PathBuf>,
192        perm: Option<u32>,
193        reopen: bool,
194        clear: bool,
195        reuse: bool,
196        clean: bool,
197        filed: bool,
198        extensioned: bool,
199        mode: Option<String>,
200        fext: Option<String>,
201        readonly: bool,
202    ) -> Result<Self, DBError>
203    where
204        S1: Into<String>,
205        S2: Into<String>,
206    {
207        let filer = BaseFiler::new(
208            name,
209            base,
210            temp,
211            head_dir_path,
212            perm,
213            reopen,
214            clear,
215            reuse,
216            clean,
217            filed,
218            extensioned,
219            mode,
220            fext,
221            Some(Self::defaults()),
222        );
223
224        let filer = match filer {
225            Ok(filer) => filer,
226            Err(e) => return Err(DBError::CoreError(format!("{}", e))),
227        };
228
229        let mut lmdber = LMDBer {
230            filer,
231            env: None,
232            readonly,
233            version: None,
234        };
235
236        if reopen {
237            lmdber.reopen(None, None, None, clear, reuse, clean, None, None)?;
238        }
239
240        Ok(lmdber)
241    }
242
243    /// Reopen the LMDB database
244    pub fn reopen(
245        &mut self,
246        temp: Option<bool>,
247        head_dir_path: Option<PathBuf>,
248        perm: Option<u32>,
249        clear: bool,
250        reuse: bool,
251        clean: bool,
252        mode: Option<String>,
253        fext: Option<String>,
254    ) -> Result<bool, DBError> {
255        let opened = self
256            .filer
257            .reopen(temp, head_dir_path, perm, clear, reuse, clean, mode, fext)
258            .map_err(|e| DBError::FilerError(format!("{}", e)))?;
259
260        // Close if already open
261        if self.env.is_some() {
262            self.close(false)?;
263        }
264
265        // Determine path
266        let dir_path = match self.filer.path() {
267            Some(p) => p,
268            None => {
269                return Err(DBError::PathError("Database path not set".into()));
270            }
271        };
272
273        // Clear directory if needed
274        if clear && !reuse {
275            if let Err(e) = fs::remove_dir_all(&dir_path) {
276                if e.kind() != std::io::ErrorKind::NotFound {
277                    return Err(DBError::IoError(format!("{}", e)));
278                }
279            }
280            fs::create_dir_all(&dir_path).map_err(|e| DBError::FilerError(format!("{}", e)))?;
281        }
282
283        // Open environment with heed
284        let mut env_builder = EnvOpenOptions::new();
285
286        // Configure environment
287        env_builder
288            .map_size(Self::MAP_SIZE)
289            .max_dbs(Self::MAX_NAMED_DBS);
290
291        let env = if self.readonly {
292            unsafe { Arc::new(env_builder.open(&dir_path)?) }
293        } else {
294            unsafe { Arc::new(env_builder.open(&dir_path)?) }
295        };
296
297        self.env = Some(env);
298
299        let result = opened && self.env.is_some();
300        self.filer.set_opened(result);
301        Ok(result)
302    }
303
304    /// Get the version of the database
305    pub fn version(&self) -> Option<&str> {
306        self.version.as_deref()
307    }
308
309    /// Set the version of the database
310    pub fn set_version(&mut self, version: String) {
311        self.version = Some(version);
312    }
313
314    pub fn close(&mut self, clear: bool) -> Result<bool, DBError> {
315        if let Some(env) = self.env.take() {
316            // With heed, we don't need to explicitly close the environment.
317            // It will be closed when dropped.
318            drop(env);
319        }
320        self.env = None;
321
322        // Clear the directory if needed
323        if clear && self.filer.path().is_some() {
324            let path = self.filer.path().unwrap();
325            if let Err(e) = fs::remove_dir_all(&path) {
326                if e.kind() != std::io::ErrorKind::NotFound {
327                    return Err(DBError::IoError(format!("{}", e)));
328                }
329            }
330        }
331
332        let result = self
333            .filer
334            .close(clear)
335            .map_err(|e| DBError::FilerError(format!("{}", e)))?;
336
337        Ok(result)
338    }
339
340    // Database operations with heed
341
342    // Create a database
343    pub fn create_database(
344        &self,
345        name: Option<&str>,
346        dup_sort: Option<bool>,
347    ) -> Result<BytesDatabase, DBError> {
348        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
349        let mut txn = env.write_txn()?;
350        let dup_sort = dup_sort.unwrap_or(false);
351
352        let mut binding = env
353            .database_options()
354            .types::<heed::types::Bytes, heed::types::Bytes>();
355        let options = binding.name(name.unwrap_or(""));
356
357        if dup_sort {
358            options.flags(DatabaseFlags::DUP_SORT);
359        }
360
361        let db = options.create(&mut txn)?;
362
363        txn.commit()?;
364        Ok(db)
365    }
366
367    // Open an existing database
368    pub fn open_database(&self, name: Option<&str>) -> Result<Option<BytesDatabase>, DBError> {
369        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
370        let mut wtxn = env.write_txn()?;
371
372        let db = env.open_database(&mut wtxn, name)?;
373
374        wtxn.commit()?;
375        Ok(db)
376    }
377
378    // Get a value
379    pub fn len(&self, db: &BytesDatabase) -> Result<u64, DBError> {
380        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
381        let rtxn = env.read_txn()?;
382
383        let result = match db.len(&rtxn) {
384            Ok(val) => val,
385            Err(_) => 0,
386        };
387
388        Ok(result)
389    }
390
391    // Put a value
392    pub fn put_val(&self, db: &BytesDatabase, key: &[u8], val: &[u8]) -> Result<bool, DBError> {
393        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
394
395        // First check if the key already exists using a read transaction
396        let rtxn = env.read_txn()?;
397        let key_exists = db.get(&rtxn, key)?.is_some();
398        rtxn.commit()?;
399
400        // If key already exists, return false (didn't add)
401        if key_exists {
402            return Ok(false);
403        }
404
405        // If key doesn't exist, add it with a write transaction
406        let mut wtxn = env.write_txn()?;
407        db.put(&mut wtxn, key, val)?;
408        wtxn.commit()?;
409
410        // Return true to indicate the value was successfully added
411        Ok(true)
412    }
413
414    // Same as put_val, kept for compatibility
415    pub fn set_val(&self, db: &BytesDatabase, key: &[u8], val: &[u8]) -> Result<bool, DBError> {
416        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
417        let mut wtxn = env.write_txn()?;
418        db.put(&mut wtxn, key, val)?;
419        wtxn.commit()?;
420        Ok(true)
421    }
422
423    // Get a value
424    pub fn get_val(&self, db: &BytesDatabase, key: &[u8]) -> Result<Option<Vec<u8>>, DBError> {
425        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
426        let rtxn = env.read_txn()?;
427
428        let result = match db.get(&rtxn, key)? {
429            Some(val) => Some(val.to_vec()),
430            None => None,
431        };
432
433        Ok(result)
434    }
435
436    /// Returns the last of the duplicated values associated with a key for databases with dupsort=true,
437    ///
438    /// # Parameters
439    /// - `db`: The database to search
440    /// - `key`: The key to look up
441    ///
442    /// # Returns
443    /// - `Ok(Some(Vec<u8>))`: The last value if found
444    /// - `Ok(None)`: If no value exists at the key
445    /// - `Err(DBError)`: If a database error occurs
446    pub fn get_val_last(&self, db: &BytesDatabase, key: &[u8]) -> Result<Option<Vec<u8>>, DBError> {
447        if key.is_empty() {
448            return Err(DBError::KeyError(
449                "Key is empty, too big, or wrong DUPFIXED size".to_string(),
450            ));
451        }
452
453        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
454        let rtxn = env.read_txn()?;
455
456        // For dupsort databases, we need to collect all values and return the last one
457        let mut last_val: Option<Vec<u8>> = None;
458
459        // Collect all duplicate values for this key
460        if let Some(mut iter) = db.get_duplicates(&rtxn, &key)? {
461            // Now we have the actual iterator, so we can use next()
462            while let Some(val_result) = iter.next() {
463                // Handle the Result from next()
464                let (_key, val) = val_result?;
465                last_val = Some(val.to_vec());
466            }
467        }
468        Ok(last_val)
469    }
470
471    // Delete a value
472    pub fn del_val(&self, db: &BytesDatabase, key: &[u8]) -> Result<bool, DBError> {
473        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
474        let mut wtxn = env.write_txn()?;
475
476        let exists = db.get(&wtxn, key)?.is_some();
477        if exists {
478            db.delete(&mut wtxn, key)?;
479            wtxn.commit()?;
480            Ok(true)
481        } else {
482            wtxn.abort();
483            Ok(false)
484        }
485    }
486
487    // Count entries in a database
488    pub fn cnt(&self, db: &BytesDatabase) -> Result<usize, DBError> {
489        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
490        let rtxn = env.read_txn()?;
491
492        // In heed 0.22.0, we can iterate to count
493        let mut count = 0;
494        let iter = db.iter(&rtxn)?;
495        for _ in iter {
496            count += 1;
497        }
498
499        Ok(count)
500    }
501
502    /// Get items with a given prefix and process them with a callback function
503    ///
504    /// # Parameters
505    /// - `db`: The database to search in
506    /// - `prefix`: The prefix to match keys against
507    /// - `cb`: Callback function that takes key-value pairs
508    ///
509    /// # Returns
510    /// - `Ok(count)`: Number of items processed
511    /// - `Err(DBError)`: If a database error occurs
512    pub fn get_top_items_iter<F>(
513        &self,
514        db: &BytesDatabase,
515        prefix: &[u8],
516        cb: F,
517    ) -> Result<usize, DBError>
518    where
519        F: FnMut(&[u8], &[u8]) -> Result<bool, DBError>,
520    {
521        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
522        let txn = env.read_txn()?;
523
524        // Get an iterator over all items in the database
525        let iter = db.iter(&txn)?;
526
527        // Process items with the callback
528        let mut count = 0;
529        let mut callback = cb;
530
531        for result in iter {
532            match result {
533                Ok((k, v)) => {
534                    // Only process items with matching prefix
535                    if k.starts_with(prefix) {
536                        count += 1;
537
538                        // Call the callback with the key-value pair
539                        // If callback returns false, stop iteration
540                        if !callback(&k, &v)? {
541                            break;
542                        }
543                    }
544                }
545                Err(e) => return Err(DBError::EnvError(e)),
546            }
547        }
548
549        Ok(count)
550    }
551
552    // Delete all values with a given prefix
553    pub fn del_top_val(&self, db: &BytesDatabase, prefix: &[u8]) -> Result<bool, DBError> {
554        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
555        let mut txn = env.write_txn()?;
556
557        // First collect keys with the prefix
558        let read_txn = env.read_txn()?;
559        let iter = db.iter(&read_txn)?;
560
561        let keys_to_delete: Vec<Vec<u8>> = iter
562            .filter_map(|result| match result {
563                Ok((k, _)) => {
564                    if k.starts_with(prefix) {
565                        Some(Ok(k.to_vec()))
566                    } else {
567                        None
568                    }
569                }
570                Err(e) => Some(Err(DBError::EnvError(e))),
571            })
572            .collect::<Result<Vec<_>, _>>()?;
573
574        // Close read transaction before starting write operations
575        drop(read_txn);
576
577        let mut deleted = false;
578        if !keys_to_delete.is_empty() {
579            for key in keys_to_delete {
580                db.delete(&mut txn, &key)?;
581                deleted = true;
582            }
583
584            txn.commit()?;
585        } else {
586            txn.abort();
587        }
588
589        Ok(deleted)
590    }
591
592    /// Write serialized bytes val to location at onkey consisting of
593    /// key + sep + serialized on in db.
594    /// Overwrites pre-existing value at onkey if any.
595    ///
596    /// # Parameters
597    /// - `db`: Named sub database in LMDB
598    /// - `key`: Key within sub db's keyspace
599    /// - `on`: Ordinal number at which to write
600    /// - `val`: Bytes to be written at onkey
601    /// - `sep`: Separator byte for constructing the composite key
602    ///
603    /// # Returns
604    /// - `Ok(true)`: If successful write (i.e., onkey not already in db)
605    /// - `Ok(false)`: Otherwise
606    /// - `Err(DBError)`: If a database error occurs
607    pub fn set_on_val(
608        &self,
609        db: &BytesDatabase,
610        key: &[u8],
611        on: Option<u64>,
612        val: &[u8],
613        sep: Option<[u8; 1]>,
614    ) -> Result<bool, DBError> {
615        let sep = sep.unwrap_or(*b".");
616        let on = on.unwrap_or(0);
617
618        // Get the environment
619        let env = self
620            .env
621            .as_ref()
622            .ok_or(DBError::DatabaseError("Not opened".to_string()))?;
623
624        // Begin a write transaction
625        let mut txn = env
626            .write_txn()
627            .map_err(|e| DBError::DatabaseError(format!("{}", e)))?;
628
629        // Create the onkey (composite key with ordinal number)
630        let onkey = if !key.is_empty() {
631            on_key(key, on, Some(sep))
632        } else {
633            key.to_vec()
634        };
635
636        // Put the value at the onkey
637        if onkey.is_empty() {
638            return Err(DBError::KeyError("Key cannot be empty".to_string()));
639        }
640
641        let result = match db.put(&mut txn, &onkey, val) {
642            Ok(_) => true,
643            Err(e) => {
644                if let Some(_) = e.to_string().to_lowercase().find("valsize") {
645                    return Err(DBError::KeyError(format!(
646                        "Key: `{:?}` is either empty, too big (for lmdb), or wrong DUPFIXED size.",
647                        onkey
648                    )));
649                }
650                return Err(DBError::DatabaseError(format!("{}", e)));
651            }
652        };
653
654        // Commit the transaction
655        txn.commit()
656            .map_err(|e| DBError::DatabaseError(format!("{}", e)))?;
657
658        Ok(result)
659    }
660
661    /// Appends val in order after last previous onkey in db where
662    /// onkey has same given key prefix but with different serialized on suffix
663    /// attached with sep.
664    ///
665    /// Returns ordinal number on, of appended entry. Appended on is 1 greater
666    /// than previous latest on at key.
667    ///
668    /// Works with either dupsort==True or False since always creates new full key.
669    ///
670    /// # Parameters
671    /// - `db`: Named sub database in LMDB
672    /// - `key`: Key within sub db's keyspace
673    /// - `val`: Bytes to be written at onkey
674    /// - `sep`: Separator byte for constructing the composite key
675    ///
676    /// # Returns
677    /// - `Ok(on)`: Ordinal number of newly appended val
678    /// - `Err(DBError)`: If a database error occurs
679    pub fn append_on_val(
680        &self,
681        db: &BytesDatabase,
682        key: &[u8],
683        val: &[u8],
684        sep: Option<[u8; 1]>,
685    ) -> Result<u64, DBError> {
686        let sep = sep.unwrap_or([b'.']);
687
688        // Get the environment
689        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
690        // Initialize with default ordinal 0
691        let mut on = 0;
692        let onkey = on_key(key, 0, Some(sep));
693
694        // Create a range iterator to find keys with the given prefix
695        // The iterator will return keys in ascending order
696
697        let rtxn = env.read_txn()?;
698        let range = (Bound::Included(onkey.as_slice()), Bound::Unbounded);
699
700        let mut iter = db.range(&rtxn, &range)?;
701        let mut last_entry = None;
702
703        for result in iter {
704            let (onkey, _) = result?;
705            let (ckey, cn) = split_on_key(onkey.as_ref(), Some(sep))?;
706            if ckey != key {
707                break;
708            }
709            last_entry = Some(cn);
710        }
711        // If we found a last entry, increment its ordinal number
712        if let Some(last_on) = last_entry {
713            // Check for overflow
714            if last_on == MAX_ON {
715                return Err(DBError::ValueError(format!(
716                    "Number part on={} for key part key={:?} exceeds maximum size.",
717                    last_on, key
718                )));
719            }
720            on = last_on + 1;
721        }
722
723        // Create the new key with the determined ordinal number
724        let onkey = on_key(key, on, Some(sep));
725
726        // Use a write transaction to add the new entry
727        let mut wtxn = env.write_txn()?;
728
729        // Check if the key already exists (should not happen if our algorithm is correct)
730        if db.get(&wtxn, &onkey)?.is_some() {
731            return Err(DBError::ValueError(format!(
732                "Key already exists: {:?}",
733                onkey
734            )));
735        }
736
737        // Put the value at the onkey
738        db.put(&mut wtxn, &onkey, val)?;
739
740        // Commit the transaction
741        wtxn.commit()?;
742
743        Ok(on)
744    }
745
746    /// Write serialized bytes val to location at onkey consisting of
747    /// key + sep + serialized on in db.
748    /// Does not overwrite.
749    ///
750    /// # Returns
751    /// - Result<bool, DBError>: True if successful write i.e onkey not already in db,
752    ///                          False otherwise
753    ///
754    /// # Parameters
755    /// - db: named sub db of lmdb
756    /// - key: key within sub db's keyspace plus trailing part on
757    /// - on: ordinal number at which write
758    /// - val: to be written at onkey
759    /// - sep: separator bytes for split
760    pub fn put_on_val(
761        &self,
762        db: &BytesDatabase,
763        key: &[u8],
764        on: u32,
765        val: &[u8],
766        sep: Option<[u8; 1]>,
767    ) -> Result<bool, DBError> {
768        let sep = sep.unwrap_or(*b".");
769        let env = self
770            .env
771            .as_ref()
772            .ok_or(DBError::DatabaseError("Not opened".to_string()))?;
773        let mut txn = env
774            .write_txn()
775            .map_err(|e| DBError::DatabaseError(format!("{}", e)))?;
776
777        let onkey = if !key.is_empty() {
778            on_key(key, on as u64, Some(sep))
779        } else {
780            key.to_vec()
781        };
782
783        // First check if the key already exists using a read transaction
784        let rtxn = env.read_txn()?;
785        let key_exists = db.get(&rtxn, &onkey)?.is_some();
786        rtxn.commit()?;
787
788        // If key already exists, return false (didn't add)
789        if key_exists {
790            return Ok(false);
791        }
792
793        db.put(&mut txn, &onkey, val).map_err(|e| {
794            if let heed::Error::Mdb(heed::MdbError::BadValSize) = e {
795                DBError::ValueError(format!(
796                    "Key: `{:?}` is either empty, too big, or wrong DUPFIXED size",
797                    onkey
798                ))
799            } else {
800                DBError::DatabaseError(format!("{}", e))
801            }
802        })?;
803
804        txn.commit()
805            .map_err(|e| DBError::DatabaseError(format!("{}", e)))?;
806        Ok(true)
807    }
808
809    /// Gets value at onkey consisting of key + sep + serialized on in db.
810    ///
811    /// # Returns
812    /// - Result<Option<Vec<u8>>, DBError>: entry at onkey or None if no entry at key
813    ///
814    /// # Parameters
815    /// - db: named sub db of lmdb
816    /// - key: key within sub db's keyspace
817    /// - on: ordinal number at which to retrieve
818    /// - sep: separator bytes for split
819    pub fn get_on_val(
820        &self,
821        db: &BytesDatabase,
822        key: &[u8],
823        on: u32,
824        sep: Option<[u8; 1]>,
825    ) -> Result<Option<Vec<u8>>, DBError> {
826        let sep = sep.unwrap_or(*b".");
827        let env = self
828            .env
829            .as_ref()
830            .ok_or(DBError::DatabaseError("Not opened".to_string()))?;
831        let txn = env
832            .read_txn()
833            .map_err(|e| DBError::DatabaseError(format!("{}", e)))?;
834
835        let onkey = if !key.is_empty() {
836            on_key(key, on as u64, Some(sep))
837        } else {
838            key.to_vec()
839        };
840
841        match db.get(&txn, &onkey).map_err(|e| {
842            if let heed::Error::Mdb(heed::MdbError::BadValSize) = e {
843                DBError::ValueError(format!(
844                    "Key: `{:?}` is either empty, too big, or wrong DUPFIXED size",
845                    onkey
846                ))
847            } else {
848                DBError::DatabaseError(format!("{}", e))
849            }
850        })? {
851            Some(val) => Ok(Some(val.to_vec())),
852            None => Ok(None),
853        }
854    }
855
856    /// Deletes value at onkey consisting of key + sep + serialized on in db.
857    ///
858    /// # Returns
859    /// - Result<bool, DBError>: True if key exists in database Else False
860    ///
861    /// # Parameters
862    /// - db: named sub db of lmdb
863    /// - key: key within sub db's keyspace
864    /// - on: ordinal number at which to delete
865    /// - sep: separator bytes for split
866    pub fn del_on_val(
867        &self,
868        db: &BytesDatabase,
869        key: &[u8],
870        on: u32,
871        sep: Option<[u8; 1]>,
872    ) -> Result<bool, DBError> {
873        let sep = sep.unwrap_or(*b".");
874        let env = self
875            .env
876            .as_ref()
877            .ok_or(DBError::DatabaseError("Not opened".to_string()))?;
878        let mut txn = env
879            .write_txn()
880            .map_err(|e| DBError::DatabaseError(format!("{}", e)))?;
881
882        let onkey = if !key.is_empty() {
883            on_key(key, on as u64, Some(sep))
884        } else {
885            key.to_vec()
886        };
887
888        let result = db.delete(&mut txn, &onkey).map_err(|e| {
889            if let heed::Error::Mdb(heed::MdbError::BadValSize) = e {
890                DBError::ValueError(format!(
891                    "Key: `{:?}` is either empty, too big, or wrong DUPFIXED size",
892                    onkey
893                ))
894            } else {
895                DBError::DatabaseError(format!("{}", e))
896            }
897        })?;
898
899        txn.commit()
900            .map_err(|e| DBError::DatabaseError(format!("{}", e)))?;
901        Ok(result)
902    }
903
904    /// Returns count of all ordinal keyed vals with key but different on tail in db
905    /// starting at ordinal number on of key.
906    ///
907    /// # Returns
908    /// - Result<usize, DBError>: count of matching entries
909    ///
910    /// # Parameters
911    /// - db: named sub db of lmdb
912    /// - key: key within sub db's keyspace (when empty, counts whole db)
913    /// - on: ordinal number at which to initiate count
914    /// - sep: separator bytes for split
915    pub fn cnt_on_vals(
916        &self,
917        db: &BytesDatabase,
918        key: Option<&[u8]>,
919        on: Option<u64>,
920        sep: Option<[u8; 1]>,
921    ) -> Result<usize, DBError> {
922        let separator = sep.unwrap_or([b'.']);
923        let key = key.unwrap_or(&[]);
924        let on = on.unwrap_or(0);
925
926        // Get environment from the Arc
927        let env = self.env.as_ref().ok_or(DBError::DatabaseError(
928            "Environment not available".to_string(),
929        ))?;
930
931        let mut count = 0;
932
933        let txn = env.read_txn()?;
934
935        // Create the starting key for the range
936        let iter = if !key.is_empty() {
937            let onkey = on_key(key, on, Some(separator));
938            let range = (Bound::Included(onkey.as_slice()), Bound::Unbounded);
939            db.range(&txn, &range)?
940        } else {
941            let range = (Bound::Unbounded, Bound::Unbounded);
942            db.range(&txn, &range)?
943        };
944
945        // Use range method to iterate over keys starting from onkey
946
947        for result in iter {
948            let (ckey, cval) = result?;
949
950            // Try to split the key to get the base part and the ordinal number
951            match split_on_key(ckey.as_ref(), Some(separator)) {
952                Ok((ckey_base, _)) => {
953                    // If key is not empty and ckey_base doesn't match key, we've moved past our range
954                    if !key.is_empty() && ckey_base != key {
955                        break;
956                    }
957                    // Increment the count for valid entries
958                    count += 1;
959                }
960                Err(_) => {
961                    // Not a splittable key, we're done
962                    println!(
963                        "{:?} = {:?}",
964                        String::from_utf8(ckey.to_vec()),
965                        String::from_utf8(cval.to_vec())
966                    );
967                    break;
968                }
969            }
970        }
971
972        Ok(count)
973    }
974
975    /// - txn: the read transaction to use
976    pub fn get_on_item_iter<F>(
977        &self,
978        db: &BytesDatabase,
979        key: Option<&[u8]>,
980        on: Option<u64>,
981        sep: Option<[u8; 1]>,
982        mut callback: F,
983    ) -> Result<(), DBError>
984    where
985        F: FnMut(Vec<u8>, u64, Vec<u8>) -> Result<bool, DBError>,
986    {
987        let separator = sep.unwrap_or([b'.']);
988        let key = key.unwrap_or(&[]);
989        let on = on.unwrap_or(0);
990
991        let env = self
992            .env
993            .as_ref()
994            .ok_or(DBError::DatabaseError("Not opened".to_string()))?;
995        let txn = env
996            .read_txn()
997            .map_err(|e| DBError::DatabaseError(format!("{}", e)))?;
998
999        // Create the starting key for the range
1000        let mut iter = if !key.is_empty() {
1001            let onkey = on_key(key, on, Some(separator));
1002            let range = (Bound::Included(onkey.as_slice()), Bound::Unbounded);
1003            db.range(&txn, &range)?
1004        } else {
1005            let range = (Bound::Unbounded, Bound::Unbounded);
1006            db.range(&txn, &range)?
1007        };
1008
1009        while let Some(result) = iter.next() {
1010            let (ckey, cval) = result?;
1011
1012            // Split the key to get the base part and the ordinal number
1013            let (ckey_base, cn) = split_on_key(ckey.as_ref(), Some(separator))?;
1014
1015            // If key is not empty and ckey_base doesn't match key, return an error
1016            // This will be caught in the take_while below to stop iteration
1017            if !key.is_empty() && ckey_base != key {
1018                break;
1019            }
1020
1021            // Call the callback with each value
1022            // If callback returns false, stop iteration
1023            if !callback(ckey_base, cn, cval.to_vec())? {
1024                break;
1025            }
1026        }
1027
1028        Ok(())
1029    }
1030
1031    /// - txn: the read transaction to use
1032    pub fn get_on_val_iter<F>(
1033        &self,
1034        db: &BytesDatabase,
1035        key: Option<&[u8]>,
1036        on: Option<u64>,
1037        sep: Option<[u8; 1]>,
1038        mut callback: F,
1039    ) -> Result<(), DBError>
1040    where
1041        F: FnMut(Vec<u8>) -> Result<bool, DBError>,
1042    {
1043        self.get_on_item_iter(&db, key, on, sep, |ckey, cn, cval| callback(cval))?;
1044
1045        Ok(())
1046    }
1047
1048    /// Add each val in vals to insertion ordered set of values all with the
1049    /// same apparent effective key for each val that is not already in set of
1050    /// vals at key.
1051    /// Uses hidden ordinal key suffix for insertion ordering.
1052    /// The suffix is appended and stripped transparently.
1053    ///
1054    /// # Parameters
1055    /// - `db`: Instance of named sub db with dupsort==False
1056    /// - `key`: Apparent effective key
1057    /// - `vals`: Serialized values to add to set of vals at key
1058    /// - `sep`: Optional separator byte (defaults to '.')
1059    ///
1060    /// # Returns
1061    /// - `Ok(bool)`: True if added to set, false if already in set
1062    /// - `Err(DBError)`: If a database error occurs
1063    pub fn put_io_set_vals(
1064        &self,
1065        db: &BytesDatabase,
1066        key: &[u8],
1067        vals: &[&[u8]],
1068        sep: Option<[u8; 1]>,
1069    ) -> Result<bool, DBError> {
1070        let sep = sep.unwrap_or([b'.']);
1071        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
1072
1073        // Create a write transaction
1074        let mut wtxn = env.write_txn()?;
1075
1076        // Start with ordinal 0
1077        let mut ion = 0;
1078        let mut result = false;
1079
1080        // Create a HashSet to track existing values at this key
1081        let mut existing_vals = HashSet::new();
1082
1083        // Create a range to find all entries with this key
1084        let start_iokey = suffix(key, ion, Some(sep));
1085
1086        // Use range to find all values with this key prefix
1087        let range = (Bound::Included(start_iokey.as_slice()), Bound::Unbounded);
1088        let iter = db.range(&wtxn, &range)?;
1089
1090        // Track the highest ordinal number seen
1091        for entry in iter {
1092            let (iokey, val) = entry?;
1093
1094            // Attempt to extract original key and ordinal
1095            if let (ckey, cion) = unsuffix(&iokey, Some(sep))? {
1096                if ckey == key {
1097                    // This is a value at our target key
1098                    existing_vals.insert(Vec::from(val));
1099                    ion = cion + 1; // ion to add at is increment of cion
1100                } else {
1101                    // We've moved past our key range
1102                    break;
1103                }
1104            }
1105        }
1106
1107        // Process each value that's not already in the set
1108        for val in vals {
1109            if !existing_vals.contains(&Vec::from(*val)) {
1110                let iokey = suffix(key, ion, Some(sep));
1111
1112                // Add the new entry
1113                db.put(&mut wtxn, &iokey, val)?;
1114
1115                // Update result and increment ion for next value
1116                result = true;
1117                ion += 1;
1118            }
1119        }
1120
1121        // Commit the transaction
1122        wtxn.commit()?;
1123
1124        Ok(result)
1125    }
1126
1127    /// Returns the insertion ordered set of values at the same apparent
1128    /// effective key.
1129    /// Uses hidden ordinal key suffix for insertion ordering.
1130    /// The suffix is appended and stripped transparently.
1131    ///
1132    /// # Parameters
1133    /// - `db`: Instance of named sub db with dupsort==False
1134    /// - `key`: Apparent effective key
1135    /// - `ion`: starting ordinal value, default 0
1136    /// - `sep`: Optional separator byte (defaults to '.')
1137    ///
1138    /// # Returns
1139    /// - `Ok(Vec<Vec<u8>>)`: The ordered list of values at this key
1140    /// - `Err(DBError)`: If a database error occurs
1141    pub fn get_io_set_vals(
1142        &self,
1143        db: &BytesDatabase,
1144        key: &[u8],
1145        ion: Option<u64>,
1146        sep: Option<[u8; 1]>,
1147    ) -> Result<Vec<Vec<u8>>, DBError> {
1148        let ion = ion.unwrap_or(0);
1149        let sep = sep.unwrap_or([b'.']);
1150        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
1151
1152        // Create a read transaction
1153        let rtxn = env.read_txn()?;
1154
1155        // Initialize the result vector
1156        let mut vals = Vec::new();
1157
1158        // Create the starting key for the range
1159        let start_iokey = suffix(key, ion, Some(sep));
1160
1161        // Create a range to find all entries with this key starting from ion
1162        let range = (Bound::Included(start_iokey.as_slice()), Bound::Unbounded);
1163        let iter = db.range(&rtxn, &range)?;
1164
1165        // Iterate through all entries in the range
1166        for entry in iter {
1167            let (iokey, val) = entry?;
1168
1169            // Attempt to extract original key and ordinal
1170            if let (ckey, _) = unsuffix(&iokey, Some(sep))? {
1171                if ckey == key {
1172                    // This is a value at our target key
1173                    vals.push(Vec::from(val));
1174                } else {
1175                    // We've moved past our key range
1176                    break;
1177                }
1178            }
1179        }
1180
1181        Ok(vals)
1182    }
1183
1184    /// Add val idempotently to insertion ordered set of values all with the
1185    /// same apparent effective key if val not already in set of vals at key.
1186    /// Uses hidden ordinal key suffix for insertion ordering.
1187    /// The suffix is appended and stripped transparently.
1188    ///
1189    /// # Parameters
1190    /// - `db`: Instance of named sub db with dupsort==False
1191    /// - `key`: Apparent effective key
1192    /// - `val`: Serialized value to add
1193    /// - `sep`: Optional separator byte (defaults to '.')
1194    ///
1195    /// # Returns
1196    /// - `Ok(bool)`: True if added to set, false if already in set
1197    /// - `Err(DBError)`: If a database error occurs
1198    pub fn add_io_set_val(
1199        &self,
1200        db: &BytesDatabase,
1201        key: &[u8],
1202        val: &[u8],
1203        sep: Option<[u8; 1]>,
1204    ) -> Result<bool, DBError> {
1205        let sep = sep.unwrap_or([b'.']);
1206        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
1207
1208        // Create a write transaction
1209        let mut wtxn = env.write_txn()?;
1210
1211        // Initialize variables
1212        let mut ion = 0;
1213        let mut found = false;
1214
1215        // Create the starting key for the range
1216        let start_iokey = suffix(key, ion, Some(sep));
1217
1218        // Create a range to find all entries with this key
1219        let range = (Bound::Included(start_iokey.as_slice()), Bound::Unbounded);
1220        let iter = db.range(&wtxn, &range)?;
1221
1222        // Iterate through all entries to check if the value exists and find the highest ion
1223        for entry in iter {
1224            let (iokey, cval) = entry?;
1225
1226            // Attempt to extract original key and ordinal
1227            if let (ckey, cion) = unsuffix(&iokey, Some(sep))? {
1228                if ckey == key {
1229                    // This is a value at our target key
1230                    if cval == val {
1231                        found = true; // Value already exists in the set
1232                    }
1233                    ion = cion + 1; // ion to add at is increment of cion
1234                } else {
1235                    // We've moved past our key range
1236                    break;
1237                }
1238            }
1239        }
1240
1241        // If the value is already in the set, return false
1242        if found {
1243            // Need to commit the transaction even though we're not making changes
1244            wtxn.commit()?;
1245            return Ok(false);
1246        }
1247
1248        // Create the key with the next available ordinal
1249        let iokey = suffix(key, ion, Some(sep));
1250
1251        // Add the new entry
1252        db.put(&mut wtxn, &iokey, val)?;
1253
1254        // Commit the transaction
1255        wtxn.commit()?;
1256
1257        Ok(true)
1258    }
1259
1260    /// Deletes all values at apparent effective key.
1261    /// Uses hidden ordinal key suffix for insertion ordering.
1262    /// The suffix is appended and stripped transparently.
1263    ///
1264    /// # Parameters
1265    /// - `db`: Instance of named sub db with dupsort==False
1266    /// - `key`: Apparent effective key
1267    /// - `sep`: Optional separator byte (defaults to '.')
1268    ///
1269    /// # Returns
1270    /// - `Ok(bool)`: True if values were deleted at key, False if no values at key
1271    /// - `Err(DBError)`: If a database error occurs
1272    pub fn del_io_set_vals(
1273        &self,
1274        db: &BytesDatabase,
1275        key: &[u8],
1276        sep: Option<[u8; 1]>,
1277    ) -> Result<bool, DBError> {
1278        let sep = sep.unwrap_or([b'.']);
1279        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
1280
1281        // Create a write transaction
1282        let mut wtxn = env.write_txn()?;
1283
1284        // Initialize result
1285        let mut result = false;
1286
1287        // Create the starting key for the range
1288        let start_iokey = suffix(key, 0, Some(sep));
1289
1290        // First collect all the keys we need to delete
1291        // (We can't modify the database while iterating over it)
1292        let mut keys_to_delete = Vec::new();
1293
1294        // Create a range to find all entries with this key
1295        let range = (Bound::Included(start_iokey.as_slice()), Bound::Unbounded);
1296        let iter = db.range(&wtxn, &range)?;
1297
1298        // Collect keys to delete
1299        for entry in iter {
1300            let (iokey, _) = entry?;
1301
1302            // Attempt to extract original key
1303            if let (ckey, _) = unsuffix(&iokey, Some(sep))? {
1304                if ckey == key {
1305                    // This is a value at our target key
1306                    keys_to_delete.push(iokey.to_vec());
1307                } else {
1308                    // We've moved past our key range
1309                    break;
1310                }
1311            }
1312        }
1313
1314        // Now delete all the keys we found
1315        for iokey in keys_to_delete {
1316            result = true; // At least one key was deleted
1317            db.delete(&mut wtxn, &iokey)?;
1318        }
1319
1320        // Commit the transaction
1321        wtxn.commit()?;
1322
1323        Ok(result)
1324    }
1325
1326    /// Deletes val at apparent effective key if exists.
1327    /// Uses hidden ordinal key suffix for insertion ordering.
1328    /// The suffix is appended and stripped transparently.
1329    ///
1330    /// Because the insertion order of val is not provided must perform a linear
1331    /// search over set of values.
1332    ///
1333    /// Another problem is that vals may get added and deleted in any order so
1334    /// the max suffix ion may creep up over time. The suffix ordinal max > 2**16
1335    /// is an impossibly large number, however, so the suffix will not max out
1336    /// practically. But it's not the most elegant solution.
1337    ///
1338    /// # Parameters
1339    /// - `db`: Instance of named sub db with dupsort==False
1340    /// - `key`: Apparent effective key
1341    /// - `val`: value to delete
1342    /// - `sep`: Optional separator byte (defaults to '.')
1343    ///
1344    /// # Returns
1345    /// - `Ok(bool)`: True if val was deleted at key, False if val not found at key
1346    /// - `Err(DBError)`: If a database error occurs
1347    pub fn del_io_set_val(
1348        &self,
1349        db: &BytesDatabase,
1350        key: &[u8],
1351        val: &[u8],
1352        sep: Option<[u8; 1]>,
1353    ) -> Result<bool, DBError> {
1354        let sep = sep.unwrap_or([b'.']);
1355        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
1356
1357        // Create a write transaction
1358        let mut wtxn = env.write_txn()?;
1359
1360        // Create the starting key for the range
1361        let start_iokey = suffix(key, 0, Some(sep));
1362
1363        // Create a range to find all entries with this key
1364        let range = (Bound::Included(start_iokey.as_slice()), Bound::Unbounded);
1365        let mut iter = db.range(&wtxn, &range)?;
1366
1367        // First collect all the keys we need to delete
1368        // (We can't modify the database while iterating over it)
1369        let mut keys_to_delete = Vec::new();
1370        // Initialize result
1371        let mut result = false;
1372
1373        // Search for the target value
1374        for entry in iter {
1375            let (iokey, cval) = entry?;
1376
1377            // Attempt to extract original key
1378            if let (ckey, _) = unsuffix(&iokey, Some(sep))? {
1379                if ckey != key {
1380                    // We've moved past our key range
1381                    break;
1382                }
1383
1384                if cval == val {
1385                    // Found the value, save it do delete later
1386                    keys_to_delete.push(iokey.to_vec());
1387                    break;
1388                }
1389            }
1390        }
1391
1392        // Now delete all the keys we found
1393        for iokey in keys_to_delete {
1394            result = true; // At least one key was deleted
1395            db.delete(&mut wtxn, &iokey)?;
1396        }
1397
1398        // Value not found, commit the transaction anyway
1399        wtxn.commit()?;
1400
1401        Ok(result)
1402    }
1403
1404    /// Erase all vals at key and then add unique vals as insertion ordered set of
1405    /// values all with the same apparent effective key.
1406    /// Uses hidden ordinal key suffix for insertion ordering.
1407    /// The suffix is appended and stripped transparently.
1408    ///
1409    /// # Parameters
1410    /// - `db`: Instance of named sub db with dupsort==False
1411    /// - `key`: Apparent effective key
1412    /// - `vals`: Serialized values to add to set of vals at key
1413    /// - `sep`: Optional separator byte (defaults to '.')
1414    ///
1415    /// # Returns
1416    /// - `Ok(bool)`: True if values were added to set
1417    /// - `Err(DBError)`: If a database error occurs
1418    pub fn set_io_set_vals(
1419        &self,
1420        db: &BytesDatabase,
1421        key: &[u8],
1422        vals: &[impl AsRef<[u8]>],
1423        sep: Option<[u8; 1]>,
1424    ) -> Result<bool, DBError> {
1425        let sep = sep.unwrap_or([b'.']);
1426
1427        // First delete all existing values at this key
1428        self.del_io_set_vals(db, key, Some(sep))?;
1429
1430        // If there are no values to add, return true (operation succeeded)
1431        if vals.is_empty() {
1432            return Ok(true);
1433        }
1434
1435        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
1436
1437        // Create a write transaction
1438        let mut wtxn = env.write_txn()?;
1439
1440        // Create a HashSet to eliminate duplicates while preserving order
1441        let mut unique_vals = Vec::new();
1442        let mut seen = HashSet::new();
1443
1444        // Deduplicate values while preserving order
1445        for val in vals {
1446            let val_bytes = val.as_ref();
1447            // Only add if we haven't seen this value before
1448            if seen.insert(val_bytes.to_vec()) {
1449                unique_vals.push(val_bytes);
1450            }
1451        }
1452
1453        // Add each unique value with ordinal suffix
1454        let mut result = false;
1455        for (i, val) in unique_vals.iter().enumerate() {
1456            let iokey = suffix(key, i as u64, Some(sep));
1457            db.put(&mut wtxn, &iokey, val)?;
1458            result = true;
1459        }
1460
1461        // Commit the transaction
1462        wtxn.commit()?;
1463
1464        Ok(result)
1465    }
1466
1467    /// Count all values with the same apparent effective key.
1468    /// Uses hidden ordinal key suffix for insertion ordering.
1469    /// The suffix is appended and stripped transparently.
1470    ///
1471    /// # Parameters
1472    /// - `db`: Instance of named sub db with dupsort==False
1473    /// - `key`: Apparent effective key
1474    /// - `sep`: Optional separator byte (defaults to '.')
1475    ///
1476    /// # Returns
1477    /// - `Ok(usize)`: Count of values in set at apparent effective key
1478    /// - `Err(DBError)`: If a database error occurs
1479    pub fn cnt_io_set_vals(
1480        &self,
1481        db: &BytesDatabase,
1482        key: &[u8],
1483        sep: Option<[u8; 1]>,
1484    ) -> Result<usize, DBError> {
1485        // Use the existing get_io_set_vals method and return the length of the result
1486        let vals = self.get_io_set_vals(db, key, None, sep)?;
1487        Ok(vals.len())
1488    }
1489
1490    /// Returns an iterator over insertion ordered set of values at same apparent effective key.
1491    /// Uses hidden ordinal key suffix for insertion ordering.
1492    /// The suffix is appended and stripped transparently.
1493    ///
1494    /// # Parameters
1495    /// - `db`: Instance of named sub db with dupsort==False
1496    /// - `key`: Apparent effective key
1497    /// - `ion`: Optional starting ordinal value, default 0
1498    /// - `sep`: Optional separator byte (defaults to '.')
1499    /// - `callback`: Function to call for each value found
1500    ///
1501    /// # Returns
1502    /// - `Ok(())`: If iteration completed successfully
1503    /// - `Err(DBError)`: If a database error occurs
1504    pub fn get_io_set_vals_iter<F>(
1505        &self,
1506        db: &BytesDatabase,
1507        key: &[u8],
1508        ion: Option<u64>,
1509        sep: Option<[u8; 1]>,
1510        mut callback: F,
1511    ) -> Result<(), DBError>
1512    where
1513        F: FnMut(&[u8]) -> Result<bool, DBError>,
1514    {
1515        let sep = sep.unwrap_or([b'.']);
1516        let ion = ion.unwrap_or(0);
1517        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
1518
1519        // Create a read transaction
1520        let rtxn = env.read_txn()?;
1521
1522        // Create the starting key for the range
1523        let start_iokey = suffix(key, ion, Some(sep));
1524
1525        // Create a range to find all entries with this key starting from ion
1526        let range = (Bound::Included(start_iokey.as_slice()), Bound::Unbounded);
1527        let iter = db.range(&rtxn, &range)?;
1528
1529        // Iterate through the entries and yield each value
1530        for entry in iter {
1531            let (iokey, val) = entry?;
1532
1533            // Attempt to extract original key
1534            if let (ckey, _) = unsuffix(&iokey, Some(sep))? {
1535                if ckey != key {
1536                    // We've moved past our key range
1537                    break;
1538                }
1539
1540                // Call the callback with the value
1541                // If callback returns false, stop iteration
1542                if !callback(&val)? {
1543                    break;
1544                }
1545            }
1546        }
1547
1548        Ok(())
1549    }
1550
1551    /// Returns the last added value at the apparent effective key if any,
1552    /// otherwise None if no entry.
1553    ///
1554    /// Uses hidden ordinal key suffix for insertion ordering.
1555    /// The suffix is appended and stripped transparently.
1556    ///
1557    /// # Parameters
1558    /// - `db`: Instance of named sub db with dupsort==False
1559    /// - `key`: Apparent effective key
1560    /// - `sep`: Optional separator byte (defaults to '.')
1561    ///
1562    /// # Returns
1563    /// - `Ok(Option<Vec<u8>>)`: The last value if found, None otherwise
1564    /// - `Err(DBError)`: If a database error occurs
1565    pub fn get_io_set_val_last(
1566        &self,
1567        db: &BytesDatabase,
1568        key: &[u8],
1569        sep: Option<[u8; 1]>,
1570    ) -> Result<Option<Vec<u8>>, DBError> {
1571        let sep = sep.unwrap_or([b'.']);
1572        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
1573
1574        // Create a read transaction
1575        let rtxn = env.read_txn()?;
1576
1577        // Make iokey at max suffix value
1578        let max_iokey = suffix(key, u64::MAX, Some(sep));
1579
1580        let mut ion = None; // No last value found yet
1581
1582        // Create a range from our key to the end of the database
1583        let range = (Bound::Included(max_iokey.as_slice()), Bound::Unbounded);
1584        let mut iter = db.range(&rtxn, &range)?;
1585
1586        // Check if max is past end of database
1587        if let Some(result) = iter.next() {
1588            let (entry_key, _) = result?;
1589
1590            // Check if this entry is for our key or for a different key
1591            if let (ckey, cion) = unsuffix(&entry_key, Some(sep))? {
1592                if ckey == key {
1593                    // 1. Cursor at entry for the same key
1594                    ion = Some(cion);
1595                } else {
1596                    // 2. Other key after our key, need to look back
1597
1598                    // Create a reverse range starting from our max key (exclusive)
1599                    let rev_range = (Bound::Unbounded, Bound::Excluded(max_iokey.as_slice()));
1600                    let mut rev_iter = db.rev_range(&rtxn, &rev_range)?;
1601
1602                    // Check the first entry in reverse order
1603                    if let Some(result) = rev_iter.next() {
1604                        let (prev_key, _) = result?;
1605
1606                        if let (ckey, cion) = unsuffix(&prev_key, Some(sep))? {
1607                            if ckey == key {
1608                                // Found a previous entry for our key
1609                                ion = Some(cion);
1610                            }
1611                        }
1612                    }
1613                }
1614            }
1615        } else {
1616            // Max is past end of database, check the last entry
1617            let rev_range = (Bound::Unbounded, Bound::Unbounded);
1618            let mut rev_iter = db.rev_range(&rtxn, &rev_range)?;
1619
1620            if let Some(result) = rev_iter.next() {
1621                let (last_key, _) = result?;
1622
1623                if let (ckey, cion) = unsuffix(&last_key, Some(sep))? {
1624                    if ckey == key {
1625                        // Last entry in db is for our key
1626                        ion = Some(cion);
1627                    }
1628                }
1629            }
1630        }
1631
1632        // If we found a matching ion, get the value
1633        if let Some(ion) = ion {
1634            let iokey = suffix(key, ion, Some(sep));
1635            if let Some(val) = db.get(&rtxn, &iokey)? {
1636                return Ok(Some(val.to_vec()));
1637            }
1638        }
1639
1640        // No matching value found
1641        Ok(None)
1642    }
1643
1644    /// Get items with a given prefix from an IO set and process them with a callback function,
1645    /// stripping the insertion order suffix from keys before passing to the callback.
1646    ///
1647    /// # Parameters
1648    /// - `db`: The database to search in
1649    /// - `top`: The prefix to match keys against (empty for all items)
1650    /// - `cb`: Callback function that takes apparent key (with suffix removed) and value pairs
1651    /// - `sep`: Optional separator byte (defaults to '.')
1652    ///
1653    /// # Returns
1654    /// - `Ok(count)`: Number of items processed
1655    /// - `Err(DBError)`: If a database error occurs
1656    pub fn get_top_io_set_items_iter<F>(
1657        &self,
1658        db: &BytesDatabase,
1659        top: &[u8],
1660        sep: Option<[u8; 1]>,
1661        cb: F,
1662    ) -> Result<usize, DBError>
1663    where
1664        F: FnMut(&[u8], &[u8]) -> Result<bool, DBError>,
1665    {
1666        let sep = sep.unwrap_or([b'.']);
1667        let mut callback = cb;
1668        let mut count = 0;
1669
1670        // Use get_top_items to iterate through the prefixed items
1671        self.get_top_items_iter(db, top, |iokey, val| {
1672            // Split the iokey to get the apparent key without the suffix
1673            if let (key, _ion) = unsuffix(iokey, Some(sep))? {
1674                count += 1;
1675                // Call the callback with the apparent key and value
1676                callback(&key, val)
1677            } else {
1678                // Skip items that don't have a valid suffix
1679                Ok(true)
1680            }
1681        })?;
1682
1683        Ok(count)
1684    }
1685
1686    /// Add value bytes as duplicate to key in database
1687    /// Adds to existing values at key if any
1688    /// Returns true if written else false if duplicate value already exists
1689    ///
1690    /// Duplicates are inserted in lexicographic order not insertion order.
1691    /// LMDB does not insert a duplicate unless it is a unique value for that key.
1692    ///
1693    /// Does inclusion test to detect if duplicate already exists
1694    /// Uses a HashSet for the duplicate inclusion test. Set inclusion scales
1695    /// with O(1) whereas list inclusion scales with O(n).
1696    ///
1697    /// # Parameters
1698    /// * `db` - opened named sub database with dupsort=True
1699    /// * `key` - bytes of key within sub db's keyspace
1700    /// * `val` - bytes of value to be written
1701    pub fn add_val(&self, db: &BytesDatabase, key: &[u8], val: &[u8]) -> Result<bool, DBError> {
1702        // Get preexisting duplicates (if any) and convert to a HashSet for O(1) inclusion test
1703        let mut dups = std::collections::HashSet::new();
1704
1705        // Use the get_vals_iter method to populate the HashSet
1706        self.get_vals_iter(db, key, |v| {
1707            dups.insert(v.to_vec());
1708            Ok(true) // Continue iteration
1709        })?;
1710
1711        // If the value is already in the set, return false (no write needed)
1712        if dups.contains(val) {
1713            return Ok(false);
1714        }
1715
1716        // Otherwise, write the value
1717        let env = match &self.env {
1718            Some(env) => env,
1719            None => return Err(DBError::DbClosed),
1720        };
1721
1722        let mut txn = match env.write_txn() {
1723            Ok(txn) => txn,
1724            Err(e) => return Err(DBError::EnvError(e)),
1725        };
1726
1727        // Add the new value
1728        match db.put(&mut txn, key, val) {
1729            Ok(_) => {}
1730            Err(e) => {
1731                return Err(DBError::KeyError(format!(
1732                    "Key: `{:?}` is either empty, too big, or wrong DUPFIXED size",
1733                    key
1734                )));
1735            }
1736        }
1737
1738        // Commit the transaction
1739        if let Err(e) = txn.commit() {
1740            return Err(DBError::EnvError(e));
1741        }
1742
1743        // If we got here, the value was successfully written
1744        Ok(true)
1745    }
1746
1747    /// Put each entry from a list of values to the given key in the database
1748    /// Adds to existing values at key if any
1749    /// Returns true if successful
1750    ///
1751    /// Duplicates are inserted in lexicographic order not insertion order.
1752    /// LMDB does not insert a duplicate unless it is a unique value for that key.
1753    ///
1754    /// # Parameters
1755    /// * `db` - opened named sub database with dupsort=True
1756    /// * `key` - bytes of key within sub db's keyspace
1757    /// * `vals` - list of values to be written
1758    pub fn put_vals(
1759        &self,
1760        db: &BytesDatabase,
1761        key: &[u8],
1762        vals: &[impl AsRef<[u8]>],
1763    ) -> Result<bool, DBError> {
1764        let env = match &self.env {
1765            Some(env) => env,
1766            None => return Err(DBError::DbClosed),
1767        };
1768
1769        let mut txn = match env.write_txn() {
1770            Ok(txn) => txn,
1771            Err(e) => return Err(DBError::EnvError(e)),
1772        };
1773
1774        for val in vals {
1775            match db.put(&mut txn, key, val.as_ref()) {
1776                Ok(_) => {} // Success returns (), no boolean to check
1777                Err(e) => {
1778                    return Err(DBError::KeyError(format!(
1779                        "Key: `{:?}` is either empty, too big, or wrong DUPFIXED size",
1780                        key
1781                    )));
1782                }
1783            }
1784        }
1785
1786        if let Err(e) = txn.commit() {
1787            return Err(DBError::EnvError(e));
1788        }
1789
1790        // Since we got here without errors, all values were added successfully
1791        Ok(true)
1792    }
1793
1794    /// Return iterator of all duplicate values at key in database
1795    /// Uses a callback function to handle each value
1796    ///
1797    /// Duplicates are retrieved in lexicographic order not insertion order.
1798    ///
1799    /// # Parameters
1800    /// * `db` - opened named sub database with dupsort=True
1801    /// * `key` - bytes of key within sub db's keyspace
1802    /// * `callback` - function that processes each value
1803    pub fn get_vals_iter<F>(
1804        &self,
1805        db: &BytesDatabase,
1806        key: &[u8],
1807        mut callback: F,
1808    ) -> Result<(), DBError>
1809    where
1810        F: FnMut(&[u8]) -> Result<bool, DBError>,
1811    {
1812        let env = match &self.env {
1813            Some(env) => env,
1814            None => return Err(DBError::DbClosed),
1815        };
1816
1817        let txn = match env.read_txn() {
1818            Ok(txn) => txn,
1819            Err(e) => return Err(DBError::EnvError(e)),
1820        };
1821
1822        // Use a prefix-based range to iterate through duplicate values
1823        // This gets all entries with exactly matching key
1824        let prefix_iter = match db.prefix_iter(&txn, &key) {
1825            Ok(iter) => iter,
1826            Err(e) => return Err(DBError::EnvError(e)),
1827        };
1828
1829        // Iterate through values and call the callback
1830        for res in prefix_iter {
1831            match res {
1832                Ok((k, val)) => {
1833                    // Make sure we only process exact key matches
1834                    if k == key {
1835                        if !callback(val)? {
1836                            break;
1837                        }
1838                    }
1839                }
1840                Err(e) => return Err(DBError::EnvError(e)),
1841            }
1842        }
1843
1844        Ok(())
1845    }
1846
1847    /// Return count of duplicate values at key in database, or zero otherwise
1848    ///
1849    /// # Parameters
1850    /// * `db` - opened named sub database with dupsort=True
1851    /// * `key` - bytes of key within sub db's keyspace
1852    pub fn cnt_vals(&self, db: &BytesDatabase, key: &[u8]) -> Result<usize, DBError> {
1853        let env = match &self.env {
1854            Some(env) => env,
1855            None => return Err(DBError::DbClosed),
1856        };
1857
1858        let txn = match env.read_txn() {
1859            Ok(txn) => txn,
1860            Err(e) => return Err(DBError::EnvError(e)),
1861        };
1862
1863        // Count the values by iterating through all entries with the key
1864        let count = self.count_duplicates(db, &txn, key)?;
1865
1866        Ok(count)
1867    }
1868
1869    // Helper method to count duplicates since we can't use cursor.count()
1870    fn count_duplicates(
1871        &self,
1872        db: &BytesDatabase,
1873        txn: &heed::RoTxn,
1874        key: &[u8],
1875    ) -> Result<usize, DBError> {
1876        let prefix_iter = match db.prefix_iter(txn, &key) {
1877            Ok(iter) => iter,
1878            Err(e) => return Err(DBError::EnvError(e)),
1879        };
1880
1881        let mut count = 0;
1882        for res in prefix_iter {
1883            match res {
1884                Ok((k, _)) => {
1885                    // Make sure we only count exact key matches
1886                    if k == key {
1887                        count += 1;
1888                    }
1889                }
1890                Err(e) => return Err(DBError::EnvError(e)),
1891            }
1892        }
1893
1894        Ok(count)
1895    }
1896
1897    /// Deletes all values at key in database if val is empty,
1898    /// otherwise deletes the duplicate that equals val
1899    ///
1900    /// Returns true if key (and val if not empty) exists in database, otherwise false
1901    ///
1902    /// # Parameters
1903    /// * `db` - opened named sub database with dupsort=True
1904    /// * `key` - bytes of key within sub db's keyspace
1905    /// * `val` - bytes of duplicate value at key to delete (empty means delete all duplicates)
1906    pub fn del_vals(
1907        &self,
1908        db: &BytesDatabase,
1909        key: &[u8],
1910        val: Option<&[u8]>,
1911    ) -> Result<bool, DBError> {
1912        let env = match &self.env {
1913            Some(env) => env,
1914            None => return Err(DBError::DbClosed),
1915        };
1916
1917        let mut txn = match env.write_txn() {
1918            Ok(txn) => txn,
1919            Err(e) => return Err(DBError::EnvError(e)),
1920        };
1921
1922        let result = match val {
1923            Some(v) if !v.is_empty() => {
1924                // Delete specific key-value pair
1925                match db.delete_one_duplicate(&mut txn, key, v) {
1926                    Ok(res) => res,
1927                    Err(e) => {
1928                        return Err(DBError::KeyError(format!(
1929                            "Key: `{:?}` is either empty, too big, or wrong DUPFIXED size",
1930                            key
1931                        )));
1932                    }
1933                }
1934            }
1935            _ => {
1936                // Delete all duplicates (empty val or None)
1937                match db.delete(&mut txn, key) {
1938                    Ok(res) => res,
1939                    Err(e) => {
1940                        return Err(DBError::KeyError(format!(
1941                            "Key: `{:?}` is either empty, too big, or wrong DUPFIXED size",
1942                            key
1943                        )));
1944                    }
1945                }
1946            }
1947        };
1948
1949        if let Err(e) = txn.commit() {
1950            return Err(DBError::EnvError(e));
1951        }
1952
1953        Ok(result)
1954    }
1955    /// Return list of duplicate values at key in db in insertion order
1956    /// Returns empty vector if no entry at key
1957    /// Removes prepended proem ordinal from each val before returning
1958    /// Assumes DB opened with dupsort=True
1959    ///
1960    /// Duplicates at a given key preserve insertion order of duplicate.
1961    /// Because lmdb is lexocographic an insertion ordering proem is prepended to
1962    /// all values that makes lexocographic order that same as insertion order.
1963    ///
1964    /// Duplicates are ordered as a pair of key plus value so prepending proem
1965    /// to each value changes duplicate ordering. Proem is 33 characters long.
1966    /// With 32 character hex string followed by '.' for essentially unlimited
1967    /// number of values which will be limited by memory.
1968    ///
1969    /// # Parameters
1970    /// - `db`: Opened named sub db with dupsort=True
1971    /// - `key`: bytes of key within sub db's keyspace
1972    ///
1973    /// # Returns
1974    /// - `Ok(Vec<Vec<u8>>)`: List of values with proem removed
1975    /// - `Err(DBError)`: If a database error occurs
1976    pub fn get_io_dup_vals(&self, db: &BytesDatabase, key: &[u8]) -> Result<Vec<Vec<u8>>, DBError> {
1977        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
1978
1979        // Create a read-only transaction
1980        let txn = env.read_txn()?;
1981
1982        let mut vals = Vec::new();
1983
1984        // Use a prefix-based range to iterate through duplicate values
1985        // This gets all entries with exactly matching key
1986        let prefix_iter = match db.prefix_iter(&txn, &key) {
1987            Ok(iter) => iter,
1988            Err(e) => return Err(DBError::EnvError(e)),
1989        };
1990
1991        // Iterate through values and extract them
1992        for res in prefix_iter {
1993            match res {
1994                Ok((k, val)) => {
1995                    // Make sure we only process exact key matches
1996                    if k == key {
1997                        // Skip values that are too short (must be at least 33 bytes for the proem)
1998                        if val.len() > 33 {
1999                            // Remove the 33-byte proem from the value
2000                            vals.push(Vec::from(&val[33..]));
2001                        }
2002                    }
2003                }
2004                Err(e) => {
2005                    // Convert BadValsizeError to KeyError, similar to Python implementation
2006                    return Err(DBError::KeyError(format!(
2007                        "Key: `{:?}` is either empty, too big (for lmdb), or wrong DUPFIXED size.",
2008                        key
2009                    )));
2010                }
2011            }
2012        }
2013
2014        Ok(vals)
2015    }
2016
2017    /// Write each entry from list of bytes vals to key in db in insertion order
2018    /// Adds to existing values at key if any
2019    /// Returns true if at least one of vals is added as dup, false otherwise
2020    /// Assumes DB opened with dupsort=True
2021    ///
2022    /// Duplicates at a given key preserve insertion order of duplicate.
2023    /// Because lmdb is lexocographic an insertion ordering proem is prepended to
2024    /// all values that makes lexocographic order that same as insertion order.
2025    ///
2026    /// Duplicates are ordered as a pair of key plus value so prepending proem
2027    /// to each value changes duplicate ordering. Proem is 33 characters long.
2028    /// With 32 character hex string followed by '.' for essentially unlimited
2029    /// number of values which will be limited by memory.
2030    ///
2031    /// # Parameters
2032    /// - `db`: Opened named sub db with dupsort=True
2033    /// - `key`: bytes of key within sub db's keyspace
2034    /// - `vals`: list of bytes of values to be written
2035    ///
2036    /// # Returns
2037    /// - `Ok(bool)`: True if at least one value was added, false otherwise
2038    /// - `Err(DBError)`: If a database error occurs
2039    pub fn put_io_dup_vals(
2040        &self,
2041        db: &BytesDatabase,
2042        key: &[u8],
2043        vals: &[impl AsRef<[u8]>],
2044    ) -> Result<bool, DBError> {
2045        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
2046
2047        // Get preexisting dups if any
2048        let existing_vals = self.get_io_dup_vals(db, key)?;
2049        let existing_set: std::collections::HashSet<Vec<u8>> = existing_vals.into_iter().collect();
2050
2051        // Create a write transaction
2052        let mut wtxn = env.write_txn()?;
2053        let mut result = false;
2054
2055        // Find the current highest index
2056        let mut idx = 0;
2057
2058        // Use prefix iterator to find all values at the key
2059        // and get the highest index
2060        let prefix_iter = match db.prefix_iter(&wtxn, key) {
2061            Ok(iter) => iter,
2062            Err(e) => {
2063                // If this is a BadValsizeError, convert to KeyError
2064                return Err(DBError::KeyError(format!(
2065                    "Key: `{:?}` is either empty, too big (for lmdb), or wrong DUPFIXED size.",
2066                    key
2067                )));
2068            }
2069        };
2070
2071        // Find the last duplicate value (highest index)
2072        let mut last_val: Option<Vec<u8>> = None;
2073
2074        for res in prefix_iter {
2075            match res {
2076                Ok((k, v)) => {
2077                    // Make sure we only process exact key matches
2078                    if k == key {
2079                        last_val = Some(v.to_vec());
2080                    }
2081                }
2082                Err(e) => {
2083                    return Err(DBError::KeyError(format!(
2084                        "Key: `{:?}` is either empty, too big (for lmdb), or wrong DUPFIXED size.",
2085                        key
2086                    )));
2087                }
2088            }
2089        }
2090
2091        // If we found a last value, extract the index from it
2092        if let Some(last_val) = last_val {
2093            if last_val.len() >= 32 {
2094                // Convert first 32 bytes to a hex string and parse as integer
2095                if let Ok(hex_str) = std::str::from_utf8(&last_val[0..32]) {
2096                    if let Ok(last_idx) = u64::from_str_radix(hex_str, 16) {
2097                        idx = last_idx + 1;
2098                    }
2099                }
2100            }
2101        }
2102
2103        // Process each value
2104        for val in vals {
2105            let val_bytes = val.as_ref();
2106
2107            // Only add if not already in the set
2108            if !existing_set.contains(val_bytes) {
2109                // Create the proem: 32 bytes of hex + '.'
2110                let proem = format!("{:032x}.", idx);
2111
2112                // Prepend proem to value
2113                let mut val_with_proem = proem.into_bytes();
2114                val_with_proem.extend_from_slice(val_bytes);
2115
2116                // Add to database with duplication
2117                db.put(&mut wtxn, key, &val_with_proem)?;
2118
2119                idx += 1;
2120                result = true;
2121            }
2122        }
2123
2124        // Commit the transaction
2125        wtxn.commit()?;
2126
2127        Ok(result)
2128    }
2129
2130    /// Add val bytes as dup in insertion order to key in db
2131    /// Adds to existing values at key if any
2132    /// Returns True if written else False if val is already a dup
2133    /// Actual value written include prepended proem ordinal
2134    /// Assumes DB opened with dupsort=True
2135    ///
2136    /// Duplicates at a given key preserve insertion order of duplicate.
2137    /// Because lmdb is lexocographic an insertion ordering proem is prepended to
2138    /// all values that makes lexocographic order that same as insertion order.
2139    ///
2140    /// Duplicates are ordered as a pair of key plus value so prepending proem
2141    /// to each value changes duplicate ordering. Proem is 33 characters long.
2142    /// With 32 character hex string followed by '.' for essentially unlimited
2143    /// number of values which will be limited by memory.
2144    ///
2145    /// # Parameters
2146    /// - `db`: Opened named sub db with dupsort=True
2147    /// - `key`: bytes of key within sub db's keyspace
2148    /// - `val`: bytes of value to be written
2149    ///
2150    /// # Returns
2151    /// - `Ok(bool)`: True if written, false if val is already a dup
2152    /// - `Err(DBError)`: If a database error occurs
2153    pub fn add_io_dup_val(
2154        &self,
2155        db: &BytesDatabase,
2156        key: &[u8],
2157        val: &[u8],
2158    ) -> Result<bool, DBError> {
2159        // Delegate to put_io_dup_vals with a single value
2160        self.put_io_dup_vals(db, key, &[val])
2161    }
2162
2163    pub fn get_io_dup_vals_iter<F>(
2164        &self,
2165        db: &BytesDatabase,
2166        key: &[u8],
2167        mut callback: F,
2168    ) -> Result<(), DBError>
2169    where
2170        F: FnMut(&[u8]) -> Result<bool, DBError>,
2171    {
2172        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
2173
2174        // Create a read-only transaction
2175        let txn = env.read_txn()?;
2176
2177        // Use a prefix-based range to iterate through duplicate values
2178        // This gets all entries with exactly matching key
2179        let prefix_iter = match db.prefix_iter(&txn, &key) {
2180            Ok(iter) => iter,
2181            Err(e) => {
2182                return Err(DBError::KeyError(format!(
2183                    "Key: `{:?}` is either empty, too big (for lmdb), or wrong DUPFIXED size.",
2184                    key
2185                )));
2186            }
2187        };
2188
2189        // Iterate through values and call the callback for each
2190        for res in prefix_iter {
2191            match res {
2192                Ok((k, val)) => {
2193                    // Make sure we only process exact key matches
2194                    if k == key {
2195                        // Skip values that are too short (must be at least 33 bytes for the proem)
2196                        if val.len() > 33 {
2197                            // Call the callback with the value with the proem removed
2198                            if !callback(&val[33..])? {
2199                                // If callback returns false, stop iteration
2200                                break;
2201                            }
2202                        }
2203                    }
2204                }
2205                Err(e) => {
2206                    return Err(DBError::KeyError(format!(
2207                        "Key: `{:?}` is either empty, too big (for lmdb), or wrong DUPFIXED size.",
2208                        key
2209                    )));
2210                }
2211            }
2212        }
2213
2214        Ok(())
2215    }
2216
2217    /// Return count of dup values at key in db, or zero otherwise
2218    /// Assumes DB opened with dupsort=True
2219    ///
2220    /// Duplicates at a given key preserve insertion order of duplicate.
2221    /// Because lmdb is lexocographic an insertion ordering proem is prepended to
2222    /// all values that makes lexocographic order that same as insertion order.
2223    ///
2224    /// Duplicates are ordered as a pair of key plus value so prepending proem
2225    /// to each value changes duplicate ordering. Proem is 33 characters long.
2226    /// With 32 character hex string followed by '.' for essentially unlimited
2227    /// number of values which will be limited by memory.
2228    ///
2229    /// # Parameters
2230    /// - `db`: Opened named sub db with dupsort=True
2231    /// - `key`: bytes of key within sub db's keyspace
2232    ///
2233    /// # Returns
2234    /// - `Ok(usize)`: Count of duplicate values at key (0 if key doesn't exist)
2235    /// - `Err(DBError)`: If a database error occurs
2236    pub fn cnt_io_dup_vals(&self, db: &BytesDatabase, key: &[u8]) -> Result<usize, DBError> {
2237        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
2238
2239        // Create a read-only transaction
2240        let txn = env.read_txn()?;
2241
2242        let mut count = 0;
2243
2244        // Use a prefix-based range to iterate through duplicate values
2245        // This gets all entries with exactly matching key
2246        let prefix_iter = match db.prefix_iter(&txn, &key) {
2247            Ok(iter) => iter,
2248            Err(_) => {
2249                return Err(DBError::KeyError(format!(
2250                    "Key: `{:?}` is either empty, too big (for lmdb), or wrong DUPFIXED size.",
2251                    key
2252                )));
2253            }
2254        };
2255
2256        // Count the number of entries with this exact key
2257        for res in prefix_iter {
2258            match res {
2259                Ok((k, _)) => {
2260                    // Make sure we only count exact key matches
2261                    if k == key {
2262                        count += 1;
2263                    }
2264                }
2265                Err(_) => {
2266                    return Err(DBError::KeyError(format!(
2267                        "Key: `{:?}` is either empty, too big (for lmdb), or wrong DUPFIXED size.",
2268                        key
2269                    )));
2270                }
2271            }
2272        }
2273
2274        Ok(count)
2275    }
2276
2277    /// Deletes all values at key in db if key present.
2278    /// Returns true if key exists and was deleted, false otherwise
2279    ///
2280    /// Duplicates at a given key preserve insertion order of duplicate.
2281    /// Because lmdb is lexocographic an insertion ordering proem is prepended to
2282    /// all values that makes lexocographic order that same as insertion order.
2283    ///
2284    /// Duplicates are ordered as a pair of key plus value so prepending proem
2285    /// to each value changes duplicate ordering. Proem is 33 characters long.
2286    /// With 32 character hex string followed by '.' for essentially unlimited
2287    /// number of values which will be limited by memory.
2288    ///
2289    /// # Parameters
2290    /// - `db`: Opened named sub db with dupsort=True
2291    /// - `key`: bytes of key within sub db's keyspace
2292    ///
2293    /// # Returns
2294    /// - `Ok(bool)`: True if key existed and was deleted, false otherwise
2295    /// - `Err(DBError)`: If a database error occurs
2296    pub fn del_io_dup_vals(&self, db: &BytesDatabase, key: &[u8]) -> Result<bool, DBError> {
2297        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
2298
2299        // Create a write transaction
2300        let mut wtxn = env.write_txn()?;
2301
2302        // Try to delete the key and all its duplicates
2303        let result = match db.delete(&mut wtxn, key) {
2304            Ok(deleted) => {
2305                // Commit the transaction if we got this far
2306                wtxn.commit()?;
2307                Ok(deleted)
2308            }
2309            Err(e) => {
2310                // Don't leave the transaction hanging
2311                let _ = wtxn.abort();
2312
2313                // Convert heed errors to our own error type
2314                Err(DBError::KeyError(format!(
2315                    "Key: `{:?}` is either empty, too big (for lmdb), or wrong DUPFIXED size.",
2316                    key
2317                )))
2318            }
2319        };
2320        result
2321    }
2322
2323    /// Deletes dup io val at key in db. Performs strip search to find match.
2324    /// Strips proems and then searches.
2325    /// Returns True if delete else False if val not present
2326    /// Assumes DB opened with dupsort=True
2327    ///
2328    /// Duplicates at a given key preserve insertion order of duplicate.
2329    /// Because lmdb is lexocographic an insertion ordering proem is prepended to
2330    /// all values that makes lexocographic order that same as insertion order
2331    /// Duplicates are ordered as a pair of key plus value so prepending proem
2332    /// to each value changes duplicate ordering. Proem is 33 characters long.
2333    /// With 32 character hex string followed by '.' for essentially unlimited
2334    /// number of values which will be limited by memory.
2335    ///
2336    /// Does a linear search so not very efficient when not deleting from the front.
2337    /// This is hack for supporting escrow which needs to delete individual dup.
2338    /// The problem is that escrow is not fixed buts stuffs gets added and
2339    /// deleted which just adds to the value of the proem. 2**16 is an impossibly
2340    /// large number so the proem will not max out practically. But its not
2341    /// an elegant solution.
2342    ///
2343    /// # Parameters
2344    /// - `db`: Opened named sub db with dupsort=False
2345    /// - `key`: bytes of key within sub db's keyspace
2346    /// - `val`: bytes of value to be deleted without insertion ordering proem
2347    ///
2348    /// # Returns
2349    /// - `Ok(bool)`: True if deleted, false if val not present
2350    /// - `Err(DBError)`: If a database error occurs
2351    pub fn del_io_dup_val(
2352        &self,
2353        db: &BytesDatabase,
2354        key: &[u8],
2355        val: &[u8],
2356    ) -> Result<bool, DBError> {
2357        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
2358
2359        // Create a write transaction
2360        let mut wtxn = env.write_txn()?;
2361
2362        // First, collect the key-value pair to delete
2363        let to_delete = {
2364            let prefix_iter = match db.prefix_iter(&wtxn, &key) {
2365                Ok(iter) => iter,
2366                Err(_) => {
2367                    // Don't abort here, just return the error
2368                    return Err(DBError::KeyError(format!(
2369                        "Key: `{:?}` is either empty, too big (for lmdb), or wrong DUPFIXED size.",
2370                        key
2371                    )));
2372                }
2373            };
2374
2375            let mut found = None;
2376            for res in prefix_iter {
2377                match res {
2378                    Ok((k, proval)) => {
2379                        // Make sure we only process exact key matches
2380                        if k == key {
2381                            // Skip values that are too short (must be at least 33 bytes for the proem)
2382                            if proval.len() > 33 {
2383                                // Compare the value without the proem
2384                                if &proval[33..] == val {
2385                                    // Found the value to delete
2386                                    found = Some((k.to_vec(), proval.to_vec()));
2387                                    break;
2388                                }
2389                            }
2390                        }
2391                    }
2392                    Err(_) => {
2393                        // Don't abort here, just return the error
2394                        return Err(DBError::KeyError(format!(
2395                            "Key: `{:?}` is either empty, too big (for lmdb), or wrong DUPFIXED size.",
2396                            key
2397                        )));
2398                    }
2399                }
2400            }
2401            found
2402        };
2403
2404        // Now handle the result when the iterator is no longer active
2405        match to_delete {
2406            Some((k, v)) => {
2407                // Delete the specific key-value pair
2408                match db.delete_one_duplicate(&mut wtxn, &k, &v) {
2409                    Ok(deleted) => {
2410                        // Commit the transaction if deletion was successful
2411                        wtxn.commit()?;
2412                        Ok(deleted)
2413                    }
2414                    Err(e) => {
2415                        // Abort the transaction and return an error
2416                        let _ = wtxn.abort();
2417                        Err(DBError::EnvError(e))
2418                    }
2419                }
2420            }
2421            None => {
2422                // No value found to delete, abort the transaction and return false
2423                let _ = wtxn.abort();
2424                Ok(false)
2425            }
2426        }
2427    }
2428
2429    /// Return last added dup value at key in db in insertion order
2430    /// Returns None if no entry at key
2431    /// Removes prepended proem ordinal from val before returning
2432    /// Assumes DB opened with dupsort=True
2433    ///
2434    /// Duplicates at a given key preserve insertion order of duplicate.
2435    /// Because lmdb is lexocographic an insertion ordering proem is prepended to
2436    /// all values that makes lexocographic order that same as insertion order.
2437    ///
2438    /// Duplicates are ordered as a pair of key plus value so prepending proem
2439    /// to each value changes duplicate ordering. Proem is 33 characters long.
2440    /// With 32 character hex string followed by '.' for essentially unlimited
2441    /// number of values which will be limited by memory.
2442    ///
2443    /// # Parameters
2444    /// - `db`: Opened named sub db with dupsort=True
2445    /// - `key`: bytes of key within sub db's keyspace
2446    ///
2447    /// # Returns
2448    /// - `Ok(Option<Vec<u8>>)`: Last value with proem removed, or None if no entry exists
2449    /// - `Err(DBError)`: If a database error occurs
2450    pub fn get_io_dup_val_last(
2451        &self,
2452        db: &BytesDatabase,
2453        key: &[u8],
2454    ) -> Result<Option<Vec<u8>>, DBError> {
2455        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
2456
2457        // Create a read-only transaction
2458        let txn = env.read_txn()?;
2459
2460        // Since heed doesn't have direct last_dup functionality like LMDB's cursor,
2461        // we'll iterate through all duplicates and keep the last one
2462        let mut last_val: Option<Vec<u8>> = None;
2463
2464        // Use a prefix-based range to iterate through duplicate values
2465        let prefix_iter = match db.prefix_iter(&txn, &key) {
2466            Ok(iter) => iter,
2467            Err(_) => {
2468                return Err(DBError::KeyError(format!(
2469                    "Key: `{:?}` is either empty, too big (for lmdb), or wrong DUPFIXED size.",
2470                    key
2471                )));
2472            }
2473        };
2474
2475        // Iterate through all values and keep the last one
2476        for res in prefix_iter {
2477            match res {
2478                Ok((k, val)) => {
2479                    // Make sure we only process exact key matches
2480                    if k == key {
2481                        // Skip values that are too short (must be at least 33 bytes for the proem)
2482                        if val.len() > 33 {
2483                            // Store this value (with proem removed)
2484                            last_val = Some(Vec::from(&val[33..]));
2485                        }
2486                    }
2487                }
2488                Err(_) => {
2489                    return Err(DBError::KeyError(format!(
2490                        "Key: `{:?}` is either empty, too big (for lmdb), or wrong DUPFIXED size.",
2491                        key
2492                    )));
2493                }
2494            }
2495        }
2496
2497        Ok(last_val)
2498    }
2499    /// Iterates over top branch of db given by key of IoDup items where each value
2500    /// has 33 byte insertion ordinal number proem (prefixed) with separator.
2501    /// Automagically removes (strips) proem before returning items.
2502    ///
2503    /// Assumes DB opened with dupsort=True
2504    ///
2505    /// Because cursor.iternext() advances cursor after returning item its safe
2506    /// to delete the item within the iteration loop. Works for both dupsort==False and dupsort==True
2507    ///
2508    /// # Parameters
2509    /// - `db`: Opened named sub db with dupsort=True
2510    /// - `top`: truncated top key, a key space prefix to get all the items
2511    ///          from multiple branches of the key space. If top key is
2512    ///          empty then gets all items in database
2513    /// - `cb`: Callback function that processes key-value pairs with proem stripped from value
2514    ///
2515    /// # Returns
2516    /// - `Ok(count)`: Number of items processed
2517    /// - `Err(DBError)`: If a database error occurs
2518    ///
2519    /// Duplicates at a given key preserve insertion order of duplicate.
2520    /// Because lmdb is lexocographic an insertion ordering proem is prepended to
2521    /// all values that makes lexocographic order that same as insertion order.
2522    ///
2523    /// Duplicates are ordered as a pair of key plus value so prepending proem
2524    /// to each value changes duplicate ordering. Proem is 33 characters long.
2525    /// With 32 character hex string followed by '.' for essentially unlimited
2526    /// number of values which will be limited by memory.
2527    pub fn get_top_io_dup_item_iter<F>(
2528        &self,
2529        db: &BytesDatabase,
2530        top: &[u8],
2531        mut cb: F,
2532    ) -> Result<usize, DBError>
2533    where
2534        F: FnMut(&[u8], &[u8]) -> Result<bool, DBError>,
2535    {
2536        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
2537
2538        // Create a read-only transaction
2539        let txn = env.read_txn()?;
2540
2541        let mut count = 0;
2542
2543        // Get a range iterator for the top key prefix
2544        let range: Box<dyn Iterator<Item = Result<(&[u8], &[u8]), heed::Error>>> = if top.is_empty()
2545        {
2546            // Empty top means get all items
2547            Box::new(db.iter(&txn)?)
2548        } else {
2549            // Get items with the specified prefix
2550            Box::new(db.prefix_iter(&txn, top)?)
2551        };
2552
2553        // Iterate over all matching items
2554        for result in range {
2555            match result {
2556                Ok((key, value)) => {
2557                    // Skip values that are too short to have a proem
2558                    if value.len() <= 33 {
2559                        continue;
2560                    }
2561
2562                    // Strip the 33-byte proem from the value
2563                    let stripped_value = &value[33..];
2564
2565                    // Call the callback with the key and stripped value
2566                    count += 1;
2567                    if !cb(&key, stripped_value)? {
2568                        // If callback returns false, stop iteration
2569                        break;
2570                    }
2571                }
2572                Err(e) => return Err(DBError::EnvError(e)),
2573            }
2574        }
2575
2576        Ok(count)
2577    }
2578
2579    /// Add val bytes as dup at onkey consisting of key + sep + serialized on in db.
2580    /// Adds to existing values at key if any.
2581    /// Returns true if written else false if dup val already exists.
2582    ///
2583    /// Duplicates are inserted in lexicographic order not insertion order.
2584    /// LMDB does not insert a duplicate unless it is a unique value for that key.
2585    ///
2586    /// Does inclusion test to detect if duplicate already exists.
2587    ///
2588    /// # Parameters
2589    /// - `db`: Opened named sub db with dupsort=True
2590    /// - `key`: Key within sub db's keyspace
2591    /// - `on`: Ordinal number for the key (default: 0)
2592    /// - `val`: Value to add at onkey as dup
2593    /// - `sep`: Separator character for split (default: b'.')
2594    ///
2595    /// # Returns
2596    /// - `Ok(bool)`: True if duplicate val added at onkey, false if duplicate val preexists
2597    /// - `Err(DBError)`: If a database error occurs
2598    pub fn add_on_io_dup_val(
2599        &self,
2600        db: &BytesDatabase,
2601        key: &[u8],
2602        on: Option<u64>,
2603        val: &[u8],
2604        sep: Option<[u8; 1]>,
2605    ) -> Result<bool, DBError> {
2606        // Generate the on_key by combining key + sep + serialized on
2607        let on_val = on.unwrap_or(0);
2608        let onkey = on_key(key, on_val, sep);
2609
2610        // Use the existing add_io_dup_val method to add the value
2611        self.add_io_dup_val(db, &onkey, val)
2612    }
2613
2614    /// Appends val in order after last previous key with same prefix in db where
2615    /// full key has key prefix and serialized on suffix attached with sep and
2616    /// value has ordinal proem prefixed.
2617    ///
2618    /// Returns ordinal number (on) of appended entry. Appended on is 1 greater
2619    /// than previous latest on at the prefix.
2620    ///
2621    /// Works with either dupsort==True or False since always creates new full key.
2622    ///
2623    /// # Parameters
2624    /// - `db`: Named sub db in lmdb
2625    /// - `key`: Key within sub db's keyspace
2626    /// - `val`: Value to append
2627    /// - `sep`: Separator character for split (default: b'.')
2628    ///
2629    /// # Returns
2630    /// - `Ok(u64)`: Ordinal number of newly appended val
2631    /// - `Err(DBError)`: If a database error occurs
2632    pub fn append_on_io_dup_val(
2633        &self,
2634        db: &BytesDatabase,
2635        key: &[u8],
2636        val: &[u8],
2637        sep: Option<[u8; 1]>,
2638    ) -> Result<u64, DBError> {
2639        // Create ordering proem
2640        let mut proem = format!("{:032x}.", 0).into_bytes();
2641
2642        // Create a new value by prepending the ordering proem
2643        let mut new_val = Vec::with_capacity(proem.len() + val.len());
2644        new_val.append(&mut proem);
2645        new_val.extend_from_slice(val);
2646
2647        // Call the existing appendOnVal method
2648        self.append_on_val(db, key, &new_val, sep)
2649    }
2650
2651    /// Deletes all dup iovals at onkey consisting of key + sep + serialized on in db.
2652    ///
2653    /// Assumes DB opened with dupsort=True
2654    ///
2655    /// # Parameters
2656    /// - `db`: Opened named sub db with dupsort=True
2657    /// - `key`: Key within sub db's keyspace
2658    /// - `on`: Ordinal number for the key (default: 0)
2659    /// - `sep`: Separator character for split (default: b'.')
2660    ///
2661    /// # Returns
2662    /// - `Ok(bool)`: True if onkey present so all dups at onkey deleted
2663    ///                False if onkey not present
2664    /// - `Err(DBError)`: If a database error occurs
2665    pub fn del_on_io_dup_vals(
2666        &self,
2667        db: &BytesDatabase,
2668        key: &[u8],
2669        on: Option<u64>,
2670        sep: Option<[u8; 1]>,
2671    ) -> Result<bool, DBError> {
2672        // Generate the on_key by combining key + sep + serialized on
2673        let on_val = on.unwrap_or(0);
2674        let onkey = on_key(key, on_val, sep);
2675
2676        // Use the existing del_io_dup_vals method to delete all duplicates at the onkey
2677        self.del_io_dup_vals(db, &onkey)
2678    }
2679
2680    /// Deletes a specific dup ioval at key onkey consisting of key + sep + serialized on in db.
2681    ///
2682    /// Returns True if deleted else False if dup val not present.
2683    /// Assumes DB opened with dupsort=True.
2684    ///
2685    /// # Parameters
2686    /// - `db`: Opened named sub db with dupsort=True
2687    /// - `key`: Key within sub db's keyspace
2688    /// - `on`: Ordinal number for the key (default: 0)
2689    /// - `val`: Value to delete at onkey
2690    /// - `sep`: Separator character for split (default: b'.')
2691    ///
2692    /// # Returns
2693    /// - `Ok(bool)`: True if duplicate val found and deleted
2694    ///                False if duplicate val does not exist at onkey
2695    /// - `Err(DBError)`: If a database error occurs
2696    pub fn del_on_io_dup_val(
2697        &self,
2698        db: &BytesDatabase,
2699        key: &[u8],
2700        on: Option<u64>,
2701        val: &[u8],
2702        sep: Option<[u8; 1]>,
2703    ) -> Result<bool, DBError> {
2704        // Generate the on_key by combining key + sep + serialized on
2705        let on_val = on.unwrap_or(0);
2706        let onkey = on_key(key, on_val, sep);
2707
2708        // Use the existing del_io_dup_val method to delete the specific duplicate value
2709        self.del_io_dup_val(db, &onkey, val)
2710    }
2711
2712    /// Returns iterator of triples (key, on, val), at each key over all ordinal
2713    /// numbered keys with same key + sep + on in db.
2714    ///
2715    /// Values are sorted by onKey(key, on) where on is ordinal number int and key is
2716    /// prefix without the ordinal suffix. Values duplicates are sorted internally
2717    /// by hidden prefixed insertion order proem ordinal.
2718    ///
2719    /// Uses callback pattern instead of Python's yield/iterator for each triple of (key, on, val).
2720    /// When key is empty then retrieves the whole db.
2721    ///
2722    /// # Parameters
2723    /// - `db`: Named sub db in lmdb
2724    /// - `key`: Key within sub db's keyspace - when key is empty (None) then retrieves whole db
2725    /// - `on`: Ordinal number at which to initiate retrieval (default: 0)
2726    /// - `sep`: Separator character for split (default: b'.')
2727    /// - `callback`: Function to call for each (key, on, val) triple found
2728    ///
2729    /// # Returns
2730    /// - `Ok(())`: If iteration completes successfully
2731    /// - `Err(DBError)`: If a database error occurs
2732    pub fn get_on_io_dup_item_iter<F>(
2733        &self,
2734        db: &BytesDatabase,
2735        key: Option<&[u8]>,
2736        on: Option<u64>,
2737        sep: Option<[u8; 1]>,
2738        mut callback: F,
2739    ) -> Result<(), DBError>
2740    where
2741        F: FnMut(Vec<u8>, u64, Vec<u8>) -> Result<bool, DBError>,
2742    {
2743        // Use the existing get_on_item_iter which returns triples of (key, on, val)
2744        self.get_on_item_iter(db, key, on, sep, |k, o, val| {
2745            // Skip values that are too short (must be at least 33 bytes for the proem)
2746            if val.len() <= 33 {
2747                return Ok(true); // Continue iteration
2748            }
2749
2750            // Strip the 33-byte proem from the value (first 33 bytes)
2751            let stripped_val = Vec::from(&val[33..]);
2752
2753            // Call the callback with the key, ordinal, and stripped value
2754            callback(k, o, stripped_val)
2755        })
2756    }
2757
2758    /// Returns iterator over values at each key with same key + sep + on in db.
2759    ///
2760    /// Values are associated with keys formed by onKey(key, on) where on is
2761    /// an ordinal number int and key is prefix without the ordinal.
2762    /// Values duplicates are sorted internally by hidden prefixed insertion order
2763    /// proem ordinal.
2764    ///
2765    /// Uses callback pattern instead of Python's yield/iterator.
2766    ///
2767    /// # Parameters
2768    /// - `db`: Named sub db in lmdb
2769    /// - `key`: Key within sub db's keyspace - when key is empty then retrieves whole db
2770    /// - `on`: Ordinal number at which to initiate retrieval (default: 0)
2771    /// - `sep`: Separator character for split (default: b'.')
2772    /// - `callback`: Function to call for each value found
2773    ///
2774    /// # Returns
2775    /// - `Ok(())`: If iteration completes successfully
2776    /// - `Err(DBError)`: If a database error occurs
2777    pub fn get_on_io_dup_val_iter<F>(
2778        &self,
2779        db: &BytesDatabase,
2780        key: Option<&[u8]>,
2781        on: Option<u64>,
2782        sep: Option<[u8; 1]>,
2783        mut callback: F,
2784    ) -> Result<(), DBError>
2785    where
2786        F: FnMut(Vec<u8>) -> Result<bool, DBError>,
2787    {
2788        // Call get_on_io_dup_item_iter which returns triples of (key, on, val)
2789        // and extract only the values
2790        self.get_on_io_dup_item_iter(db, key, on, sep, |_key, _on, val| {
2791            // For each item, call the callback with just the value
2792            callback(val)
2793        })
2794    }
2795    /// Returns iterator-like access to triples (key, on, val) of last insertion ordered
2796    /// duplicate at each key over all ordinal numbered keys with same full key
2797    /// of key + sep + on in db. Values are sorted by on_key(key, on) where on is ordinal
2798    /// number int and key is prefix sans on.
2799    /// Values duplicates are sorted internally by hidden prefixed insertion order
2800    /// proem ordinal
2801    /// Callback receives triples of (key, on, val)
2802    ///
2803    /// When key is empty then retrieves whole db
2804    ///
2805    /// # Parameters
2806    /// * `db` - named sub db in lmdb
2807    /// * `key` - key within sub db's keyspace plus trailing part on (when empty, retrieves whole db)
2808    /// * `on` - ordinal number at which to initiate retrieval
2809    /// * `sep` - separator character for split
2810    /// * `callback` - function that processes each (key, on, val) triple, returns bool to continue or stop
2811    pub fn get_on_io_dup_last_item_iter<F>(
2812        &self,
2813        db: &BytesDatabase,
2814        key: Option<&[u8]>,
2815        on: Option<u64>,
2816        sep: Option<[u8; 1]>,
2817        mut callback: F,
2818    ) -> Result<(), DBError>
2819    where
2820        F: FnMut(Vec<u8>, u64, Vec<u8>) -> Result<bool, DBError>,
2821    {
2822        if !self.opened() {
2823            return Err(DBError::DbClosed);
2824        }
2825
2826        let env = self.env.as_ref().ok_or(DBError::DbClosed)?;
2827        let txn = env.read_txn()?;
2828        let sep_byte = sep.unwrap_or([b'.']);
2829
2830        if let Some(k) = key {
2831            // Original logic for when key is provided
2832            if !k.is_empty() {
2833                let start_key = keys::on_key(k, on.unwrap_or(0), Some(sep_byte));
2834                let mut current_key = start_key;
2835
2836                loop {
2837                    // Use range to get a "cursor" positioned at or after our key
2838                    let range = (Bound::Included(&current_key[..]), Bound::Unbounded);
2839                    let mut range_iter = db.range(&txn, &range)?;
2840
2841                    if let Some(Ok((found_key, _))) = range_iter.next() {
2842                        let found_key_vec = found_key.to_vec();
2843
2844                        // Check if we're still within the desired key range
2845                        let (base_key, found_on) =
2846                            keys::split_on_key(&found_key_vec, Some(sep_byte))?;
2847
2848                        if base_key != k {
2849                            // We've moved to a different base key
2850                            break;
2851                        }
2852
2853                        // For this key, we need to get the last duplicate value
2854                        let mut last_value: Option<Vec<u8>> = None;
2855                        let mut prefix_iter = db.prefix_iter(&txn, &found_key_vec)?;
2856
2857                        while let Some(Ok((_, val))) = prefix_iter.next() {
2858                            last_value = Some(val.to_vec());
2859                        }
2860
2861                        if let Some(val) = last_value {
2862                            // Process the value
2863                            let val_without_proem = if val.len() > 33 {
2864                                val[33..].to_vec()
2865                            } else {
2866                                val
2867                            };
2868
2869                            // Call the callback
2870                            let continue_iteration =
2871                                callback(base_key.clone(), found_on, val_without_proem)?;
2872                            if !continue_iteration {
2873                                return Ok(());
2874                            }
2875
2876                            // Move to next ordinal number
2877                            current_key = keys::on_key(&base_key, found_on + 1, Some(sep_byte));
2878                        } else {
2879                            // No duplicates found (this shouldn't happen)
2880                            // Move to next key just in case
2881                            current_key = keys::on_key(&base_key, found_on + 1, Some(sep_byte));
2882                        }
2883                    } else {
2884                        // No more keys in range
2885                        break;
2886                    }
2887                }
2888            }
2889        } else {
2890            // New handling for when key is None - iterate through all unique base keys
2891            // Get a range iterator for all keys
2892            let range = (Bound::Unbounded, Bound::Unbounded);
2893            let mut all_keys = db.range(&txn, &range)?;
2894
2895            // Keep track of the last seen base key and ordinal
2896            let mut last_base: Option<Vec<u8>> = None;
2897            let mut last_on: Option<u64> = None;
2898
2899            while let Some(Ok((found_key, _))) = all_keys.next() {
2900                let found_key_vec = found_key.to_vec();
2901
2902                // Extract base key and ordinal
2903                let (base_key, found_on) = keys::split_on_key(&found_key_vec, Some(sep_byte))?;
2904
2905                // Skip if we've already processed this base+on combination
2906                if last_base.as_ref() == Some(&base_key) && last_on == Some(found_on) {
2907                    continue;
2908                }
2909
2910                // Get all values for this key to find the last one
2911                let mut last_value: Option<Vec<u8>> = None;
2912                let mut prefix_iter = db.prefix_iter(&txn, &found_key_vec)?;
2913
2914                while let Some(Ok((_, val))) = prefix_iter.next() {
2915                    last_value = Some(val.to_vec());
2916                }
2917
2918                if let Some(val) = last_value {
2919                    // Process the value
2920                    let val_without_proem = if val.len() > 33 {
2921                        val[33..].to_vec()
2922                    } else {
2923                        val
2924                    };
2925
2926                    // Call the callback
2927                    let continue_iteration =
2928                        callback(base_key.clone(), found_on, val_without_proem)?;
2929                    if !continue_iteration {
2930                        return Ok(());
2931                    }
2932                }
2933
2934                // Update last seen key and ordinal
2935                last_base = Some(base_key);
2936                last_on = Some(found_on);
2937            }
2938        }
2939
2940        Ok(())
2941    }
2942
2943    /// Returns iterator going backwards of triples (key, on, val), of insertion ordered
2944    /// item at each key over all ordinal numbered keys with same full key of key + sep + on in db.
2945    ///
2946    /// Values are sorted by onKey(key, on) where on is ordinal number int and
2947    /// key is prefix sans on. Values duplicates are sorted internally by hidden prefixed
2948    /// insertion order proem ordinal.
2949    ///
2950    /// Backwards means decreasing numerical value of duplicate proem, for each on,
2951    /// decreasing numerical value on for each key and decreasing lexicographic
2952    /// order of each key prefix.
2953    ///
2954    /// Uses callback pattern instead of Python's yield/iterator for each triple of (key, on, val).
2955    /// When key is empty then retrieves the whole db.
2956    ///
2957    /// # Parameters
2958    /// - `db`: Named sub db in lmdb
2959    /// - `key`: Key within sub db's keyspace - when key is empty (None) then retrieves whole db
2960    /// - `on`: Ordinal number at which to initiate retrieval
2961    /// - `sep`: Separator character for split (default: b'.')
2962    /// - `callback`: Function to call for each (key, on, val) triple found
2963    ///
2964    /// # Returns
2965    /// - `Ok(())`: If iteration completes successfully
2966    /// - `Err(DBError)`: If a database error occurs
2967    pub fn get_on_io_dup_item_back_iter<F>(
2968        &self,
2969        db: &BytesDatabase,
2970        key: Option<&[u8]>,
2971        on: Option<u64>,
2972        sep: Option<[u8; 1]>,
2973        mut callback: F,
2974    ) -> Result<(), DBError>
2975    where
2976        F: FnMut(Vec<u8>, u64, Vec<u8>) -> Result<bool, DBError>,
2977    {
2978        let env = match self.env() {
2979            Some(env) => env,
2980            None => return Err(DBError::DbClosed),
2981        };
2982
2983        let txn = env.read_txn().map_err(DBError::from)?;
2984        let sep_byte = sep.unwrap_or([b'.']);
2985
2986        // If no key is provided, we need to iterate over the entire database backwards
2987        if key.is_none() {
2988            // For full database iteration, we'll need to collect and sort all entries
2989            let mut all_entries: Vec<(Vec<u8>, u64, Vec<u8>, Vec<u8>)> = Vec::new();
2990
2991            // First collect all entries
2992            let iter = db.iter(&txn).map_err(DBError::from)?;
2993            for result in iter {
2994                let (on_key, val) = result.map_err(DBError::from)?;
2995
2996                if let Ok((ckey, on_val)) = keys::split_on_key(&on_key, Some(sep_byte)) {
2997                    // Skip values that are too short
2998                    if val.len() <= 33 {
2999                        continue;
3000                    }
3001
3002                    // Extract the proem (first 33 bytes) for sorting
3003                    let proem = Vec::from(&val[..33]);
3004
3005                    // Extract the value portion after the header
3006                    let stripped_val = Vec::from(&val[33..]);
3007
3008                    // Store the key, ordinal, proem, and value
3009                    all_entries.push((ckey, on_val, proem, stripped_val));
3010                }
3011            }
3012
3013            // Sort by key (reverse), then by ordinal (reverse), then by proem (reverse)
3014            all_entries.sort_by(|(key1, on1, proem1, _), (key2, on2, proem2, _)| {
3015                match key2.cmp(key1) {
3016                    std::cmp::Ordering::Equal => match on2.cmp(on1) {
3017                        std::cmp::Ordering::Equal => proem2.cmp(proem1),
3018                        other => other,
3019                    },
3020                    other => other,
3021                }
3022            });
3023
3024            // Process the sorted entries
3025            for (ckey, on_val, _, stripped_val) in all_entries {
3026                let continue_iteration = callback(ckey, on_val, stripped_val)?;
3027                if !continue_iteration {
3028                    break;
3029                }
3030            }
3031
3032            return Ok(());
3033        }
3034
3035        // Get the key and target ordinal
3036        let key_bytes = key.unwrap();
3037        let max_on = on.unwrap_or(u64::MAX);
3038
3039        // Create a prefix for our key (key + separator)
3040        let mut prefix = Vec::with_capacity(key_bytes.len() + 1);
3041        prefix.extend_from_slice(key_bytes);
3042        prefix.push(sep_byte[0]);
3043
3044        // We need to collect all entries for each ordinal to properly sort them
3045        let mut entries_by_ordinal: std::collections::HashMap<u64, Vec<(Vec<u8>, Vec<u8>)>> =
3046            std::collections::HashMap::new();
3047
3048        // Format the max ordinal as 32 hex digits with leading zeros
3049        let mut upper_key = prefix.clone();
3050        let on_str = format!("{:032x}", max_on);
3051        upper_key.extend_from_slice(on_str.as_bytes());
3052
3053        // The key range we want is from prefix (exclusive) to upper_key (inclusive)
3054        // For rev_range, we still specify it in ascending order, but iteration will be in reverse
3055        let range = (
3056            Bound::Excluded(&prefix[..]),
3057            Bound::Included(&upper_key[..]),
3058        );
3059
3060        // Create iterator for the range
3061        let range_iter = db.range(&txn, &range).map_err(DBError::from)?;
3062
3063        // Collect all entries in the range, grouped by ordinal
3064        for result in range_iter {
3065            let (on_key, val) = result.map_err(DBError::from)?;
3066
3067            // Skip values that are too short
3068            if val.len() <= 33 {
3069                continue;
3070            }
3071
3072            if let Ok((ckey, on_val)) = keys::split_on_key(&on_key, Some(sep_byte)) {
3073                // Since we're already filtering by prefix in the range, ckey should equal key_bytes
3074                if !ckey.eq(key_bytes) || on_val > max_on {
3075                    continue;
3076                }
3077
3078                // Extract the proem for sorting
3079                let proem = Vec::from(&val[..33]);
3080
3081                // Extract the value portion after the header
3082                let stripped_val = Vec::from(&val[33..]);
3083
3084                // Add to the collection for this ordinal
3085                entries_by_ordinal
3086                    .entry(on_val)
3087                    .or_default()
3088                    .push((proem, stripped_val));
3089            }
3090        }
3091
3092        // Process ordinals in reverse order
3093        let mut ordinals: Vec<u64> = entries_by_ordinal.keys().cloned().collect();
3094        ordinals.sort_by(|a, b| b.cmp(a)); // Sort in reverse
3095
3096        for ordinal in ordinals {
3097            let entries = entries_by_ordinal.get_mut(&ordinal).unwrap();
3098
3099            // Sort entries for this ordinal by proem in reverse order
3100            entries.sort_by(|(proem1, _), (proem2, _)| proem2.cmp(proem1));
3101
3102            // Process each entry
3103            for (_, stripped_val) in entries.drain(..) {
3104                let continue_iteration = callback(key_bytes.to_vec(), ordinal, stripped_val)?;
3105                if !continue_iteration {
3106                    return Ok(());
3107                }
3108            }
3109        }
3110
3111        Ok(())
3112    }
3113
3114    /// Returns iterator of val of last insertion ordered duplicate at each
3115    /// key over all ordinal numbered keys with same full key
3116    /// of key + sep + on in db. Values are sorted by onKey(key, on) where on
3117    /// is ordinal number int and key is prefix sans on.
3118    /// Values duplicates are sorted internally by hidden prefixed insertion order
3119    /// proem ordinal
3120    ///
3121    /// when key is empty then retrieves whole db
3122    ///
3123    /// # Arguments
3124    /// * `db` - named sub db in lmdb
3125    /// * `key` - key within sub db's keyspace plus trailing part on
3126    ///           when key is empty then retrieves whole db
3127    /// * `on` - ordinal number at which to initiate retrieval
3128    /// * `sep` - separator character for split
3129    ///
3130    /// # Returns
3131    /// Callback invoked for each last dup val at each onkey
3132    pub fn get_on_io_dup_last_val_iter<F>(
3133        &self,
3134        db: &BytesDatabase,
3135        key: Option<&[u8]>,
3136        on: Option<u64>,
3137        sep: Option<[u8; 1]>,
3138        mut callback: F,
3139    ) -> Result<(), DBError>
3140    where
3141        F: FnMut(Vec<u8>) -> Result<bool, DBError>,
3142    {
3143        self.get_on_io_dup_last_item_iter(db, key, on, sep, |_key, _on, val| callback(val))
3144    }
3145
3146    /// Returns iterator going backwards of values,
3147    /// of insertion ordered item at each key over all ordinal numbered keys
3148    /// with same full key of key + sep + on in db.
3149    /// Values are sorted by onKey(key, on) where on is ordinal number int and
3150    /// key is prefix sans on.
3151    /// Values duplicates are sorted internally by hidden prefixed insertion order
3152    /// proem ordinal
3153    /// Backwards means decreasing numerical value of duplicate proem, for each on,
3154    /// decreasing numerical value on for each key and decreasing lexicographic
3155    /// order of each key prefix.
3156    ///
3157    /// when key is empty then retrieves whole db
3158    ///
3159    /// # Arguments
3160    /// * `db` - named sub db in lmdb
3161    /// * `key` - key within sub db's keyspace plus trailing part on
3162    ///           when key is empty then retrieves whole db
3163    /// * `on` - ordinal number at which to initiate retrieval
3164    /// * `sep` - separator character for split
3165    ///
3166    /// # Returns
3167    /// Callback invoked for each val in backwards order
3168    pub fn get_on_io_dup_val_back_iter<F>(
3169        &self,
3170        db: &BytesDatabase,
3171        key: Option<&[u8]>,
3172        on: Option<u64>,
3173        sep: Option<[u8; 1]>,
3174        mut callback: F,
3175    ) -> Result<(), DBError>
3176    where
3177        F: FnMut(Vec<u8>) -> Result<bool, DBError>,
3178    {
3179        self.get_on_io_dup_item_back_iter(db, key, on, sep, |_key, _on, val| callback(val))
3180    }
3181}
3182
3183impl Drop for LMDBer {
3184    fn drop(&mut self) {
3185        // Clean up resources when dropped
3186        let _ = self.close(false);
3187    }
3188}
3189
3190#[cfg(test)]
3191mod tests {
3192    use super::*;
3193    use std::path::Path;
3194
3195    use crate::keri::db::dbing::keys::{dg_key, sn_key};
3196
3197    #[test]
3198    fn test_lmdber() {
3199        // Test LMDBer creation
3200        let mut databaser = LMDBer::builder()
3201            .name("main")
3202            .temp(false)
3203            .build()
3204            .expect("Failed to create LMDBer");
3205
3206        assert_eq!(databaser.name(), "main");
3207        assert_eq!(databaser.temp(), false);
3208        assert!(
3209            databaser.env().is_some(),
3210            "Environment should be initialized"
3211        );
3212
3213        // Check path ends with "keri/db/main" or ".keri/db/main" (alt path)
3214        let path_str = databaser.path().expect("Path should be available");
3215        let ends_with_keri = path_str.ends_with(&format!(
3216            "keri{}db{}main",
3217            std::path::MAIN_SEPARATOR,
3218            std::path::MAIN_SEPARATOR
3219        ));
3220        let ends_with_dot_keri = path_str.ends_with(&format!(
3221            ".keri{}db{}main",
3222            std::path::MAIN_SEPARATOR,
3223            std::path::MAIN_SEPARATOR
3224        ));
3225        assert!(
3226            ends_with_keri || ends_with_dot_keri,
3227            "Path {:?} should end with keri/db/main or .keri/db/main",
3228            path_str
3229        );
3230
3231        // Check path exists
3232        assert!(Path::new(&path_str).exists());
3233        assert!(databaser.opened());
3234
3235        // Test key generation functions
3236        let pre = b"BAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc";
3237        let dig = b"EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4";
3238        let sn = 3;
3239
3240        let expected_sn_key =
3241            b"BAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc.00000000000000000000000000000003";
3242        let expected_dg_key = b"BAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc.EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4";
3243
3244        assert_eq!(sn_key(pre, sn), expected_sn_key);
3245        assert_eq!(dg_key(pre, dig), expected_dg_key);
3246
3247        // Close and clear the database
3248        databaser.close(true).expect("Failed to close database");
3249        assert!(!Path::new(&path_str).exists());
3250        assert!(!databaser.opened());
3251
3252        // Test not opened on init
3253        let mut databaser = LMDBer::builder()
3254            .name("main")
3255            .temp(false)
3256            .reopen(false)
3257            .build()
3258            .expect("Failed to create LMDBer with reopen=false");
3259
3260        assert_eq!(databaser.name(), "main");
3261        assert_eq!(databaser.temp(), false);
3262        assert!(!databaser.opened());
3263        assert!(databaser.path().is_none());
3264        assert!(databaser.env().is_none());
3265
3266        // Reopen the database
3267        databaser
3268            .reopen(Some(false), None, None, false, false, false, None, None)
3269            .expect("Failed to reopen database");
3270
3271        assert!(databaser.opened());
3272        assert!(databaser.env().is_some());
3273
3274        let path_str = databaser
3275            .path()
3276            .expect("Path should be available after reopen");
3277        let ends_with_keri = path_str.ends_with(&format!(
3278            "keri{}db{}main",
3279            std::path::MAIN_SEPARATOR,
3280            std::path::MAIN_SEPARATOR
3281        ));
3282        let ends_with_dot_keri = path_str.ends_with(&format!(
3283            ".keri{}db{}main",
3284            std::path::MAIN_SEPARATOR,
3285            std::path::MAIN_SEPARATOR
3286        ));
3287        assert!(
3288            ends_with_keri || ends_with_dot_keri,
3289            "Path {:?} should end with keri/db/main or .keri/db/main",
3290            path_str
3291        );
3292
3293        assert!(Path::new(&path_str).exists());
3294
3295        // Test key generation functions again
3296        assert_eq!(sn_key(pre, sn), expected_sn_key);
3297        assert_eq!(dg_key(pre, dig), expected_dg_key);
3298
3299        // Close and clear the database again
3300        databaser.close(true).expect("Failed to close database");
3301        assert!(!Path::new(&path_str).exists());
3302        assert!(!databaser.opened());
3303    }
3304
3305    #[test]
3306    fn test_lmdb_basic_operations() -> Result<(), DBError> {
3307        // Create a temporary LMDBer instance
3308        let mut lmdber = LMDBer::builder().temp(true).build()?;
3309
3310        // Scope to ensure dber is dropped properly (similar to Python's with statement)
3311        {
3312            // Assert that temp is true
3313            assert_eq!(lmdber.temp(), true);
3314
3315            // Define key and value
3316            let key = b"A".to_vec();
3317            let val = b"whatever".to_vec();
3318
3319            // Open a database named "beep."
3320            let db = lmdber
3321                .create_database(Some("beeb."), None)
3322                .expect("Failed to create database");
3323
3324            // Test get_val on non-existent key
3325            let result = lmdber.get_val(&db, &key)?;
3326            assert_eq!(result, None);
3327
3328            // Test del_val on non-existent key
3329            let result = lmdber.del_val(&db, &key)?;
3330            assert_eq!(result, false);
3331
3332            // Test put_val (first time should return true)
3333            let result = lmdber.put_val(&db, &key, &val)?;
3334            assert_eq!(result, true);
3335
3336            // Test put_val again (second time should return false since key already exists)
3337            let result = lmdber.put_val(&db, &key, &val)?;
3338            assert_eq!(result, false);
3339
3340            // Test set_val (should return true as it overwrites)
3341            let result = lmdber.set_val(&db, &key, &val)?;
3342            assert_eq!(result, true);
3343
3344            // Test get_val to verify value was stored
3345            let result = lmdber.get_val(&db, &key)?;
3346            assert_eq!(result, Some(val.clone()));
3347
3348            // Test del_val to remove the key-value pair
3349            let result = lmdber.del_val(&db, &key)?;
3350            assert_eq!(result, true);
3351
3352            // Test get_val again to confirm deletion
3353            let result = lmdber.get_val(&db, &key)?;
3354            assert_eq!(result, None);
3355        }
3356
3357        lmdber.close(true)?;
3358        Ok(())
3359    }
3360
3361    #[test]
3362    fn test_get_top_item_iter_and_del_top_val() -> Result<(), DBError> {
3363        // Create a new LMDBer instance with a temporary database
3364        let mut lmdber = LMDBer::new(
3365            "test_db", "", true,  // temp
3366            None,  // head dir path
3367            None,  // perm
3368            true,  // reopen
3369            true,  // clear
3370            false, // reuse
3371            false, // clean
3372            false, // filed
3373            false, // extensioned
3374            None,  // mode
3375            None,  // fext
3376            false, // readonly
3377        )?;
3378
3379        // Create a test database
3380        let db = lmdber
3381            .create_database(Some("test_db"), None)
3382            .expect("Failed to open database");
3383
3384        // Insert test values
3385        let key = b"a.1".to_vec();
3386        let val = b"wow".to_vec();
3387        assert!(lmdber.put_val(&db, &key, &val)?);
3388
3389        let key = b"a.2".to_vec();
3390        let val = b"wee".to_vec();
3391        assert!(lmdber.put_val(&db, &key, &val)?);
3392
3393        let key = b"b.1".to_vec();
3394        let val = b"woo".to_vec();
3395        assert!(lmdber.put_val(&db, &key, &val)?);
3396
3397        // Test get_top_item_iter to retrieve all items
3398        let mut items = Vec::new();
3399        lmdber.get_top_items_iter(&db, b"", |k, v| {
3400            items.push((k.to_vec(), v.to_vec()));
3401            Ok(true)
3402        })?;
3403
3404        assert_eq!(
3405            items,
3406            vec![
3407                (b"a.1".to_vec(), b"wow".to_vec()),
3408                (b"a.2".to_vec(), b"wee".to_vec()),
3409                (b"b.1".to_vec(), b"woo".to_vec()),
3410            ]
3411        );
3412
3413        // Test deleting values with a specific prefix
3414        assert!(lmdber.del_top_val(&db, b"a.")?);
3415
3416        // Test get_top_item_iter after deletion
3417        let mut items = Vec::new();
3418        lmdber.get_top_items_iter(&db, b"", |k, v| {
3419            items.push((k.to_vec(), v.to_vec()));
3420            Ok(true)
3421        })?;
3422
3423        assert_eq!(items, vec![(b"b.1".to_vec(), b"woo".to_vec()),]);
3424
3425        // Clean up
3426        lmdber.close(true)?;
3427
3428        Ok(())
3429    }
3430
3431    #[test]
3432    fn test_cnt() -> Result<(), DBError> {
3433        // Create a temporary directory for the database
3434        // Create a new LMDBer instance with a temporary database
3435        let mut lmdber = LMDBer::new(
3436            "test_db", "", true,  // temp
3437            None,  // head_dir_path
3438            None,  // perm
3439            true,  // reopen
3440            true,  // clear
3441            false, // reuse
3442            false, // clean
3443            false, // filed
3444            false, // extensioned
3445            None,  // mode
3446            None,  // fext
3447            false, // readonly
3448        )?;
3449
3450        // Create a test database with dupsort flag
3451        let db = lmdber
3452            .create_database(Some("test_db"), Some(true))
3453            .expect("Failed to open database");
3454
3455        // Test empty database count
3456        assert_eq!(lmdber.cnt(&db)?, 0);
3457
3458        // Insert test values
3459        let key0 = b"key0".to_vec();
3460        let key1 = b"key1".to_vec();
3461        let val1 = b"val1".to_vec();
3462        let val2 = b"val2".to_vec();
3463
3464        assert!(lmdber.put_val(&db, &key0, &val1)?);
3465        assert!(lmdber.put_val(&db, &key1, &val2)?); // With DUP_SORT, we can have multiple values for the same key
3466
3467        // Test count after insertion
3468        assert_eq!(lmdber.cnt(&db)?, 2);
3469
3470        // Add more items
3471        let key2 = b"key2".to_vec();
3472        let val3 = b"val3".to_vec();
3473        assert!(lmdber.put_val(&db, &key2, &val3)?);
3474
3475        // Test count again
3476        assert_eq!(lmdber.cnt(&db)?, 3);
3477
3478        // Delete items
3479        assert!(lmdber.del_val(&db, &key1)?);
3480
3481        // Test count after deletion
3482        assert_eq!(lmdber.cnt(&db)?, 2);
3483
3484        // Clean up
3485        lmdber.close(true)?;
3486
3487        Ok(())
3488    }
3489    #[test]
3490    fn test_dup_vals_methods() -> Result<(), DBError> {
3491        // Set up temporary database
3492        let dber = LMDBer::builder().temp(true).build()?;
3493
3494        let key = b"A";
3495        let vals = [b"z", b"m", b"x", b"a"];
3496
3497        // Create a database with dupsort enabled
3498        let db = dber.create_database(Some("boop."), Some(true))?;
3499
3500        // Test initial empty state
3501        let mut retrieved_vals = Vec::new();
3502        dber.get_vals_iter(&db, key, |v| {
3503            retrieved_vals.push(v.to_vec());
3504            Ok(true)
3505        })?;
3506        assert_eq!(retrieved_vals, Vec::<Vec<u8>>::new());
3507
3508        assert_eq!(dber.del_vals(&db, key, None)?, false);
3509        assert_eq!(dber.cnt_vals(&db, key)?, 0);
3510
3511        // Test putting values
3512        assert_eq!(dber.put_vals(&db, key, &vals)?, true);
3513
3514        // Values should be stored in lexicographic order
3515        let mut retrieved_vals = Vec::new();
3516        dber.get_vals_iter(&db, key, |v| {
3517            retrieved_vals.push(v.to_vec());
3518            Ok(true)
3519        })?;
3520
3521        // Convert retrieved_vals Vec<Vec<u8>> to Vec<&[u8]> for easier comparison
3522        let retrieved_vals_refs: Vec<&[u8]> = retrieved_vals.iter().map(|v| v.as_slice()).collect();
3523        assert_eq!(retrieved_vals_refs, [b"a", b"m", b"x", b"z"]);
3524
3525        // Test count
3526        assert_eq!(dber.cnt_vals(&db, key)?, vals.len());
3527
3528        // Test putting a duplicate value - should succeed but not change stored values
3529        assert_eq!(dber.put_vals(&db, key, &[b"a"])?, true);
3530
3531        let mut retrieved_vals = Vec::new();
3532        dber.get_vals_iter(&db, key, |v| {
3533            retrieved_vals.push(v.to_vec());
3534            Ok(true)
3535        })?;
3536        let retrieved_vals_refs: Vec<&[u8]> = retrieved_vals.iter().map(|v| v.as_slice()).collect();
3537        assert_eq!(retrieved_vals_refs, [b"a", b"m", b"x", b"z"]);
3538
3539        // Test add_val with existing and new values
3540        assert_eq!(dber.add_val(&db, key, b"a")?, false); // duplicate
3541        assert_eq!(dber.add_val(&db, key, b"b")?, true); // new value
3542
3543        let mut retrieved_vals = Vec::new();
3544        dber.get_vals_iter(&db, key, |v| {
3545            retrieved_vals.push(v.to_vec());
3546            Ok(true)
3547        })?;
3548        let retrieved_vals_refs: Vec<&[u8]> = retrieved_vals.iter().map(|v| v.as_slice()).collect();
3549        assert_eq!(retrieved_vals_refs, [b"a", b"b", b"m", b"x", b"z"]);
3550
3551        // Test get_vals_iter explicitly
3552        let mut iter_vals = Vec::new();
3553        dber.get_vals_iter(&db, key, |v| {
3554            iter_vals.push(v.to_vec());
3555            Ok(true)
3556        })?;
3557        let iter_vals_refs: Vec<&[u8]> = iter_vals.iter().map(|v| v.as_slice()).collect();
3558        assert_eq!(iter_vals_refs, [b"a", b"b", b"m", b"x", b"z"]);
3559
3560        // Test the get_val_last function
3561        let last_val = dber.get_val_last(&db, key)?;
3562        assert_eq!(last_val, Some(b"z".to_vec()));
3563
3564        // Test deleting all values
3565        assert_eq!(dber.del_vals(&db, key, None)?, true);
3566
3567        let mut retrieved_vals = Vec::new();
3568        dber.get_vals_iter(&db, key, |v| {
3569            retrieved_vals.push(v.to_vec());
3570            Ok(true)
3571        })?;
3572        assert_eq!(retrieved_vals, Vec::<Vec<u8>>::new());
3573
3574        // Test deleting individual values
3575        assert_eq!(dber.put_vals(&db, key, &vals)?, true);
3576
3577        for val in &vals {
3578            assert_eq!(dber.del_vals(&db, key, Some(*val))?, true);
3579        }
3580
3581        let mut retrieved_vals = Vec::new();
3582        dber.get_vals_iter(&db, key, |v| {
3583            retrieved_vals.push(v.to_vec());
3584            Ok(true)
3585        })?;
3586        assert_eq!(retrieved_vals, Vec::<Vec<u8>>::new());
3587
3588        // Test deleting values while iterating
3589        assert_eq!(dber.put_vals(&db, key, &vals)?, true);
3590
3591        let mut vals_to_delete = Vec::new();
3592        dber.get_vals_iter(&db, key, |v| {
3593            vals_to_delete.push(v.to_vec());
3594            Ok(true)
3595        })?;
3596
3597        for val in vals_to_delete {
3598            assert_eq!(dber.del_vals(&db, key, Some(&val))?, true);
3599        }
3600
3601        let mut retrieved_vals = Vec::new();
3602        dber.get_vals_iter(&db, key, |v| {
3603            retrieved_vals.push(v.to_vec());
3604            Ok(true)
3605        })?;
3606        assert_eq!(retrieved_vals, Vec::<Vec<u8>>::new());
3607
3608        Ok(())
3609    }
3610
3611    #[test]
3612    fn test_io_dup_vals_methods() -> Result<(), DBError> {
3613        // Set up temporary database
3614        let dber = LMDBer::builder().temp(true).build()?;
3615
3616        let key = b"A";
3617        let vals = [b"z", b"m", b"x", b"a"];
3618
3619        // Create a database with dupsort enabled
3620        let db = dber.create_database(Some("peep."), Some(true))?;
3621
3622        // Test initial empty state
3623        assert_eq!(dber.get_io_dup_vals(&db, key)?, Vec::<Vec<u8>>::new());
3624        assert_eq!(dber.get_io_dup_val_last(&db, key)?, None);
3625        assert_eq!(dber.cnt_io_dup_vals(&db, key)?, 0);
3626        assert_eq!(dber.del_io_dup_vals(&db, key)?, false);
3627
3628        // Test putting values
3629        assert_eq!(dber.put_io_dup_vals(&db, key, &vals)?, true);
3630
3631        // Values should be stored in insertion order (not lexicographically)
3632        let retrieved_vals = dber.get_io_dup_vals(&db, key)?;
3633
3634        // Convert retrieved_vals to Vec<&[u8]> for easier comparison
3635        let retrieved_vals_refs: Vec<&[u8]> = retrieved_vals.iter().map(|v| v.as_slice()).collect();
3636
3637        // Check that values are preserved in insertion order, not sorted
3638        assert_eq!(retrieved_vals_refs, vals);
3639
3640        // Test count
3641        assert_eq!(dber.cnt_io_dup_vals(&db, key)?, vals.len());
3642
3643        // Test get_io_dup_val_last
3644        assert_eq!(dber.get_io_dup_val_last(&db, key)?, Some(vals[3].to_vec()));
3645
3646        // Test putting a duplicate value - should fail because 'a' already exists
3647        assert_eq!(dber.put_io_dup_vals(&db, key, &[b"a"])?, false);
3648
3649        // Values should be unchanged
3650        let retrieved_vals = dber.get_io_dup_vals(&db, key)?;
3651        let retrieved_vals_refs: Vec<&[u8]> = retrieved_vals.iter().map(|v| v.as_slice()).collect();
3652        assert_eq!(retrieved_vals_refs, vals);
3653
3654        // Test add_io_dup_val
3655        assert_eq!(dber.add_io_dup_val(&db, key, b"b")?, true); // new value
3656        assert_eq!(dber.add_io_dup_val(&db, key, b"a")?, false); // duplicate
3657
3658        // Check values after adding 'b'
3659        let expected_vals = vec![b"z", b"m", b"x", b"a", b"b"];
3660        let retrieved_vals = dber.get_io_dup_vals(&db, key)?;
3661        let retrieved_vals_refs: Vec<&[u8]> = retrieved_vals.iter().map(|v| v.as_slice()).collect();
3662        assert_eq!(retrieved_vals_refs, expected_vals);
3663
3664        // Test using iterator to get values
3665        let mut iter_vals = Vec::new();
3666        dber.get_io_dup_vals_iter(&db, key, |v| {
3667            iter_vals.push(v.to_vec());
3668            Ok(true)
3669        })?;
3670        let iter_vals_refs: Vec<&[u8]> = iter_vals.iter().map(|v| v.as_slice()).collect();
3671        assert_eq!(iter_vals_refs, expected_vals);
3672
3673        // Test deleting all values
3674        assert_eq!(dber.del_io_dup_vals(&db, key)?, true);
3675        assert_eq!(dber.get_io_dup_vals(&db, key)?, Vec::<Vec<u8>>::new());
3676
3677        // Test deleting individual values
3678        assert_eq!(dber.put_io_dup_vals(&db, key, &vals)?, true);
3679
3680        for val in &vals {
3681            assert_eq!(dber.del_io_dup_val(&db, key, *val)?, true);
3682        }
3683
3684        assert_eq!(dber.get_io_dup_vals(&db, key)?, Vec::<Vec<u8>>::new());
3685
3686        // Test deleting sorted values
3687        assert_eq!(dber.put_io_dup_vals(&db, key, &vals)?, true);
3688
3689        let mut sorted_vals = vals.to_vec();
3690        sorted_vals.sort();
3691
3692        for val in sorted_vals {
3693            assert_eq!(dber.del_io_dup_val(&db, key, val)?, true);
3694        }
3695
3696        assert_eq!(dber.get_io_dup_vals(&db, key)?, Vec::<Vec<u8>>::new());
3697
3698        // Test delete and add in odd order (matches Python test)
3699        assert_eq!(dber.put_io_dup_vals(&db, key, &vals)?, true);
3700        assert_eq!(dber.del_io_dup_val(&db, key, vals[2])?, true); // Delete "x"
3701        assert_eq!(dber.add_io_dup_val(&db, key, b"w")?, true); // Add "w"
3702        assert_eq!(dber.del_io_dup_val(&db, key, vals[0])?, true); // Delete "z"
3703        assert_eq!(dber.add_io_dup_val(&db, key, b"e")?, true); // Add "e"
3704
3705        // Final state should be [b"m", b"a", b"w", b"e"]
3706        let expected_vals = vec![b"m", b"a", b"w", b"e"];
3707        let retrieved_vals = dber.get_io_dup_vals(&db, key)?;
3708        let retrieved_vals_refs: Vec<&[u8]> = retrieved_vals.iter().map(|v| v.as_slice()).collect();
3709        assert_eq!(retrieved_vals_refs, expected_vals);
3710
3711        // Test the top iterator
3712        // First, add some more values with different keys
3713        let key2 = b"B";
3714        assert_eq!(dber.put_io_dup_vals(&db, key2, &[b"1", b"2", b"3"])?, true);
3715
3716        // Use get_top_io_dup_item_iter to traverse all keys
3717        let mut all_items = Vec::new();
3718        dber.get_top_io_dup_item_iter(&db, b"", |k, v| {
3719            all_items.push((k.to_vec(), v.to_vec()));
3720            Ok(true)
3721        })?;
3722
3723        // Check that we get all items with their proems stripped
3724        assert_eq!(all_items.len(), 7); // 4 for key "A" + 3 for key "B"
3725
3726        // Test with a specific key prefix
3727        let mut a_items = Vec::new();
3728        dber.get_top_io_dup_item_iter(&db, b"A", |k, v| {
3729            a_items.push((k.to_vec(), v.to_vec()));
3730            Ok(true)
3731        })?;
3732
3733        assert_eq!(a_items.len(), 4); // Only the 4 items for key "A"
3734
3735        Ok(())
3736    }
3737
3738    #[test]
3739    fn test_on_key_value_methods() -> Result<(), DBError> {
3740        // Set up a temporary directory for the test
3741        // Create a new LMDBer instance for testing
3742        let lmdber = LMDBer::builder().name("test_db").temp(true).build()?;
3743
3744        // Create "seen." database
3745        let db = lmdber.create_database(Some("seen."), None)?;
3746
3747        // Define test prefixes and digests
3748        let pre_a = b"BBKY1sKmgyjAiUDdUBPNPyrSz_ad_Qf9yzhDNZlEKiMc";
3749        let pre_b = b"EH7Oq9oxCgYa-nnNLvwhp9sFZpALILlRYyB-6n4WDi7w";
3750        let pre_c = b"EIDA1n-WiBA0A8YOqnKrB-wWQYYC49i5zY_qrIZIicQg";
3751        let pre_d = b"EAYC49i5zY_qrIZIicQgIDA1n-WiBA0A8YOqnKrB-wWQ";
3752
3753        // Define separator
3754        let sep = *b".";
3755        let key_a0 = on_key(pre_a, 0, None);
3756
3757        let key_b0 = on_key(pre_b, 0, None);
3758        let key_b1 = on_key(pre_b, 1, None);
3759        let key_b2 = on_key(pre_b, 2, None);
3760        let key_b3 = on_key(pre_b, 3, None);
3761        let key_b4 = on_key(pre_b, 4, None);
3762
3763        let key_c0 = on_key(pre_c, 0, None);
3764
3765        let dig_a = b"EA73b7reENuBahMJsMTLbeyyNPsfTRzKRWtJ3ytmInvw";
3766
3767        let dig_u = b"EB73b7reENuBahMJsMTLbeyyNPsfTRzKRWtJ3ytmInvw";
3768        let dig_v = b"EC4vCeJswIBJlO3RqE-wsE72Vt3wAceJ_LzqKvbDtBSY";
3769        let dig_w = b"EDAyl33W9ja_wLX85UrzRnL4KNzlsIKIA7CrD04nVX1w";
3770        let dig_x = b"EEnwxEm5Bg5s5aTLsgQCNpubIYzwlvMwZIzdOM0Z3u7o";
3771        let dig_y = b"EFrq74_Q11S2vHx1gpK_46Ik5Q7Yy9K1zZ5BavqGDKnk";
3772
3773        let dig_c = b"EG5RimdY_OWoreR-Z-Q5G81-I4tjASJCaP_MqkBbtM2w";
3774
3775        // Test basic key-value operations
3776        assert_eq!(lmdber.get_val(&db, &key_a0)?, None);
3777        assert_eq!(lmdber.del_val(&db, &key_a0)?, false);
3778        assert_eq!(lmdber.put_val(&db, &key_a0, dig_a)?, true);
3779        assert_eq!(lmdber.get_val(&db, &key_a0)?, Some(dig_a.to_vec()));
3780        assert_eq!(lmdber.put_val(&db, &key_a0, dig_a)?, false);
3781        assert_eq!(lmdber.set_val(&db, &key_a0, dig_a)?, true);
3782        assert_eq!(lmdber.get_val(&db, &key_a0)?, Some(dig_a.to_vec()));
3783        assert_eq!(
3784            lmdber.get_on_val(&db, pre_a, 0, None)?,
3785            Some(dig_a.to_vec())
3786        );
3787        assert_eq!(lmdber.del_val(&db, &key_a0)?, true);
3788        assert_eq!(lmdber.get_val(&db, &key_a0)?, None);
3789        assert_eq!(lmdber.get_on_val(&db, pre_a, 0, None)?, None);
3790
3791        // Test ordinal-numbered key-value operations
3792        assert_eq!(lmdber.put_on_val(&db, pre_a, 0, dig_a, None)?, true);
3793        assert_eq!(
3794            lmdber.get_on_val(&db, pre_a, 0, None)?,
3795            Some(dig_a.to_vec())
3796        );
3797        assert_eq!(lmdber.put_on_val(&db, pre_a, 0, dig_a, None)?, false);
3798        assert_eq!(lmdber.set_on_val(&db, pre_a, Some(0), dig_a, None)?, true);
3799        assert_eq!(
3800            lmdber.get_on_val(&db, pre_a, 0, None)?,
3801            Some(dig_a.to_vec())
3802        );
3803        assert_eq!(lmdber.del_on_val(&db, pre_a, 0, None)?, true);
3804        assert_eq!(lmdber.get_on_val(&db, pre_a, 0, None)?, None);
3805
3806        // Test append_on_val
3807        // Empty database
3808        assert_eq!(lmdber.get_val(&db, &key_b0)?, None);
3809        let on = lmdber.append_on_val(&db, pre_b, dig_u, Some(sep))?;
3810        assert_eq!(on, 0);
3811        assert_eq!(lmdber.get_val(&db, &key_b0)?, Some(dig_u.to_vec()));
3812        assert_eq!(lmdber.del_val(&db, &key_b0)?, true);
3813        assert_eq!(lmdber.get_val(&db, &key_b0)?, None);
3814
3815        // Earlier pre in database only
3816        assert_eq!(lmdber.put_val(&db, &key_a0, dig_a)?, true);
3817        let on = lmdber.append_on_val(&db, pre_b, dig_u, Some(sep))?;
3818        assert_eq!(on, 0);
3819        assert_eq!(lmdber.get_val(&db, &key_b0)?, Some(dig_u.to_vec()));
3820        assert_eq!(lmdber.del_val(&db, &key_b0)?, true);
3821        assert_eq!(lmdber.get_val(&db, &key_b0)?, None);
3822
3823        // Earlier and later pre in db but not same pre
3824        assert_eq!(lmdber.get_val(&db, &key_a0)?, Some(dig_a.to_vec()));
3825        assert_eq!(lmdber.put_val(&db, &key_c0, dig_c)?, true);
3826        let on = lmdber.append_on_val(&db, pre_b, dig_u, Some(sep))?;
3827        assert_eq!(on, 0);
3828        assert_eq!(lmdber.get_val(&db, &key_b0)?, Some(dig_u.to_vec()));
3829        assert_eq!(lmdber.del_val(&db, &key_b0)?, true);
3830        assert_eq!(lmdber.get_val(&db, &key_b0)?, None);
3831
3832        // Later pre only
3833        assert_eq!(lmdber.del_val(&db, &key_a0)?, true);
3834        assert_eq!(lmdber.get_val(&db, &key_a0)?, None);
3835        assert_eq!(lmdber.get_val(&db, &key_c0)?, Some(dig_c.to_vec()));
3836        assert_eq!(lmdber.len(&db)?, 1);
3837        let on = lmdber.append_on_val(&db, pre_b, dig_u, Some(sep))?;
3838        assert_eq!(on, 0);
3839        assert_eq!(lmdber.len(&db)?, 2);
3840        assert_eq!(lmdber.get_val(&db, &key_b0)?, Some(dig_u.to_vec()));
3841        assert_eq!(lmdber.del_val(&db, &key_b0)?, true);
3842        assert_eq!(lmdber.get_val(&db, &key_b0)?, None);
3843        let on = lmdber.append_on_val(&db, pre_b, dig_u, Some(sep))?;
3844        assert_eq!(on, 0);
3845        assert_eq!(lmdber.get_val(&db, &key_b0)?, Some(dig_u.to_vec()));
3846
3847        // Earlier pre and later pre and earlier entry for same pre
3848        assert_eq!(lmdber.put_val(&db, &key_a0, dig_a)?, true);
3849        assert_eq!(lmdber.len(&db)?, 3);
3850        let on = lmdber.append_on_val(&db, pre_b, dig_v, Some(sep))?;
3851        assert_eq!(on, 1);
3852        assert_eq!(lmdber.len(&db)?, 4);
3853        assert_eq!(lmdber.get_val(&db, &key_b1)?, Some(dig_v.to_vec()));
3854        let mut items = Vec::new();
3855        lmdber.get_on_item_iter(&db, Some(pre_b), None, Some(sep), |ckey, cn, cval| {
3856            let key = on_key(ckey, cn, Some(sep));
3857            items.push((key, cn, cval));
3858            Ok(true)
3859        })?;
3860        assert_eq!(items.len(), 2);
3861
3862        // Earlier entry for same pre but only same pre
3863        assert_eq!(lmdber.del_val(&db, &key_a0)?, true);
3864        assert_eq!(lmdber.get_val(&db, &key_a0)?, None);
3865        assert_eq!(lmdber.del_val(&db, &key_c0)?, true);
3866        assert_eq!(lmdber.get_val(&db, &key_c0)?, None);
3867        assert_eq!(lmdber.len(&db)?, 2);
3868
3869        // Add more values for pre_b
3870        assert_eq!(lmdber.cnt_on_vals(&db, Some(pre_b), None, Some(sep))?, 2);
3871        let on = lmdber.append_on_val(&db, pre_b, dig_w, None)?;
3872        assert_eq!(on, 2);
3873        assert_eq!(lmdber.get_val(&db, &key_b2)?, Some(dig_w.to_vec()));
3874
3875        let on = lmdber.append_on_val(&db, pre_b, dig_x, Some(sep))?;
3876        assert_eq!(on, 3);
3877        assert_eq!(lmdber.get_val(&db, &key_b3)?, Some(dig_x.to_vec()));
3878
3879        let on = lmdber.append_on_val(&db, pre_b, dig_y, Some(sep))?;
3880        assert_eq!(on, 4);
3881        assert_eq!(lmdber.get_val(&db, &key_b4)?, Some(dig_y.to_vec()));
3882
3883        assert_eq!(lmdber.append_on_val(&db, pre_d, dig_y, Some(sep))?, 0);
3884
3885        // Test cnt_on_vals
3886        assert_eq!(lmdber.cnt_on_vals(&db, Some(pre_b), None, None)?, 5);
3887        assert_eq!(lmdber.cnt_on_vals(&db, Some(&[]), None, None)?, 6); // all keys
3888        assert_eq!(lmdber.cnt_on_vals(&db, None, None, None)?, 6); // all keys
3889
3890        let mut items = Vec::new();
3891        lmdber.get_on_item_iter(&db, Some(pre_b), None, Some(sep), |ckey, cn, cval| {
3892            items.push((ckey, cn, cval));
3893            Ok(true)
3894        })?;
3895        assert_eq!(
3896            items,
3897            vec![
3898                (pre_b.to_vec(), 0, dig_u.to_vec()),
3899                (pre_b.to_vec(), 1, dig_v.to_vec()),
3900                (pre_b.to_vec(), 2, dig_w.to_vec()),
3901                (pre_b.to_vec(), 3, dig_x.to_vec()),
3902                (pre_b.to_vec(), 4, dig_y.to_vec())
3903            ]
3904        );
3905
3906        // // Resume replay pre_b events at on = 3
3907        let mut items = Vec::new();
3908        lmdber.get_on_item_iter(&db, Some(pre_b), Some(3), Some(sep), |ckey, cn, cval| {
3909            items.push((ckey, cn, cval));
3910            Ok(true)
3911        })?;
3912        assert_eq!(
3913            items,
3914            vec![
3915                (pre_b.to_vec(), 3, dig_x.to_vec()),
3916                (pre_b.to_vec(), 4, dig_y.to_vec())
3917            ]
3918        );
3919
3920        // // Resume replay pre_b events at on = 5
3921        let mut items = Vec::new();
3922        lmdber.get_on_item_iter(&db, Some(pre_b), Some(5), Some(sep), |ckey, cn, cval| {
3923            items.push((ckey, cn, cval));
3924            Ok(true)
3925        })?;
3926        assert_eq!(items, vec![]);
3927
3928        // Replay all events in database with pre events before and after
3929        assert_eq!(lmdber.put_val(&db, &key_a0, dig_a)?, true);
3930        assert_eq!(lmdber.put_val(&db, &key_c0, dig_c)?, true);
3931
3932        let mut items = Vec::new();
3933        lmdber.get_on_item_iter(&db, Some(&[]), None, Some(sep), |ckey, cn, cval| {
3934            items.push((ckey, cn, cval));
3935            Ok(true)
3936        })?;
3937
3938        assert_eq!(
3939            items,
3940            vec![
3941                (pre_a.to_vec(), 0, dig_a.to_vec()),
3942                (pre_d.to_vec(), 0, dig_y.to_vec()),
3943                (pre_b.to_vec(), 0, dig_u.to_vec()),
3944                (pre_b.to_vec(), 1, dig_v.to_vec()),
3945                (pre_b.to_vec(), 2, dig_w.to_vec()),
3946                (pre_b.to_vec(), 3, dig_x.to_vec()),
3947                (pre_b.to_vec(), 4, dig_y.to_vec()),
3948                (pre_c.to_vec(), 0, dig_c.to_vec())
3949            ]
3950        );
3951
3952        let mut items = Vec::new();
3953        lmdber.get_on_item_iter(&db, None, None, Some(sep), |ckey, cn, cval| {
3954            items.push((ckey, cn, cval));
3955            Ok(true)
3956        })?;
3957        assert_eq!(
3958            items,
3959            vec![
3960                (pre_a.to_vec(), 0, dig_a.to_vec()),
3961                (pre_d.to_vec(), 0, dig_y.to_vec()),
3962                (pre_b.to_vec(), 0, dig_u.to_vec()),
3963                (pre_b.to_vec(), 1, dig_v.to_vec()),
3964                (pre_b.to_vec(), 2, dig_w.to_vec()),
3965                (pre_b.to_vec(), 3, dig_x.to_vec()),
3966                (pre_b.to_vec(), 4, dig_y.to_vec()),
3967                (pre_c.to_vec(), 0, dig_c.to_vec())
3968            ]
3969        );
3970
3971        // Resume replay all starting at pre_b on=2
3972        let (top, on) = split_on_key(&key_b2, Some(*b"."))?;
3973        let mut items = Vec::new();
3974        lmdber.get_on_item_iter(&db, Some(&top), Some(on), Some(sep), |ckey, cn, cval| {
3975            items.push((ckey, cn, cval));
3976            Ok(true)
3977        })?;
3978        assert_eq!(
3979            items,
3980            vec![
3981                (top.clone(), 2, dig_w.to_vec()),
3982                (top.clone(), 3, dig_x.to_vec()),
3983                (top.clone(), 4, dig_y.to_vec())
3984            ]
3985        );
3986
3987        // Resume replay all starting at pre_c on=1
3988        let mut items = Vec::new();
3989        lmdber.get_on_item_iter(&db, Some(pre_c), Some(1), Some(sep), |ckey, cn, cval| {
3990            items.push((ckey, cn, cval));
3991            Ok(true)
3992        })?;
3993        assert_eq!(items, vec![]);
3994
3995        let mut items = Vec::new();
3996        lmdber.get_on_val_iter(&db, Some(pre_b), None, Some(sep), |cval| {
3997            items.push(cval);
3998            Ok(true)
3999        })?;
4000        assert_eq!(
4001            items,
4002            vec![
4003                dig_u.to_vec(),
4004                dig_v.to_vec(),
4005                dig_w.to_vec(),
4006                dig_x.to_vec(),
4007                dig_y.to_vec()
4008            ]
4009        );
4010
4011        // // Resume replay pre_b events at on = 3
4012        let mut items = Vec::new();
4013        lmdber.get_on_val_iter(&db, Some(pre_b), Some(3), Some(sep), |cval| {
4014            items.push(cval);
4015            Ok(true)
4016        })?;
4017        assert_eq!(items, vec![dig_x.to_vec(), dig_y.to_vec()]);
4018
4019        Ok(())
4020    }
4021
4022    #[test]
4023    fn test_basic_io_set() -> Result<(), DBError> {
4024        // Create a temporary database
4025        let lmdber = LMDBer::builder().temp(true).build()?;
4026        let db = lmdber.create_database(Some("test_io_set"), Some(false))?;
4027
4028        // Test key and values
4029        let key = b"test_key";
4030        let vals1: [&[u8]; 3] = [b"value1", b"value2", b"value3"];
4031        let vals2: [&[u8]; 3] = [b"value3", b"value4", b"value5"];
4032
4033        // Put the first set of values
4034        lmdber.put_io_set_vals(&db, key, &vals1, None)?;
4035
4036        // Get the values
4037        let result1 = lmdber.get_io_set_vals(&db, key, None, None)?;
4038        assert_eq!(result1.len(), 3);
4039        assert_eq!(result1[0], b"value1".to_vec());
4040        assert_eq!(result1[1], b"value2".to_vec());
4041        assert_eq!(result1[2], b"value3".to_vec());
4042
4043        // Put another set of values, some overlapping
4044        lmdber.put_io_set_vals(&db, key, &vals2, None)?;
4045
4046        // Get the values again
4047        let result2 = lmdber.get_io_set_vals(&db, key, None, None)?;
4048
4049        // Should have 5 values with no duplicates, in insertion order
4050        assert_eq!(result2.len(), 5);
4051        assert_eq!(result2[0], b"value1".to_vec());
4052        assert_eq!(result2[1], b"value2".to_vec());
4053        assert_eq!(result2[2], b"value3".to_vec());
4054        assert_eq!(result2[3], b"value4".to_vec());
4055        assert_eq!(result2[4], b"value5".to_vec());
4056
4057        // Test starting from a specific ordinal
4058        let result3 = lmdber.get_io_set_vals(&db, key, Some(2), None)?;
4059        assert_eq!(result3.len(), 3);
4060        assert_eq!(result3[0], b"value3".to_vec());
4061        assert_eq!(result3[1], b"value4".to_vec());
4062        assert_eq!(result3[2], b"value5".to_vec());
4063
4064        Ok(())
4065    }
4066
4067    #[test]
4068    fn test_add_io_set_val() -> Result<(), DBError> {
4069        // Create a temporary database
4070        let lmdber = LMDBer::builder().temp(true).build()?;
4071        let db = lmdber.create_database(Some("test_io_set"), Some(false))?;
4072
4073        // Test key and values
4074        let key = b"test_key";
4075        let val1 = b"value1";
4076        let val2 = b"value2";
4077
4078        // Add the first value
4079        let result1 = lmdber.add_io_set_val(&db, key, val1, None)?;
4080        assert!(result1, "First add should return true");
4081
4082        // Add the same value again
4083        let result2 = lmdber.add_io_set_val(&db, key, val1, None)?;
4084        assert!(!result2, "Adding same value should return false");
4085
4086        // Add a different value
4087        let result3 = lmdber.add_io_set_val(&db, key, val2, None)?;
4088        assert!(result3, "Adding new value should return true");
4089
4090        // Verify the values were added in the correct order
4091        let values = lmdber.get_io_set_vals(&db, key, None, None)?;
4092        assert_eq!(values.len(), 2);
4093        assert_eq!(values[0], val1.to_vec());
4094        assert_eq!(values[1], val2.to_vec());
4095
4096        Ok(())
4097    }
4098
4099    #[test]
4100    fn test_del_io_set_vals() -> Result<(), DBError> {
4101        // Create a temporary database
4102        let lmdber = LMDBer::builder().temp(true).build()?;
4103        let db = lmdber.create_database(Some("test_io_set"), Some(false))?;
4104
4105        // Test key and values
4106        let key1 = b"test_key1";
4107        let key2 = b"test_key2";
4108        let vals: [&[u8]; 3] = [b"value1", b"value2", b"value3"];
4109
4110        // Add values to key1
4111        lmdber.put_io_set_vals(&db, key1, &vals, None)?;
4112
4113        // Add values to key2
4114        lmdber.put_io_set_vals(&db, key2, &vals, None)?;
4115
4116        // Verify values were added
4117        let values1 = lmdber.get_io_set_vals(&db, key1, None, None)?;
4118        let values2 = lmdber.get_io_set_vals(&db, key2, None, None)?;
4119        assert_eq!(values1.len(), 3);
4120        assert_eq!(values2.len(), 3);
4121
4122        // Delete values for key1
4123        let result1 = lmdber.del_io_set_vals(&db, key1, None)?;
4124        assert!(result1, "Should return true when values are deleted");
4125
4126        // Verify key1 values are gone
4127        let values1_after = lmdber.get_io_set_vals(&db, key1, None, None)?;
4128        assert_eq!(values1_after.len(), 0, "All values should be deleted");
4129
4130        // Verify key2 values still exist
4131        let values2_after = lmdber.get_io_set_vals(&db, key2, None, None)?;
4132        assert_eq!(values2_after.len(), 3, "Key2 values should be untouched");
4133
4134        // Try to delete values for a key that doesn't exist
4135        let result2 = lmdber.del_io_set_vals(&db, b"nonexistent_key", None)?;
4136        assert!(!result2, "Should return false when no values exist");
4137
4138        Ok(())
4139    }
4140
4141    #[test]
4142    fn test_del_io_set_val() -> Result<(), DBError> {
4143        // Create a temporary database
4144        let lmdber = LMDBer::builder().temp(true).build()?;
4145        let db = lmdber.create_database(Some("test_io_set"), Some(false))?;
4146
4147        // Test key and values
4148        let key = b"test_key";
4149        let val1 = b"value1";
4150        let val2 = b"value2";
4151        let val3 = b"value3";
4152
4153        // Add values to key
4154        let vals: [&[u8]; 3] = [val1, val2, val3];
4155        lmdber.put_io_set_vals(&db, key, &vals, None)?;
4156
4157        // Verify values were added
4158        let values = lmdber.get_io_set_vals(&db, key, None, None)?;
4159        assert_eq!(values.len(), 3);
4160
4161        // Delete the second value
4162        let result1 = lmdber.del_io_set_val(&db, key, val2, None)?;
4163        assert!(result1, "Should return true when value is deleted");
4164
4165        // Verify value was deleted
4166        let values_after = lmdber.get_io_set_vals(&db, key, None, None)?;
4167        assert_eq!(values_after.len(), 2);
4168        assert_eq!(values_after[0], val1.to_vec());
4169        assert_eq!(values_after[1], val3.to_vec());
4170
4171        // Try to delete a value that doesn't exist
4172        let result2 = lmdber.del_io_set_val(&db, key, b"nonexistent_value", None)?;
4173        assert!(!result2, "Should return false when value doesn't exist");
4174
4175        // Try to delete from a key that doesn't exist
4176        let result3 = lmdber.del_io_set_val(&db, b"nonexistent_key", val1, None)?;
4177        assert!(!result3, "Should return false when key doesn't exist");
4178
4179        Ok(())
4180    }
4181
4182    #[test]
4183    fn test_set_io_set_vals() -> Result<(), DBError> {
4184        // Create a temporary database
4185        let lmdber = LMDBer::builder().temp(true).build()?;
4186        let db = lmdber.create_database(Some("test_io_set"), Some(false))?;
4187
4188        // Test key and values
4189        let key = b"test_key";
4190        let vals1 = [b"value1", b"value2", b"value3"];
4191
4192        // Set values for key
4193        let result1 = lmdber.set_io_set_vals(&db, key, &vals1, None)?;
4194        assert!(result1, "Should return true when values are set");
4195
4196        // Verify values were added
4197        let values1 = lmdber.get_io_set_vals(&db, key, None, None)?;
4198        assert_eq!(values1.len(), 3);
4199        assert_eq!(values1[0], b"value1".to_vec());
4200        assert_eq!(values1[1], b"value2".to_vec());
4201        assert_eq!(values1[2], b"value3".to_vec());
4202
4203        // Replace with a new set of values, including duplicates
4204        let vals2 = [b"value4", b"value5", b"value4"]; // Note the duplicate
4205        let result2 = lmdber.set_io_set_vals(&db, key, &vals2, None)?;
4206        assert!(result2, "Should return true when values are set");
4207
4208        // Verify original values were removed and new unique values added
4209        let values2 = lmdber.get_io_set_vals(&db, key, None, None)?;
4210        assert_eq!(values2.len(), 2); // Should deduplicate
4211        assert_eq!(values2[0], b"value4".to_vec());
4212        assert_eq!(values2[1], b"value5".to_vec());
4213
4214        // Set with empty array
4215        let empty: [&[u8]; 0] = [];
4216        let result3 = lmdber.set_io_set_vals(&db, key, &empty, None)?;
4217        assert!(result3, "Should return true even with empty array");
4218
4219        // Verify all values were removed
4220        let values3 = lmdber.get_io_set_vals(&db, key, None, None)?;
4221        assert_eq!(values3.len(), 0, "All values should be removed");
4222
4223        Ok(())
4224    }
4225
4226    #[test]
4227    fn test_cnt_io_set_vals() -> Result<(), DBError> {
4228        // Create a temporary database
4229        let lmdber = LMDBer::builder().temp(true).build()?;
4230        let db = lmdber.create_database(Some("test_io_set"), Some(false))?;
4231
4232        // Test key and values
4233        let key1 = b"test_key1";
4234        let key2 = b"test_key2";
4235        let vals1: [&[u8]; 3] = [b"value1", b"value2", b"value3"];
4236        let vals2: [&[u8]; 2] = [b"value4", b"value5"];
4237
4238        // Count values for non-existent key
4239        let count1 = lmdber.cnt_io_set_vals(&db, key1, None)?;
4240        assert_eq!(count1, 0, "Count for non-existent key should be 0");
4241
4242        // Add values to key1
4243        lmdber.put_io_set_vals(&db, key1, &vals1, None)?;
4244
4245        // Count values for key1
4246        let count2 = lmdber.cnt_io_set_vals(&db, key1, None)?;
4247        assert_eq!(count2, 3, "Count should match number of values added");
4248
4249        // Add values to key2
4250        lmdber.put_io_set_vals(&db, key2, &vals2, None)?;
4251
4252        // Count values for key2
4253        let count3 = lmdber.cnt_io_set_vals(&db, key2, None)?;
4254        assert_eq!(count3, 2, "Count should match number of values added");
4255
4256        // Delete a value from key1
4257        lmdber.del_io_set_val(&db, key1, b"value2", None)?;
4258
4259        // Count values for key1 again
4260        let count4 = lmdber.cnt_io_set_vals(&db, key1, None)?;
4261        assert_eq!(count4, 2, "Count should be updated after deletion");
4262
4263        // Delete all values for key2
4264        lmdber.del_io_set_vals(&db, key2, None)?;
4265
4266        // Count values for key2 again
4267        let count5 = lmdber.cnt_io_set_vals(&db, key2, None)?;
4268        assert_eq!(count5, 0, "Count should be 0 after deleting all values");
4269
4270        Ok(())
4271    }
4272
4273    #[test]
4274    fn test_get_io_set_vals_iter() -> Result<(), DBError> {
4275        // Create a temporary database
4276        let lmdber = LMDBer::builder().temp(true).build()?;
4277        let db = lmdber.create_database(Some("test_io_set"), Some(false))?;
4278
4279        // Test key and values
4280        let key = b"test_key";
4281        let vals: [&[u8]; 5] = [b"value1", b"value2", b"value3", b"value4", b"value5"];
4282
4283        // Add values to key
4284        lmdber.put_io_set_vals(&db, key, &vals, None)?;
4285
4286        // Test full iteration
4287        let mut collected_vals = Vec::new();
4288        lmdber.get_io_set_vals_iter(&db, key, None, None, |val| {
4289            collected_vals.push(val.to_vec());
4290            Ok(true) // Continue iteration
4291        })?;
4292
4293        assert_eq!(collected_vals.len(), 5);
4294        assert_eq!(collected_vals[0], b"value1".to_vec());
4295        assert_eq!(collected_vals[4], b"value5".to_vec());
4296
4297        // Test starting from a specific ordinal
4298        let mut collected_vals2 = Vec::new();
4299        lmdber.get_io_set_vals_iter(&db, key, Some(2), None, |val| {
4300            collected_vals2.push(val.to_vec());
4301            Ok(true) // Continue iteration
4302        })?;
4303
4304        assert_eq!(collected_vals2.len(), 3); // Should only get values 3, 4, 5
4305        assert_eq!(collected_vals2[0], b"value3".to_vec());
4306        assert_eq!(collected_vals2[2], b"value5".to_vec());
4307
4308        // Test early termination
4309        let mut collected_vals3 = Vec::new();
4310        lmdber.get_io_set_vals_iter(&db, key, None, None, |val| {
4311            collected_vals3.push(val.to_vec());
4312            // Stop after collecting 2 values
4313            Ok(collected_vals3.len() < 2)
4314        })?;
4315
4316        assert_eq!(collected_vals3.len(), 2); // Should only get values 1, 2
4317        assert_eq!(collected_vals3[0], b"value1".to_vec());
4318        assert_eq!(collected_vals3[1], b"value2".to_vec());
4319
4320        // Test non-existent key
4321        let mut non_existent_vals = Vec::new();
4322        lmdber.get_io_set_vals_iter(&db, b"non_existent_key", None, None, |val| {
4323            non_existent_vals.push(val.to_vec());
4324            Ok(true)
4325        })?;
4326
4327        assert_eq!(non_existent_vals.len(), 0); // Should not collect any values
4328
4329        Ok(())
4330    }
4331
4332    #[test]
4333    fn test_get_io_set_val_last() -> Result<(), DBError> {
4334        // Create a temporary database
4335        let lmdber = LMDBer::builder().temp(true).build()?;
4336        let db = lmdber.create_database(Some("test_io_set"), Some(false))?;
4337
4338        // Test key and values
4339        let key1 = b"test_key1";
4340        let key2 = b"test_key2";
4341
4342        // Test with empty database
4343        let result1 = lmdber.get_io_set_val_last(&db, key1, None)?;
4344        assert_eq!(result1, None, "Should return None for non-existent key");
4345
4346        // Add a single value to key1
4347        let val1 = b"value1";
4348        lmdber.add_io_set_val(&db, key1, val1, None)?;
4349
4350        // Check last value for key1
4351        let result2 = lmdber.get_io_set_val_last(&db, key1, None)?;
4352        assert_eq!(result2, Some(val1.to_vec()), "Should return the only value");
4353
4354        // Add more values to key1
4355        let val2 = b"value2";
4356        let val3 = b"value3";
4357        lmdber.add_io_set_val(&db, key1, val2, None)?;
4358        lmdber.add_io_set_val(&db, key1, val3, None)?;
4359
4360        // Check last value for key1
4361        let result3 = lmdber.get_io_set_val_last(&db, key1, None)?;
4362        assert_eq!(
4363            result3,
4364            Some(val3.to_vec()),
4365            "Should return the last added value"
4366        );
4367
4368        // Add values to key2, which lexicographically follows key1
4369        let val4 = b"value4";
4370        lmdber.add_io_set_val(&db, key2, val4, None)?;
4371
4372        // Check last value for key1 again (should be unaffected by key2)
4373        let result4 = lmdber.get_io_set_val_last(&db, key1, None)?;
4374        assert_eq!(
4375            result4,
4376            Some(val3.to_vec()),
4377            "Should still return the last value for key1"
4378        );
4379
4380        // Check last value for key2
4381        let result5 = lmdber.get_io_set_val_last(&db, key2, None)?;
4382        assert_eq!(
4383            result5,
4384            Some(val4.to_vec()),
4385            "Should return the only value for key2"
4386        );
4387
4388        // Add values to a key that lexicographically precedes key1
4389        let key0 = b"test_key0";
4390        let val5 = b"value5";
4391        lmdber.add_io_set_val(&db, key0, val5, None)?;
4392
4393        // Check last values for all keys
4394        let result6 = lmdber.get_io_set_val_last(&db, key0, None)?;
4395        assert_eq!(
4396            result6,
4397            Some(val5.to_vec()),
4398            "Should return the only value for key0"
4399        );
4400
4401        let result7 = lmdber.get_io_set_val_last(&db, key1, None)?;
4402        assert_eq!(
4403            result7,
4404            Some(val3.to_vec()),
4405            "Should still return the last value for key1"
4406        );
4407
4408        // Delete all values for key1
4409        lmdber.del_io_set_vals(&db, key1, None)?;
4410
4411        // Check last value for key1 after deletion
4412        let result8 = lmdber.get_io_set_val_last(&db, key1, None)?;
4413        assert_eq!(
4414            result8, None,
4415            "Should return None after all values are deleted"
4416        );
4417
4418        Ok(())
4419    }
4420    #[test]
4421    fn test_on_io_dup_methods() -> Result<(), DBError> {
4422        // Create LMDBer instance for testing
4423        let mut dber = LMDBer::builder()
4424            .name("test_on_io_dup")
4425            .temp(true)
4426            .build()?;
4427
4428        // Create database with dupsort enabled
4429        let ldb = dber.create_database(Some("log."), Some(true))?;
4430
4431        // Define test prefixes
4432        let pre_a = b"A";
4433        let pre_b = b"B";
4434
4435        // First prefix (preA)
4436        let mut sn = 0;
4437        let key = keys::sn_key(pre_a, sn);
4438        let vals_a0 = vec![b"echo".to_vec(), b"bravo".to_vec()];
4439        let items_a0 = vec![
4440            (pre_a.to_vec(), sn, vals_a0[0].clone()),
4441            (pre_a.to_vec(), sn, vals_a0[1].clone()),
4442        ];
4443
4444        assert_eq!(dber.add_io_dup_val(&ldb, &key, &vals_a0[0])?, true);
4445        assert_eq!(dber.add_io_dup_val(&ldb, &key, &vals_a0[1])?, true);
4446
4447        sn += 1;
4448        let key = keys::sn_key(pre_a, sn);
4449        let vals_a1 = vec![
4450            b"sue".to_vec(),
4451            b"bob".to_vec(),
4452            b"val".to_vec(),
4453            b"zoe".to_vec(),
4454        ];
4455        let items_a1 = vec![
4456            (pre_a.to_vec(), sn, vals_a1[0].clone()),
4457            (pre_a.to_vec(), sn, vals_a1[1].clone()),
4458            (pre_a.to_vec(), sn, vals_a1[2].clone()),
4459            (pre_a.to_vec(), sn, vals_a1[3].clone()),
4460        ];
4461
4462        assert_eq!(dber.put_io_dup_vals(&ldb, &key, &vals_a1)?, true);
4463
4464        sn += 1;
4465        let key = keys::sn_key(pre_a, sn);
4466        let vals_a2 = vec![b"fish".to_vec(), b"bat".to_vec(), b"snail".to_vec()];
4467        let items_a2 = vec![
4468            (pre_a.to_vec(), sn, vals_a2[0].clone()),
4469            (pre_a.to_vec(), sn, vals_a2[1].clone()),
4470            (pre_a.to_vec(), sn, vals_a2[2].clone()),
4471        ];
4472
4473        assert_eq!(dber.put_io_dup_vals(&ldb, &key, &vals_a2)?, true);
4474
4475        // Second prefix (preB)
4476        sn = 0;
4477        let key = keys::sn_key(pre_b, sn);
4478        let vals_b0 = vec![b"gamma".to_vec(), b"beta".to_vec()];
4479        let items_b0 = vec![
4480            (pre_b.to_vec(), sn, vals_b0[0].clone()),
4481            (pre_b.to_vec(), sn, vals_b0[1].clone()),
4482        ];
4483
4484        assert_eq!(dber.add_io_dup_val(&ldb, &key, &vals_b0[0])?, true);
4485        assert_eq!(dber.add_io_dup_val(&ldb, &key, &vals_b0[1])?, true);
4486
4487        sn += 1;
4488        let key = keys::sn_key(pre_b, sn);
4489        let vals_b1 = vec![
4490            b"mary".to_vec(),
4491            b"peter".to_vec(),
4492            b"john".to_vec(),
4493            b"paul".to_vec(),
4494        ];
4495        let items_b1 = vec![
4496            (pre_b.to_vec(), sn, vals_b1[0].clone()),
4497            (pre_b.to_vec(), sn, vals_b1[1].clone()),
4498            (pre_b.to_vec(), sn, vals_b1[2].clone()),
4499            (pre_b.to_vec(), sn, vals_b1[3].clone()),
4500        ];
4501
4502        assert_eq!(dber.put_io_dup_vals(&ldb, &key, &vals_b1)?, true);
4503
4504        sn += 1;
4505        let key = keys::sn_key(pre_b, sn);
4506        let vals_b2 = vec![b"dog".to_vec(), b"cat".to_vec(), b"bird".to_vec()];
4507        let items_b2 = vec![
4508            (pre_b.to_vec(), sn, vals_b2[0].clone()),
4509            (pre_b.to_vec(), sn, vals_b2[1].clone()),
4510            (pre_b.to_vec(), sn, vals_b2[2].clone()),
4511        ];
4512
4513        assert_eq!(dber.put_io_dup_vals(&ldb, &key, &vals_b2)?, true);
4514
4515        // Test get_on_io_dup_last_item_iter with preA
4516        let mut items = Vec::new();
4517        dber.get_on_io_dup_last_item_iter(&ldb, Some(pre_a), None, None, |key, on, val| {
4518            items.push((key, on, val));
4519            Ok(true)
4520        })?;
4521
4522        let last_items = vec![
4523            (pre_a.to_vec(), 0, vals_a0.last().unwrap().clone()),
4524            (pre_a.to_vec(), 1, vals_a1.last().unwrap().clone()),
4525            (pre_a.to_vec(), 2, vals_a2.last().unwrap().clone()),
4526        ];
4527
4528        assert_eq!(items, last_items);
4529
4530        // Test get_on_io_dup_last_item_iter with preA and on=1
4531        let mut items = Vec::new();
4532        dber.get_on_io_dup_last_item_iter(&ldb, Some(pre_a), Some(1), None, |key, on, val| {
4533            items.push((key, on, val));
4534            Ok(true)
4535        })?;
4536
4537        let last_items = vec![
4538            (pre_a.to_vec(), 1, vals_a1.last().unwrap().clone()),
4539            (pre_a.to_vec(), 2, vals_a2.last().unwrap().clone()),
4540        ];
4541
4542        assert_eq!(items, last_items);
4543
4544        // Test get_on_io_dup_last_val_iter with preA
4545        let mut vals = Vec::new();
4546        dber.get_on_io_dup_last_val_iter(&ldb, Some(pre_a), None, None, |val| {
4547            vals.push(val);
4548            Ok(true)
4549        })?;
4550
4551        let last_vals = vec![
4552            vals_a0.last().unwrap().clone(),
4553            vals_a1.last().unwrap().clone(),
4554            vals_a2.last().unwrap().clone(),
4555        ];
4556
4557        assert_eq!(vals, last_vals);
4558
4559        // Test get_on_io_dup_last_val_iter with preA and on=1
4560        let mut vals = Vec::new();
4561        dber.get_on_io_dup_last_val_iter(&ldb, Some(pre_a), Some(1), None, |val| {
4562            vals.push(val);
4563            Ok(true)
4564        })?;
4565
4566        let last_vals = vec![
4567            vals_a1.last().unwrap().clone(),
4568            vals_a2.last().unwrap().clone(),
4569        ];
4570
4571        assert_eq!(vals, last_vals);
4572
4573        // Test get_on_io_dup_last_item_iter with preB
4574        let mut items = Vec::new();
4575        dber.get_on_io_dup_last_item_iter(&ldb, Some(pre_b), None, None, |key, on, val| {
4576            items.push((key, on, val));
4577            Ok(true)
4578        })?;
4579
4580        let last_items = vec![
4581            (pre_b.to_vec(), 0, vals_b0.last().unwrap().clone()),
4582            (pre_b.to_vec(), 1, vals_b1.last().unwrap().clone()),
4583            (pre_b.to_vec(), 2, vals_b2.last().unwrap().clone()),
4584        ];
4585
4586        assert_eq!(items, last_items);
4587
4588        // Test get_on_io_dup_last_val_iter with preB
4589        let mut vals = Vec::new();
4590        dber.get_on_io_dup_last_val_iter(&ldb, Some(pre_b), None, None, |val| {
4591            vals.push(val);
4592            Ok(true)
4593        })?;
4594
4595        let last_vals = vec![
4596            vals_b0.last().unwrap().clone(),
4597            vals_b1.last().unwrap().clone(),
4598            vals_b2.last().unwrap().clone(),
4599        ];
4600
4601        assert_eq!(vals, last_vals);
4602
4603        // Test get_on_io_dup_last_item_iter with all items
4604        let mut items = Vec::new();
4605        dber.get_on_io_dup_last_item_iter(&ldb, None, None, None, |key, on, val| {
4606            items.push((key, on, val));
4607            Ok(true)
4608        })?;
4609
4610        let last_items = vec![
4611            (pre_a.to_vec(), 0, vals_a0.last().unwrap().clone()),
4612            (pre_a.to_vec(), 1, vals_a1.last().unwrap().clone()),
4613            (pre_a.to_vec(), 2, vals_a2.last().unwrap().clone()),
4614            (pre_b.to_vec(), 0, vals_b0.last().unwrap().clone()),
4615            (pre_b.to_vec(), 1, vals_b1.last().unwrap().clone()),
4616            (pre_b.to_vec(), 2, vals_b2.last().unwrap().clone()),
4617        ];
4618
4619        assert_eq!(items, last_items);
4620
4621        // Test get_on_io_dup_last_val_iter with all items
4622        let mut vals = Vec::new();
4623        dber.get_on_io_dup_last_val_iter(&ldb, None, None, None, |val| {
4624            vals.push(val);
4625            Ok(true)
4626        })?;
4627
4628        let last_vals = vec![
4629            vals_a0.last().unwrap().clone(),
4630            vals_a1.last().unwrap().clone(),
4631            vals_a2.last().unwrap().clone(),
4632            vals_b0.last().unwrap().clone(),
4633            vals_b1.last().unwrap().clone(),
4634            vals_b2.last().unwrap().clone(),
4635        ];
4636
4637        assert_eq!(vals, last_vals);
4638
4639        // Test back iter with preB and on=3
4640        let mut items = Vec::new();
4641        dber.get_on_io_dup_item_back_iter(&ldb, Some(pre_b), Some(3), None, |key, on, val| {
4642            items.push((key, on, val));
4643            Ok(true)
4644        })?;
4645
4646        let expected_items = vec![
4647            (pre_b.to_vec(), 2, b"bird".to_vec()),
4648            (pre_b.to_vec(), 2, b"cat".to_vec()),
4649            (pre_b.to_vec(), 2, b"dog".to_vec()),
4650            (pre_b.to_vec(), 1, b"paul".to_vec()),
4651            (pre_b.to_vec(), 1, b"john".to_vec()),
4652            (pre_b.to_vec(), 1, b"peter".to_vec()),
4653            (pre_b.to_vec(), 1, b"mary".to_vec()),
4654            (pre_b.to_vec(), 0, b"beta".to_vec()),
4655            (pre_b.to_vec(), 0, b"gamma".to_vec()),
4656        ];
4657
4658        assert_eq!(items, expected_items);
4659
4660        // Test val back iter with preB and on=3
4661        let mut vals = Vec::new();
4662        dber.get_on_io_dup_val_back_iter(&ldb, Some(pre_b), Some(3), None, |val| {
4663            vals.push(val);
4664            Ok(true)
4665        })?;
4666
4667        let expected_vals = vec![
4668            b"bird".to_vec(),
4669            b"cat".to_vec(),
4670            b"dog".to_vec(),
4671            b"paul".to_vec(),
4672            b"john".to_vec(),
4673            b"peter".to_vec(),
4674            b"mary".to_vec(),
4675            b"beta".to_vec(),
4676            b"gamma".to_vec(),
4677        ];
4678
4679        assert_eq!(vals, expected_vals);
4680
4681        // Test back iter with preB and on=1
4682        let mut items = Vec::new();
4683        dber.get_on_io_dup_item_back_iter(&ldb, Some(pre_b), Some(1), None, |key, on, val| {
4684            items.push((key, on, val));
4685            Ok(true)
4686        })?;
4687
4688        let expected_items = vec![
4689            (pre_b.to_vec(), 1, b"paul".to_vec()),
4690            (pre_b.to_vec(), 1, b"john".to_vec()),
4691            (pre_b.to_vec(), 1, b"peter".to_vec()),
4692            (pre_b.to_vec(), 1, b"mary".to_vec()),
4693            (pre_b.to_vec(), 0, b"beta".to_vec()),
4694            (pre_b.to_vec(), 0, b"gamma".to_vec()),
4695        ];
4696
4697        assert_eq!(items, expected_items);
4698
4699        // Test val back iter with preB and on=1
4700        let mut vals = Vec::new();
4701        dber.get_on_io_dup_val_back_iter(&ldb, Some(pre_b), Some(1), None, |val| {
4702            vals.push(val);
4703            Ok(true)
4704        })?;
4705
4706        let expected_vals = vec![
4707            b"paul".to_vec(),
4708            b"john".to_vec(),
4709            b"peter".to_vec(),
4710            b"mary".to_vec(),
4711            b"beta".to_vec(),
4712            b"gamma".to_vec(),
4713        ];
4714
4715        assert_eq!(vals, expected_vals);
4716
4717        // Test back iter with preA and on=5
4718        let mut items = Vec::new();
4719        dber.get_on_io_dup_item_back_iter(&ldb, Some(pre_a), Some(5), None, |key, on, val| {
4720            items.push((key, on, val));
4721            Ok(true)
4722        })?;
4723
4724        let expected_items = vec![
4725            (pre_a.to_vec(), 2, b"snail".to_vec()),
4726            (pre_a.to_vec(), 2, b"bat".to_vec()),
4727            (pre_a.to_vec(), 2, b"fish".to_vec()),
4728            (pre_a.to_vec(), 1, b"zoe".to_vec()),
4729            (pre_a.to_vec(), 1, b"val".to_vec()),
4730            (pre_a.to_vec(), 1, b"bob".to_vec()),
4731            (pre_a.to_vec(), 1, b"sue".to_vec()),
4732            (pre_a.to_vec(), 0, b"bravo".to_vec()),
4733            (pre_a.to_vec(), 0, b"echo".to_vec()),
4734        ];
4735
4736        assert_eq!(items, expected_items);
4737
4738        // Test val back iter with preA and on=5
4739        let mut vals = Vec::new();
4740        dber.get_on_io_dup_val_back_iter(&ldb, Some(pre_a), Some(5), None, |val| {
4741            vals.push(val);
4742            Ok(true)
4743        })?;
4744
4745        let expected_vals = vec![
4746            b"snail".to_vec(),
4747            b"bat".to_vec(),
4748            b"fish".to_vec(),
4749            b"zoe".to_vec(),
4750            b"val".to_vec(),
4751            b"bob".to_vec(),
4752            b"sue".to_vec(),
4753            b"bravo".to_vec(),
4754            b"echo".to_vec(),
4755        ];
4756
4757        assert_eq!(vals, expected_vals);
4758
4759        // Test back iter with preA and on=0
4760        let mut items = Vec::new();
4761        dber.get_on_io_dup_item_back_iter(&ldb, Some(pre_a), Some(0), None, |key, on, val| {
4762            items.push((key, on, val));
4763            Ok(true)
4764        })?;
4765
4766        let expected_items = vec![
4767            (pre_a.to_vec(), 0, b"bravo".to_vec()),
4768            (pre_a.to_vec(), 0, b"echo".to_vec()),
4769        ];
4770
4771        assert_eq!(items, expected_items);
4772
4773        // Test val back iter with preA and on=0
4774        let mut vals = Vec::new();
4775        dber.get_on_io_dup_val_back_iter(&ldb, Some(pre_a), Some(0), None, |val| {
4776            vals.push(val);
4777            Ok(true)
4778        })?;
4779
4780        let expected_vals = vec![b"bravo".to_vec(), b"echo".to_vec()];
4781
4782        assert_eq!(vals, expected_vals);
4783
4784        // All items from last to first
4785        let mut items = Vec::new();
4786        dber.get_on_io_dup_item_back_iter(&ldb, None, None, None, |key, on, val| {
4787            items.push((key, on, val));
4788            Ok(true)
4789        })?;
4790
4791        let expected_items = vec![
4792            (pre_b.to_vec(), 2, b"bird".to_vec()),
4793            (pre_b.to_vec(), 2, b"cat".to_vec()),
4794            (pre_b.to_vec(), 2, b"dog".to_vec()),
4795            (pre_b.to_vec(), 1, b"paul".to_vec()),
4796            (pre_b.to_vec(), 1, b"john".to_vec()),
4797            (pre_b.to_vec(), 1, b"peter".to_vec()),
4798            (pre_b.to_vec(), 1, b"mary".to_vec()),
4799            (pre_b.to_vec(), 0, b"beta".to_vec()),
4800            (pre_b.to_vec(), 0, b"gamma".to_vec()),
4801            (pre_a.to_vec(), 2, b"snail".to_vec()),
4802            (pre_a.to_vec(), 2, b"bat".to_vec()),
4803            (pre_a.to_vec(), 2, b"fish".to_vec()),
4804            (pre_a.to_vec(), 1, b"zoe".to_vec()),
4805            (pre_a.to_vec(), 1, b"val".to_vec()),
4806            (pre_a.to_vec(), 1, b"bob".to_vec()),
4807            (pre_a.to_vec(), 1, b"sue".to_vec()),
4808            (pre_a.to_vec(), 0, b"bravo".to_vec()),
4809            (pre_a.to_vec(), 0, b"echo".to_vec()),
4810        ];
4811
4812        assert_eq!(items, expected_items);
4813
4814        // All values from last to first
4815        let mut vals = Vec::new();
4816        dber.get_on_io_dup_val_back_iter(&ldb, None, None, None, |val| {
4817            vals.push(val);
4818            Ok(true)
4819        })?;
4820
4821        let expected_vals = vec![
4822            b"bird".to_vec(),
4823            b"cat".to_vec(),
4824            b"dog".to_vec(),
4825            b"paul".to_vec(),
4826            b"john".to_vec(),
4827            b"peter".to_vec(),
4828            b"mary".to_vec(),
4829            b"beta".to_vec(),
4830            b"gamma".to_vec(),
4831            b"snail".to_vec(),
4832            b"bat".to_vec(),
4833            b"fish".to_vec(),
4834            b"zoe".to_vec(),
4835            b"val".to_vec(),
4836            b"bob".to_vec(),
4837            b"sue".to_vec(),
4838            b"bravo".to_vec(),
4839            b"echo".to_vec(),
4840        ];
4841
4842        assert_eq!(vals, expected_vals);
4843
4844        // Test additional OnIoDup methods
4845        let key = b"Z";
4846        assert_eq!(dber.append_on_io_dup_val(&ldb, key, b"k", None)?, 0);
4847        assert_eq!(dber.append_on_io_dup_val(&ldb, key, b"l", None)?, 1);
4848        assert_eq!(dber.append_on_io_dup_val(&ldb, key, b"m", None)?, 2);
4849        assert_eq!(dber.append_on_io_dup_val(&ldb, key, b"n", None)?, 3);
4850
4851        assert_eq!(dber.cnt_on_vals(&ldb, Some(key), None, None)?, 4);
4852
4853        // Test get_on_io_dup_val_iter
4854        let mut vals = Vec::new();
4855        dber.get_on_io_dup_val_iter(&ldb, Some(key), None, None, |val| {
4856            vals.push(val);
4857            Ok(true)
4858        })?;
4859
4860        let expected_vals = vec![b"k".to_vec(), b"l".to_vec(), b"m".to_vec(), b"n".to_vec()];
4861
4862        assert_eq!(vals, expected_vals);
4863
4864        // Test get_on_io_dup_val_iter with on=2
4865        let mut vals = Vec::new();
4866        dber.get_on_io_dup_val_iter(&ldb, Some(key), Some(2), None, |val| {
4867            vals.push(val);
4868            Ok(true)
4869        })?;
4870
4871        let expected_vals = vec![b"m".to_vec(), b"n".to_vec()];
4872
4873        assert_eq!(vals, expected_vals);
4874
4875        // Test get_on_io_dup_item_iter
4876        let mut items = Vec::new();
4877        dber.get_on_io_dup_item_iter(&ldb, Some(key), None, None, |key, on, val| {
4878            items.push((key, on, val));
4879            Ok(true)
4880        })?;
4881
4882        let expected_items = vec![
4883            (b"Z".to_vec(), 0, b"k".to_vec()),
4884            (b"Z".to_vec(), 1, b"l".to_vec()),
4885            (b"Z".to_vec(), 2, b"m".to_vec()),
4886            (b"Z".to_vec(), 3, b"n".to_vec()),
4887        ];
4888
4889        assert_eq!(items, expected_items);
4890
4891        // Test get_on_io_dup_item_iter with on=2
4892        let mut items = Vec::new();
4893        dber.get_on_io_dup_item_iter(&ldb, Some(key), Some(2), None, |key, on, val| {
4894            items.push((key, on, val));
4895            Ok(true)
4896        })?;
4897
4898        let expected_items = vec![
4899            (b"Z".to_vec(), 2, b"m".to_vec()),
4900            (b"Z".to_vec(), 3, b"n".to_vec()),
4901        ];
4902
4903        assert_eq!(items, expected_items);
4904
4905        // Test back iter with key=Z and on=3
4906        let mut items = Vec::new();
4907        dber.get_on_io_dup_item_back_iter(&ldb, Some(key), Some(3), None, |key, on, val| {
4908            items.push((key, on, val));
4909            Ok(true)
4910        })?;
4911
4912        let expected_items = vec![
4913            (b"Z".to_vec(), 3, b"n".to_vec()),
4914            (b"Z".to_vec(), 2, b"m".to_vec()),
4915            (b"Z".to_vec(), 1, b"l".to_vec()),
4916            (b"Z".to_vec(), 0, b"k".to_vec()),
4917        ];
4918
4919        assert_eq!(items, expected_items);
4920
4921        // Test back iter with key=Z and on=4
4922        let mut items = Vec::new();
4923        dber.get_on_io_dup_item_back_iter(&ldb, Some(key), Some(4), None, |key, on, val| {
4924            items.push((key, on, val));
4925            Ok(true)
4926        })?;
4927
4928        let expected_items = vec![
4929            (b"Z".to_vec(), 3, b"n".to_vec()),
4930            (b"Z".to_vec(), 2, b"m".to_vec()),
4931            (b"Z".to_vec(), 1, b"l".to_vec()),
4932            (b"Z".to_vec(), 0, b"k".to_vec()),
4933        ];
4934
4935        assert_eq!(items, expected_items);
4936
4937        // Test back iter with key=Z and on=2
4938        let mut items = Vec::new();
4939        dber.get_on_io_dup_item_back_iter(&ldb, Some(key), Some(2), None, |key, on, val| {
4940            items.push((key, on, val));
4941            Ok(true)
4942        })?;
4943
4944        let expected_items = vec![
4945            (b"Z".to_vec(), 2, b"m".to_vec()),
4946            (b"Z".to_vec(), 1, b"l".to_vec()),
4947            (b"Z".to_vec(), 0, b"k".to_vec()),
4948        ];
4949
4950        assert_eq!(items, expected_items);
4951
4952        // Test with key=Y
4953        let key = b"Y";
4954        assert_eq!(
4955            dber.add_on_io_dup_val(&ldb, key, Some(0), b"r", None)?,
4956            true
4957        );
4958        assert_eq!(
4959            dber.add_on_io_dup_val(&ldb, key, Some(0), b"s", None)?,
4960            true
4961        );
4962        assert_eq!(
4963            dber.add_on_io_dup_val(&ldb, key, Some(1), b"t", None)?,
4964            true
4965        );
4966        assert_eq!(
4967            dber.add_on_io_dup_val(&ldb, key, Some(1), b"u", None)?,
4968            true
4969        );
4970
4971        assert_eq!(dber.cnt_on_vals(&ldb, Some(key), None, None)?, 4);
4972
4973        let mut items = Vec::new();
4974        dber.get_on_io_dup_item_iter(&ldb, Some(key), None, None, |key, on, val| {
4975            items.push((key, on, val));
4976            Ok(true)
4977        })?;
4978
4979        let expected_items = vec![
4980            (b"Y".to_vec(), 0, b"r".to_vec()),
4981            (b"Y".to_vec(), 0, b"s".to_vec()),
4982            (b"Y".to_vec(), 1, b"t".to_vec()),
4983            (b"Y".to_vec(), 1, b"u".to_vec()),
4984        ];
4985
4986        assert_eq!(items, expected_items);
4987
4988        // Test del_on_io_dup_val and del_on_io_dup_vals
4989        assert_eq!(
4990            dber.del_on_io_dup_val(&ldb, key, Some(0), b"s", None)?,
4991            true
4992        );
4993        assert_eq!(dber.del_on_io_dup_vals(&ldb, key, Some(1), None)?, true);
4994
4995        let mut items = Vec::new();
4996        dber.get_on_io_dup_item_iter(&ldb, Some(key), None, None, |key, on, val| {
4997            items.push((key, on, val));
4998            Ok(true)
4999        })?;
5000
5001        let expected_items = vec![(b"Y".to_vec(), 0, b"r".to_vec())];
5002
5003        assert_eq!(items, expected_items);
5004
5005        // Close the database
5006        dber.close(true)?;
5007
5008        Ok(())
5009    }
5010
5011    #[test]
5012    fn test_io_set_methods() -> Result<(), DBError> {
5013        Ok(())
5014    }
5015}