Skip to main content

witness/
actor.rs

1//! Single-owner KERI state for the witness.
2//!
3//! All libkeri state lives in one `libkeri::Habery` owned by a dedicated OS
4//! thread. HTTP handlers never touch it directly: they send a [`Command`] over a
5//! channel and await the reply. Because exactly one thread ever holds the
6//! state, there are no locks, so there is nothing to deadlock on.
7//!
8//! The Habery is opened *on* the owner thread, and a plain thread (not a tokio
9//! task) is used because libkeri is synchronous and does blocking work (LMDB,
10//! signing, Argon2 stretching) that must not stall the async runtime.
11
12use std::panic::{self, AssertUnwindSafe};
13use std::path::PathBuf;
14use std::thread;
15
16use anyhow::{anyhow, Context, Result};
17use tokio::sync::{mpsc, oneshot};
18use tracing::{debug, error, info, warn};
19
20use libkeri::keri::Roles;
21use libkeri::{Habery, HaberyOptions, MakeHabArgs, Matter, Rawifiable, Salter, Serder, SerderKERI};
22
23use crate::config::WitnessNetworkConfig;
24
25/// Max queued commands before handlers wait (backpressure).
26const QUEUE_DEPTH: usize = 64;
27
28/// Error returned by the actor, mapped to an HTTP status by the server.
29#[derive(Debug)]
30pub enum ActorError {
31    /// The request was malformed or not acceptable.
32    BadRequest(String),
33    /// The requested resource does not exist here.
34    NotFound(String),
35    /// The witness failed internally while handling a valid request.
36    Internal(String),
37    /// The actor thread is gone (exited or crashed); the request was not handled.
38    Unavailable,
39}
40
41impl std::fmt::Display for ActorError {
42    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
43        match self {
44            ActorError::BadRequest(m) => write!(f, "bad request: {}", m),
45            ActorError::NotFound(m) => write!(f, "not found: {}", m),
46            ActorError::Internal(m) => write!(f, "internal error: {}", m),
47            ActorError::Unavailable => write!(f, "witness actor unavailable"),
48        }
49    }
50}
51
52/// Outcome of submitting an event to `/receipts` (keripy `ReceiptEnd.on_post`).
53pub enum ReceiptOutcome {
54    /// Event accepted and this witness is in its witness list: signed `rct`.
55    Receipted(Vec<u8>),
56    /// Event not (yet) accepted into this witness's KEL (HTTP 202).
57    Pending,
58}
59
60type Reply<T> = oneshot::Sender<Result<T, ActorError>>;
61
62/// Work the actor can perform. Each variant carries its own reply channel.
63enum Command {
64    Receipt { body: Vec<u8>, attachments: String, reply: Reply<ReceiptOutcome> },
65    Ingest { msg: Vec<u8>, reply: Reply<()> },
66    Oobi { aid: String, role: Option<String>, reply: Reply<Vec<u8>> },
67}
68
69/// Cheap, cloneable handle used by HTTP handlers to talk to the actor.
70#[derive(Clone)]
71pub struct ActorHandle {
72    tx: mpsc::Sender<Command>,
73}
74
75impl ActorHandle {
76    /// Submit an event for receipting (body JSON + CESR attachments).
77    pub async fn receipt(&self, body: Vec<u8>, attachments: String) -> Result<ReceiptOutcome, ActorError> {
78        self.request(|reply| Command::Receipt { body, attachments, reply }).await
79    }
80
81    /// Ingest a CESR stream from a peer (receipts, events) as a non-local source.
82    pub async fn ingest(&self, msg: Vec<u8>) -> Result<(), ActorError> {
83        self.request(|reply| Command::Ingest { msg, reply }).await
84    }
85
86    /// The OOBI response stream for `aid` (only this witness's own AID).
87    pub async fn oobi(&self, aid: String, role: Option<String>) -> Result<Vec<u8>, ActorError> {
88        self.request(|reply| Command::Oobi { aid, role, reply }).await
89    }
90
91    async fn request<T>(&self, make: impl FnOnce(Reply<T>) -> Command) -> Result<T, ActorError> {
92        let (reply, rx) = oneshot::channel();
93        self.tx.send(make(reply)).await.map_err(|_| ActorError::Unavailable)?;
94        rx.await.map_err(|_| ActorError::Unavailable)?
95    }
96}
97
98/// Spawn the owner thread, open the witness Habery on it, and return a handle
99/// plus the witness AID once initialization has finished.
100pub async fn spawn(
101    alias: String,
102    base_dir: PathBuf,
103    config: WitnessNetworkConfig,
104) -> Result<(ActorHandle, String)> {
105    let (tx, mut rx) = mpsc::channel::<Command>(QUEUE_DEPTH);
106    let (ready_tx, ready_rx) = oneshot::channel::<Result<String>>();
107
108    thread::Builder::new()
109        .name(format!("witness-{}", alias))
110        .spawn(move || {
111            let mut actor = match WitnessActor::open(&alias, base_dir, &config) {
112                Ok(actor) => {
113                    let _ = ready_tx.send(Ok(actor.pre.clone()));
114                    actor
115                }
116                Err(e) => {
117                    let _ = ready_tx.send(Err(e));
118                    return;
119                }
120            };
121
122            // Runs until every ActorHandle has been dropped.
123            while let Some(cmd) = rx.blocking_recv() {
124                actor.handle(cmd);
125            }
126            info!("Witness actor for {} shutting down", alias);
127        })
128        .context("Failed to spawn witness actor thread")?;
129
130    let pre = ready_rx
131        .await
132        .map_err(|_| anyhow!("Witness actor thread exited during initialization"))??;
133
134    Ok((ActorHandle { tx }, pre))
135}
136
137/// State owned by the actor thread. Only ever accessed from that one thread.
138struct WitnessActor {
139    alias: String,
140    pre: String,
141    hby: Habery,
142    /// Signed /loc/scheme and /end/role replies served with the OOBI
143    /// (keripy builds these from the config file on startup).
144    oobi_replies: Vec<u8>,
145}
146
147impl WitnessActor {
148    /// Open the witness Habery the way keripy's `kli witness demo` does:
149    /// `Habery(name=alias, salt=Salter(raw=demo_salt).qb64, temp=False)` then
150    /// `makeHab(name=alias, transferable=False)` on first start.
151    fn open(alias: &str, base_dir: PathBuf, config: &WitnessNetworkConfig) -> Result<Self> {
152        let salt = Salter::new(Some(&witness_salt(alias)), None, None)
153            .context("Failed to create salter")?
154            .qb64();
155        let mut hby = Habery::open(
156            alias,
157            HaberyOptions {
158                head_dir: Some(base_dir),
159                salt: Some(salt),
160                lax: true, // keripy witness Kevery: lax=True
161                ..Default::default()
162            },
163        )
164        .context("Failed to open witness keystore")?;
165
166        if hby.hab_by_name(alias).is_err() {
167            info!("Incepting witness AID for {}", alias);
168            hby.make_hab(
169                alias,
170                MakeHabArgs { transferable: Some(false), ..Default::default() },
171            )
172            .context("Failed to incept witness AID")?;
173        }
174
175        let hab = hby.hab_by_name(alias).context("Witness Hab missing after inception")?;
176        let pre = hab.pre.clone().ok_or_else(|| anyhow!("Witness Hab has no prefix"))?;
177
178        // Location replies in scheme order (keripy serves them sorted by
179        // scheme: http before tcp), then the controller end role, all stamped
180        // with the config file's dt so they are deterministic.
181        let mut curls: Vec<(String, String)> = config
182            .curls
183            .iter()
184            .filter_map(|url| {
185                let scheme = url.split("://").next()?.to_string();
186                Some((scheme, url.clone()))
187            })
188            .collect();
189        curls.sort();
190        let mut oobi_replies = Vec::new();
191        for (scheme, url) in curls {
192            oobi_replies.extend(
193                hab.make_loc_scheme(url, None, Some(scheme), Some(config.dt.clone()))
194                    .context("Failed to sign location reply")?,
195            );
196        }
197        oobi_replies.extend(
198            hab.make_end_role(pre.clone(), Some(Roles::Controller), Some(true), Some(config.dt.clone()))
199                .context("Failed to sign end role reply")?,
200        );
201        drop(hab);
202
203        info!("Witness {} ready: {}", alias, pre);
204        Ok(Self { alias: alias.to_string(), pre, hby, oobi_replies })
205    }
206
207    fn handle(&mut self, cmd: Command) {
208        match cmd {
209            Command::Receipt { body, attachments, reply } => {
210                let result = self.guarded("receipt", |a| a.receipt(&body, &attachments));
211                let _ = reply.send(result);
212            }
213            Command::Ingest { msg, reply } => {
214                let result = self.guarded("ingest", |a| a.ingest(&msg));
215                let _ = reply.send(result);
216            }
217            Command::Oobi { aid, role, reply } => {
218                let result = self.guarded("oobi", |a| a.oobi(&aid, role.as_deref()));
219                let _ = reply.send(result);
220            }
221        }
222    }
223
224    /// Run one command, turning a panic into an `Internal` error so a single bad
225    /// request (e.g. one that hits a libkeri `todo!()`) cannot take down the witness.
226    fn guarded<T>(
227        &mut self,
228        what: &str,
229        f: impl FnOnce(&mut Self) -> Result<T, ActorError>,
230    ) -> Result<T, ActorError> {
231        match panic::catch_unwind(AssertUnwindSafe(|| f(self))) {
232            Ok(result) => result,
233            Err(payload) => {
234                let msg = payload
235                    .downcast_ref::<&str>()
236                    .map(|s| s.to_string())
237                    .or_else(|| payload.downcast_ref::<String>().cloned())
238                    .unwrap_or_else(|| "unknown panic".to_string());
239                error!("Witness actor panicked in {}: {}", what, msg);
240                Err(ActorError::Internal(format!("panic in {}: {}", what, msg)))
241            }
242        }
243    }
244
245    /// keripy `ReceiptEnd.on_post`.
246    fn receipt(&mut self, body: &[u8], attachments: &str) -> Result<ReceiptOutcome, ActorError> {
247        let serder = SerderKERI::from_raw(body, None)
248            .map_err(|e| ActorError::BadRequest(format!("Invalid KERI event: {}", e)))?;
249        let pre = serder.pre().ok_or_else(|| ActorError::BadRequest("event has no prefix".into()))?;
250        let ilk = serder.ilk().map(|i| format!("{:?}", i).to_lowercase()).unwrap_or_default();
251        if !matches!(ilk.as_str(), "icp" | "rot" | "ixn" | "dip" | "drt") {
252            return Err(ActorError::BadRequest(format!("invalid event type ({}) for receipting", ilk)));
253        }
254        let said = serder.said().unwrap_or_default().to_string();
255        let sn = serder.sn().unwrap_or(0);
256        info!("Receipt request: pre={} sn={} said={}", pre, sn, said);
257
258        // Process as a local source, as keripy does, so the witness accepts an
259        // event it witnesses before the toad is met. Escrowable failures (e.g.
260        // partial signatures) are not request errors: fall through to 202.
261        let msg = libkeri_http::join(body, attachments);
262        if let Err(e) = self.hby.ingest(&msg, true) {
263            warn!("Event {} for {} not accepted: {}", said, pre, e);
264        }
265
266        let Some(kever) = self.hby.kevers().get(&pre) else {
267            return Ok(ReceiptOutcome::Pending);
268        };
269        if !kever.wits().contains(&self.pre) {
270            return Err(ActorError::BadRequest(format!(
271                "{} is not a valid witness for {} event at {}: wits={:?}",
272                self.pre, pre, sn, kever.wits()
273            )));
274        }
275        // Only receipt the version of the event this witness actually accepted.
276        let accepted = self
277            .hby
278            .db()
279            .get_ke_last(&libkeri::keri::db::dbing::keys::sn_key(&pre, sn))
280            .map_err(|e| ActorError::Internal(e.to_string()))?;
281        if accepted.as_deref() != Some(said.as_str()) {
282            return Ok(ReceiptOutcome::Pending);
283        }
284
285        let rct = self
286            .hby
287            .hab_by_name(&self.alias)
288            .and_then(|mut hab| hab.receipt(&serder))
289            .map_err(|e| ActorError::Internal(format!("receipt failed: {}", e)))?;
290        info!("Receipted {} sn={} for {}", pre, sn, said);
291        debug!("Receipt: {}", String::from_utf8_lossy(&rct));
292        Ok(ReceiptOutcome::Receipted(rct))
293    }
294
295    /// keripy `HttpEnd`: events and receipts pushed by peers (non-local source).
296    fn ingest(&mut self, msg: &[u8]) -> Result<(), ActorError> {
297        match self.hby.ingest(msg, false) {
298            Ok(n) => debug!("Ingested {} message(s)", n),
299            Err(e) => {
300                warn!("Ingest failed: {}", e);
301                debug!("Failed message: {}", String::from_utf8_lossy(msg));
302            }
303        }
304        Ok(())
305    }
306
307    /// keripy `OOBIEnd`: this witness's own AID (KEL plus signed replies), or
308    /// an AID it witnesses once that AID's latest event is fully witnessed
309    /// (that AID's KEL plus this witness's replies).
310    fn oobi(&mut self, aid: &str, role: Option<&str>) -> Result<Vec<u8>, ActorError> {
311        if aid != self.pre {
312            return self.witnessed_oobi(aid, role);
313        }
314        if let Some(role) = role {
315            if role != "controller" && role != "witness" {
316                return Err(ActorError::NotFound(format!("unsupported role {}", role)));
317            }
318        }
319        let hab = self
320            .hby
321            .hab_by_name(&self.alias)
322            .map_err(|e| ActorError::Internal(e.to_string()))?;
323        let mut msg = hab
324            .replay(None, None)
325            .map_err(|e| ActorError::Internal(format!("replay failed: {}", e)))?;
326        msg.extend_from_slice(&self.oobi_replies);
327        Ok(msg)
328    }
329}
330
331impl WitnessActor {
332    /// OOBI for an AID this witness witnesses (keripy `OOBIEnd.on_get` with
333    /// `fullyWitnessed`): 404 until its latest event holds toad witness
334    /// signatures, so a validator never gets a KEL it would have to escrow.
335    fn witnessed_oobi(&mut self, aid: &str, role: Option<&str>) -> Result<Vec<u8>, ActorError> {
336        if let Some(role) = role {
337            if role != "witness" {
338                return Err(ActorError::NotFound(format!("unsupported role {} for {}", role, aid)));
339            }
340        }
341        let Some(kever) = self.hby.kevers().get(aid) else {
342            return Err(ActorError::NotFound(format!("unknown AID {}", aid)));
343        };
344        if !kever.wits().contains(&self.pre) {
345            return Err(ActorError::NotFound(format!("not a witness for {}", aid)));
346        }
347        let said = kever
348            .serder()
349            .and_then(|s| s.said().map(str::to_string))
350            .ok_or_else(|| ActorError::Internal(format!("no latest event for {}", aid)))?;
351        let toad = kever.toader().map(|t| t.num() as usize).unwrap_or(0);
352        let wigs: Vec<Vec<u8>> = self
353            .hby
354            .db()
355            .wigs
356            .get(&[aid, said.as_str()])
357            .map_err(|e| ActorError::Internal(e.to_string()))?;
358        if wigs.len() < toad {
359            return Err(ActorError::NotFound(format!(
360                "{} not fully witnessed yet ({} of {} witness signatures)",
361                aid,
362                wigs.len(),
363                toad
364            )));
365        }
366        let hab = self
367            .hby
368            .hab_by_name(&self.alias)
369            .map_err(|e| ActorError::Internal(e.to_string()))?;
370        let mut msg = hab
371            .replay(Some(aid), None)
372            .map_err(|e| ActorError::Internal(format!("replay failed: {}", e)))?;
373        msg.extend_from_slice(&self.oobi_replies);
374        Ok(msg)
375    }
376}
377
378/// Returns the 16-byte salt for a known witness alias, matching keripy demo salts.
379fn witness_salt(alias: &str) -> Vec<u8> {
380    match alias {
381        "wan" => b"wann-the-witness".to_vec(),
382        "wil" => b"will-the-witness".to_vec(),
383        "wes" => b"wess-the-witness".to_vec(),
384        "wit" => b"witn-the-witness".to_vec(),
385        "wub" => b"wubl-the-witness".to_vec(),
386        "wyz" => b"wyzs-the-witness".to_vec(),
387        _ => {
388            // For unknown aliases, derive a 16-byte salt from the name
389            let mut salt = format!("{:_<16}", alias).into_bytes();
390            salt.truncate(16);
391            salt
392        }
393    }
394}