//! Canonical byte encoding helpers. //! //! Chapter 8 §"Binary Format Companion" defers the exact byte-level layout //! (varint conventions, field bit widths, struct alignment) to a separate //! specification that does not yet exist. To make the bundle's atomic-commit, //! crash-recovery, and re-serialization guarantees testable now, this crate //! defines a concrete, fixed-convention canonical encoding (see `DECISIONS.md`, //! and the parallel choice in `epiphany-core` DECISIONS P11-4). The conventions: //! //! * Integers are little-endian (matching the spec's hash-preimage convention, //! which serializes `uncompressed_length` as `to_le_bytes`, Chapter 8 //! §"Domain-Separated Preimages"). //! * Variable-length fields are `u32` length-prefixed. //! * `Option`s are a single presence byte (`0`/`1`) followed by the payload. //! * Vectors are a `u32` count followed by each element, in canonical order //! (Appendix D §"Ordered Iteration"); ordering is the encoder's job, fixed at //! serialization time so re-encoding is byte-stable. //! //! [`Writer`] is infallible (it grows a `Vec`). [`Reader`] is *total* and //! bounds-checked: every read either consumes exactly the bytes it needs or //! returns a [`DecodeError`]. It never panics and never trusts a length prefix //! before checking it against the bytes actually remaining, so a corrupt or //! adversarial buffer cannot drive an over-allocation. /// Why decoding a canonical byte buffer failed. Decoding is partial: not every /// byte string is a valid structure, and invalid input is always reported, /// never silently accepted (Appendix D: readers treat malformed canonical bytes /// as corruption). #[derive(Clone, PartialEq, Eq, Debug)] pub enum DecodeError { /// A read ran past the end of the buffer. UnexpectedEof { needed: usize, remaining: usize }, /// Bytes remained after a fixed-shape structure was fully decoded. TrailingBytes { remaining: usize }, /// An enum discriminant did not name a known variant. InvalidDiscriminant { what: &'static str, value: u64 }, /// A length prefix exceeded the bytes actually present (corruption, or a /// truncated buffer). Caught *before* any allocation. LengthOverflow { declared: u64, remaining: usize }, /// A presence byte for an `Option` was neither `0` nor `1`. InvalidPresenceByte(u8), /// A text field was not valid UTF-8. InvalidUtf8, /// A structural rule was violated (described by a stable, locale-independent /// reason string). Malformed(&'static str), } impl core::fmt::Display for DecodeError { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { DecodeError::UnexpectedEof { needed, remaining } => { write!( f, "unexpected end of input: needed {needed}, had {remaining}" ) } DecodeError::TrailingBytes { remaining } => { write!(f, "{remaining} trailing bytes after a fixed-shape value") } DecodeError::InvalidDiscriminant { what, value } => { write!(f, "invalid {what} discriminant {value}") } DecodeError::LengthOverflow { declared, remaining, } => { write!( f, "declared length {declared} exceeds {remaining} remaining bytes" ) } DecodeError::InvalidPresenceByte(b) => write!(f, "invalid option presence byte {b}"), DecodeError::InvalidUtf8 => f.write_str("invalid UTF-8 in canonical text field"), DecodeError::Malformed(why) => write!(f, "malformed canonical bytes: {why}"), } } } impl std::error::Error for DecodeError {} /// A growable canonical byte sink. Encoding is infallible and deterministic. #[derive(Default)] pub struct Writer { buf: Vec, } impl Writer { /// A fresh writer. #[inline] pub fn new() -> Self { Writer { buf: Vec::new() } } /// A writer pre-sized for `cap` bytes. #[inline] pub fn with_capacity(cap: usize) -> Self { Writer { buf: Vec::with_capacity(cap), } } /// Consumes the writer, returning the encoded bytes. #[inline] pub fn into_bytes(self) -> Vec { self.buf } /// The bytes written so far. #[inline] pub fn as_bytes(&self) -> &[u8] { &self.buf } /// Bytes written so far. #[inline] pub fn len(&self) -> usize { self.buf.len() } /// Whether nothing has been written. #[inline] pub fn is_empty(&self) -> bool { self.buf.is_empty() } #[inline] pub fn put_u8(&mut self, v: u8) -> &mut Self { self.buf.push(v); self } #[inline] pub fn put_u16(&mut self, v: u16) -> &mut Self { self.buf.extend_from_slice(&v.to_le_bytes()); self } #[inline] pub fn put_u32(&mut self, v: u32) -> &mut Self { self.buf.extend_from_slice(&v.to_le_bytes()); self } #[inline] pub fn put_u64(&mut self, v: u64) -> &mut Self { self.buf.extend_from_slice(&v.to_le_bytes()); self } #[inline] pub fn put_u128(&mut self, v: u128) -> &mut Self { self.buf.extend_from_slice(&v.to_le_bytes()); self } #[inline] pub fn put_i64(&mut self, v: i64) -> &mut Self { self.buf.extend_from_slice(&v.to_le_bytes()); self } /// Raw bytes with no length prefix (the caller knows the width). #[inline] pub fn put_bytes(&mut self, bytes: &[u8]) -> &mut Self { self.buf.extend_from_slice(bytes); self } /// A `u32`-length-prefixed byte string. A length that would overflow the /// `u32` prefix is rejected in **all** build profiles (a hard `assert!`, not /// a `debug_assert!`): aborting beats silently truncating the prefix and /// emitting corrupt bytes. v0 inputs are bounded far below 4 GiB by the /// reader resource limits, so this is unreachable in practice. #[inline] pub fn put_var_bytes(&mut self, bytes: &[u8]) -> &mut Self { assert!( bytes.len() <= u32::MAX as usize, "var-bytes length {} overflows the u32 prefix", bytes.len() ); self.put_u32(bytes.len() as u32); self.buf.extend_from_slice(bytes); self } /// A `bool` as a single `0`/`1` byte. #[inline] pub fn put_bool(&mut self, v: bool) -> &mut Self { self.put_u8(v as u8) } /// An optional value: a presence byte, then `encode` if present. #[inline] pub fn put_opt( &mut self, value: &Option, encode: impl FnOnce(&mut Self, &T), ) -> &mut Self { match value { Some(v) => { self.put_u8(1); encode(self, v); } None => { self.put_u8(0); } } self } /// A length-prefixed, canonically ordered sequence. The caller is /// responsible for having ordered `items` canonically before encoding. #[inline] pub fn put_seq(&mut self, items: &[T], mut encode: impl FnMut(&mut Self, &T)) -> &mut Self { assert!( items.len() <= u32::MAX as usize, "sequence length {} overflows the u32 count", items.len() ); self.put_u32(items.len() as u32); for item in items { encode(self, item); } self } } /// A bounds-checked cursor over canonical bytes. Every read is total: it either /// consumes exactly the bytes it needs or returns [`DecodeError::UnexpectedEof`]. pub struct Reader<'a> { bytes: &'a [u8], pos: usize, } impl<'a> Reader<'a> { /// Wraps a byte slice. #[inline] pub fn new(bytes: &'a [u8]) -> Self { Reader { bytes, pos: 0 } } /// Bytes not yet consumed. #[inline] pub fn remaining(&self) -> usize { self.bytes.len() - self.pos } /// Errors unless the buffer is fully consumed. Call after decoding a /// fixed-shape structure to reject trailing bytes (the encoding is exact). #[inline] pub fn finish(&self) -> Result<(), DecodeError> { if self.remaining() == 0 { Ok(()) } else { Err(DecodeError::TrailingBytes { remaining: self.remaining(), }) } } #[inline] fn take(&mut self, n: usize) -> Result<&'a [u8], DecodeError> { if n > self.remaining() { return Err(DecodeError::UnexpectedEof { needed: n, remaining: self.remaining(), }); } let out = &self.bytes[self.pos..self.pos + n]; self.pos += n; Ok(out) } #[inline] pub fn take_array(&mut self) -> Result<[u8; N], DecodeError> { let mut arr = [0u8; N]; arr.copy_from_slice(self.take(N)?); Ok(arr) } #[inline] pub fn get_u8(&mut self) -> Result { Ok(self.take(1)?[0]) } #[inline] pub fn get_u16(&mut self) -> Result { Ok(u16::from_le_bytes(self.take_array()?)) } #[inline] pub fn get_u32(&mut self) -> Result { Ok(u32::from_le_bytes(self.take_array()?)) } #[inline] pub fn get_u64(&mut self) -> Result { Ok(u64::from_le_bytes(self.take_array()?)) } #[inline] pub fn get_u128(&mut self) -> Result { Ok(u128::from_le_bytes(self.take_array()?)) } #[inline] pub fn get_i64(&mut self) -> Result { Ok(i64::from_le_bytes(self.take_array()?)) } /// A `u32`-length-prefixed byte string, *borrowed* from the input rather /// than copied. The length prefix is checked against the bytes remaining /// before the slice is taken. The zero-copy counterpart of /// [`Reader::get_var_bytes`], for callers that need positions or slices /// into the original buffer (e.g. the per-envelope offsets the operation /// index records). pub fn get_var_slice(&mut self) -> Result<&'a [u8], DecodeError> { let len = self.get_u32()? as usize; if len > self.remaining() { return Err(DecodeError::LengthOverflow { declared: len as u64, remaining: self.remaining(), }); } self.take(len) } /// A `u32`-length-prefixed byte string, copied out. The length prefix is /// checked against the bytes remaining *before* any allocation. pub fn get_var_bytes(&mut self) -> Result, DecodeError> { Ok(self.get_var_slice()?.to_vec()) } /// A `u32`-length-prefixed UTF-8 string. pub fn get_string(&mut self) -> Result { let bytes = self.get_var_bytes()?; String::from_utf8(bytes).map_err(|_| DecodeError::InvalidUtf8) } /// A `bool` from a single `0`/`1` byte. pub fn get_bool(&mut self) -> Result { match self.get_u8()? { 0 => Ok(false), 1 => Ok(true), other => Err(DecodeError::InvalidPresenceByte(other)), } } /// An optional value: a presence byte, then `decode` if present. pub fn get_opt( &mut self, decode: impl FnOnce(&mut Self) -> Result, ) -> Result, DecodeError> { match self.get_u8()? { 0 => Ok(None), 1 => Ok(Some(decode(self)?)), other => Err(DecodeError::InvalidPresenceByte(other)), } } /// A length-prefixed sequence. The count is checked against the bytes /// remaining before allocating, so a corrupt count cannot drive the loop /// past the buffer (each element is at least one byte, so `count <= /// remaining`). The *reserved* capacity is additionally capped: a multi-byte /// element type would otherwise let `count` (bounded only by byte count) /// reserve several times the input size, so we grow from a small reservation /// instead. pub fn get_seq( &mut self, mut decode: impl FnMut(&mut Self) -> Result, ) -> Result, DecodeError> { const MAX_RESERVE: usize = 1024; let count = self.get_u32()? as usize; if count > self.remaining() { return Err(DecodeError::LengthOverflow { declared: count as u64, remaining: self.remaining(), }); } let mut out = Vec::with_capacity(count.min(MAX_RESERVE)); for _ in 0..count { out.push(decode(self)?); } Ok(out) } } #[cfg(test)] mod tests { use super::*; #[test] fn primitives_round_trip() { let mut w = Writer::new(); w.put_u8(0x12) .put_u16(0x3456) .put_u32(0x789a_bcde) .put_u64(0x0102_0304_0506_0708) .put_u128(0x1) .put_i64(-5) .put_bool(true) .put_var_bytes(b"hello"); let bytes = w.into_bytes(); let mut r = Reader::new(&bytes); assert_eq!(r.get_u8().unwrap(), 0x12); assert_eq!(r.get_u16().unwrap(), 0x3456); assert_eq!(r.get_u32().unwrap(), 0x789a_bcde); assert_eq!(r.get_u64().unwrap(), 0x0102_0304_0506_0708); assert_eq!(r.get_u128().unwrap(), 1); assert_eq!(r.get_i64().unwrap(), -5); assert!(r.get_bool().unwrap()); assert_eq!(r.get_var_bytes().unwrap(), b"hello"); assert!(r.finish().is_ok()); } #[test] fn short_read_is_an_error_not_a_panic() { let mut r = Reader::new(&[0u8; 3]); assert_eq!( r.get_u64(), Err(DecodeError::UnexpectedEof { needed: 8, remaining: 3 }) ); } #[test] fn corrupt_length_prefix_cannot_over_allocate() { // A 4-byte buffer claiming a 4-billion-byte payload must error, not OOM. let mut bytes = u32::MAX.to_le_bytes().to_vec(); bytes.push(0); // one stray payload byte let mut r = Reader::new(&bytes); assert!(matches!( r.get_var_bytes(), Err(DecodeError::LengthOverflow { .. }) )); } #[test] fn seq_and_opt_round_trip() { let mut w = Writer::new(); w.put_seq(&[1u32, 2, 3], |w, v| { w.put_u32(*v); }); w.put_opt(&Some(7u8), |w, v| { w.put_u8(*v); }); w.put_opt(&None::, |w, v| { w.put_u8(*v); }); let bytes = w.into_bytes(); let mut r = Reader::new(&bytes); assert_eq!(r.get_seq(|r| r.get_u32()).unwrap(), vec![1, 2, 3]); assert_eq!(r.get_opt(|r| r.get_u8()).unwrap(), Some(7)); assert_eq!(r.get_opt(|r| r.get_u8()).unwrap(), None); assert!(r.finish().is_ok()); } #[test] fn trailing_bytes_rejected() { let mut r = Reader::new(&[1, 2, 3]); assert_eq!(r.get_u8().unwrap(), 1); assert_eq!(r.finish(), Err(DecodeError::TrailingBytes { remaining: 2 })); } }