From b67675563333e20c56b0f169d590d1aaa7f7baa5 Mon Sep 17 00:00:00 2001 From: Serkyo Date: Wed, 8 Jul 2026 16:21:54 +0200 Subject: [PATCH] feat(shared): add SYNC per-chunk record encoding --- Cargo.lock | 127 ++++++++++++++++++- crates/shared/Cargo.toml | 3 + crates/shared/src/lib.rs | 1 + crates/shared/src/save/error.rs | 53 ++++++++ crates/shared/src/save/mod.rs | 10 ++ crates/shared/src/save/record.rs | 209 +++++++++++++++++++++++++++++++ 6 files changed, 400 insertions(+), 3 deletions(-) create mode 100644 crates/shared/src/save/error.rs create mode 100644 crates/shared/src/save/mod.rs create mode 100644 crates/shared/src/save/record.rs diff --git a/Cargo.lock b/Cargo.lock index d7734df..3103951 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,6 +143,15 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -238,7 +247,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde", - "spin", + "spin 0.10.0", "wasm-bindgen", "wasm-bindgen-futures", "windows-sys 0.61.2", @@ -304,7 +313,7 @@ dependencies = [ "crossbeam-queue", "derive_more", "futures-lite", - "heapless", + "heapless 0.9.3", "web-task", ] @@ -454,6 +463,15 @@ dependencies = [ "winit", ] +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "cocoa" version = "0.25.0" @@ -651,6 +669,18 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76" +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "equivalent" version = "1.0.2" @@ -848,6 +878,15 @@ dependencies = [ "windows", ] +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + [[package]] name = "hash32" version = "0.3.1" @@ -874,13 +913,27 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin 0.9.8", + "stable_deref_trait", +] + [[package]] name = "heapless" version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4" dependencies = [ - "hash32", + "hash32 0.3.1", "portable-atomic", "stable_deref_trait", ] @@ -1033,6 +1086,15 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.29" @@ -1505,6 +1567,19 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + [[package]] name = "presser" version = "0.3.1" @@ -1706,6 +1781,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "scripting" version = "0.1.0" @@ -1803,7 +1884,10 @@ dependencies = [ "fastrand", "glam 0.33.2", "noise", + "postcard", "serde", + "thiserror 2.0.18", + "zstd", ] [[package]] @@ -1883,6 +1967,15 @@ dependencies = [ "serde", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "spin" version = "0.10.0" @@ -2734,3 +2827,31 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/crates/shared/Cargo.toml b/crates/shared/Cargo.toml index c0b416f..8785ddb 100644 --- a/crates/shared/Cargo.toml +++ b/crates/shared/Cargo.toml @@ -13,4 +13,7 @@ bytemuck = { workspace = true } fastrand = "2.4.1" glam = { workspace = true } noise = "0.9" +postcard = { version = "1.1.3", features = ["use-std"] } serde = { version = "1.0", features = ["derive"] } +thiserror = "2.0.18" +zstd = "0.13.3" diff --git a/crates/shared/src/lib.rs b/crates/shared/src/lib.rs index 380c9ed..77e1965 100644 --- a/crates/shared/src/lib.rs +++ b/crates/shared/src/lib.rs @@ -7,3 +7,4 @@ pub mod generator; pub mod world; +pub mod save; diff --git a/crates/shared/src/save/error.rs b/crates/shared/src/save/error.rs new file mode 100644 index 0000000..481f016 --- /dev/null +++ b/crates/shared/src/save/error.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//! Error types for reading and writing the on-disk save format. + +use thiserror::Error; + +/// A failure encountered while encoding or decoding a save-format record. +#[derive(Debug, Error)] +pub enum SaveError { + /// The leading magic bytes did not match the expected record tag. + #[error("record magic mismatch: expected {expected:?}, found {found:?}")] + BadMagic { + /// The magic the record was expected to begin with. + expected: [u8; 4], + /// The magic actually found at the start of the buffer. + found: [u8; 4], + }, + + /// The buffer ended before a field of the declared size could be read. + #[error("record truncated at offset {offset}: needed {needed} bytes, {available} available")] + Truncated { + /// The byte offset at which the read was attempted. + offset: usize, + /// The number of bytes the field required. + needed: usize, + /// The number of bytes actually remaining from `offset`. + available: usize, + }, + + /// The decompressed payload length did not match the length the header declared. + #[error("payload length mismatch: header declared {expected} bytes, decompressed {actual}")] + LengthMismatch { + /// The uncompressed length recorded in the header. + expected: usize, + /// The length actually produced by decompression. + actual: usize, + }, + + /// The payload was too large for its length to fit the 32-bit header field. + #[error("payload too large to frame: {len} bytes exceeds the u32 length field")] + PayloadTooLarge { + /// The oversized payload length in bytes. + len: usize, + }, + + /// The payload could not be (de)serialized by `postcard`. + #[error("payload serialization failed")] + Serialization(#[from] postcard::Error), + + /// Compression or decompression failed at the I/O layer (`zstd`). + #[error("payload compression failed")] + Compression(#[from] std::io::Error), +} diff --git a/crates/shared/src/save/mod.rs b/crates/shared/src/save/mod.rs new file mode 100644 index 0000000..b3db647 --- /dev/null +++ b/crates/shared/src/save/mod.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//! On-disk save format: the framing that persists modified chunks to region files. +//! +//! The format is built bottom-up. The smallest unit is the `SYNC` per-chunk [`record`], which wraps one [`crate::world::ChunkData`] in a self-describing, compressed frame. Region-level framing (the `SYNR` file and its header table) is layered on top of it. All parsing treats on-disk bytes as untrusted and reports failures through [`SaveError`]. + +mod error; +pub mod record; + +pub use error::SaveError; diff --git a/crates/shared/src/save/record.rs b/crates/shared/src/save/record.rs new file mode 100644 index 0000000..7114830 --- /dev/null +++ b/crates/shared/src/save/record.rs @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//! The `SYNC` per-chunk record: the on-disk framing around one [`ChunkData`]. +//! +//! A record is a fixed-size header followed by a zstd-compressed, postcard-serialized [`ChunkData`] payload. All multi-byte integers are little-endian. The framing itself is never compressed, so a repair tool can read the header without decompressing. + +use super::error::SaveError; +use crate::world::ChunkData; + +/// The magic tag every chunk record begins with. +const MAGIC: [u8; 4] = *b"SYNC"; + +/// The current schema version of the `ChunkData` payload, written into every new record. +pub const CHUNK_FORMAT_VERSION: u16 = 1; + +/// The zstd compression level used for chunk payloads: level 3 favours speed, per the save-format design. +const ZSTD_LEVEL: i32 = 3; + +/// The size in bytes of the fixed record header: magic + version + flags + timestamp + two length fields. +const HEADER_LEN: usize = 4 + 2 + 2 + 8 + 4 + 4; + +/// The non-payload header fields of a decoded record. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RecordMeta { + /// The schema version the payload was written under, used to drive per-chunk migration on load. + pub chunk_format_version: u16, + /// Reserved record flags; currently always zero. + pub flags: u16, + /// The wall-clock time the chunk was last modified, in milliseconds since the Unix epoch. + pub last_modified: u64, +} + +/// Encodes `data` into a `SYNC` record, stamping it with `last_modified` (unix-ms). +pub fn encode(data: &ChunkData, last_modified: u64) -> Result, SaveError> { + let uncompressed = postcard::to_stdvec(data)?; + let compressed = zstd::encode_all(uncompressed.as_slice(), ZSTD_LEVEL)?; + + // Checked conversion so an oversized payload fails loudly instead of truncating. + let uncompressed_len = + u32::try_from(uncompressed.len()).map_err(|_| SaveError::PayloadTooLarge { + len: uncompressed.len(), + })?; + let compressed_len = u32::try_from(compressed.len()).map_err(|_| SaveError::PayloadTooLarge { + len: compressed.len(), + })?; + + let mut out = Vec::with_capacity(HEADER_LEN + compressed.len()); + out.extend_from_slice(&MAGIC); + out.extend_from_slice(&CHUNK_FORMAT_VERSION.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&last_modified.to_le_bytes()); + out.extend_from_slice(&uncompressed_len.to_le_bytes()); + out.extend_from_slice(&compressed_len.to_le_bytes()); + out.extend_from_slice(&compressed); + Ok(out) +} + +/// Decodes a `SYNC` record, returning its header metadata and the reconstructed [`ChunkData`]. +/// +/// `bytes` is untrusted on-disk input, so every field is bounds-checked and the decompressed +/// payload length is validated against the header before deserialization is attempted. +pub fn decode(bytes: &[u8]) -> Result<(RecordMeta, ChunkData), SaveError> { + let mut reader = Reader::new(bytes); + + let magic = reader.take_array::<4>()?; + if magic != MAGIC { + return Err(SaveError::BadMagic { + expected: MAGIC, + found: magic, + }); + } + + let chunk_format_version = u16::from_le_bytes(reader.take_array()?); + let flags = u16::from_le_bytes(reader.take_array()?); + let last_modified = u64::from_le_bytes(reader.take_array()?); + // Widening u32 -> usize is lossless on every supported (64-bit) target. + let uncompressed_len = u32::from_le_bytes(reader.take_array()?) as usize; + let compressed_len = u32::from_le_bytes(reader.take_array()?) as usize; + + let payload = reader.take(compressed_len)?; + let decompressed = zstd::decode_all(payload)?; + if decompressed.len() != uncompressed_len { + return Err(SaveError::LengthMismatch { + expected: uncompressed_len, + actual: decompressed.len(), + }); + } + + let data: ChunkData = postcard::from_bytes(&decompressed)?; + let meta = RecordMeta { + chunk_format_version, + flags, + last_modified, + }; + Ok((meta, data)) +} + +/// A forward-only cursor over a byte slice that bounds-checks every read. +struct Reader<'a> { + /// The full buffer being read. + bytes: &'a [u8], + /// The offset of the next unread byte. + offset: usize, +} + +impl<'a> Reader<'a> { + /// Creates a reader positioned at the start of `bytes`. + fn new(bytes: &'a [u8]) -> Self { + Self { bytes, offset: 0 } + } + + /// Returns the next `n` bytes and advances the cursor, or [`SaveError::Truncated`] if fewer remain. + fn take(&mut self, n: usize) -> Result<&'a [u8], SaveError> { + let end = self + .offset + .checked_add(n) + .ok_or(SaveError::Truncated { + offset: self.offset, + needed: n, + available: self.bytes.len().saturating_sub(self.offset), + })?; + let slice = self.bytes.get(self.offset..end).ok_or(SaveError::Truncated { + offset: self.offset, + needed: n, + available: self.bytes.len().saturating_sub(self.offset), + })?; + self.offset = end; + Ok(slice) + } + + /// Returns the next `N` bytes as a fixed-size array and advances the cursor. + fn take_array(&mut self) -> Result<[u8; N], SaveError> { + let mut array = [0u8; N]; + array.copy_from_slice(self.take(N)?); + Ok(array) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::world::{BlockId, ChunkPos}; + + /// Builds a representative modified chunk with a few edits spanning the local index range. + fn sample() -> ChunkData { + let mut data = ChunkData::new(ChunkPos::new(1, -2, 3), 7); + data.set(0, BlockId(4)); + data.set(1000, BlockId(9)); + data.set(32_767, BlockId(2)); + data + } + + #[test] + fn round_trips_payload_and_metadata() -> Result<(), SaveError> { + let data = sample(); + let bytes = encode(&data, 123_456)?; + let (meta, decoded) = decode(&bytes)?; + + assert_eq!(decoded, data); + assert_eq!(meta.chunk_format_version, CHUNK_FORMAT_VERSION); + assert_eq!(meta.flags, 0); + assert_eq!(meta.last_modified, 123_456); + Ok(()) + } + + #[test] + fn rejects_bad_magic() -> Result<(), SaveError> { + let mut bytes = encode(&sample(), 0)?; + bytes[0] = b'X'; + assert!(matches!(decode(&bytes), Err(SaveError::BadMagic { .. }))); + Ok(()) + } + + #[test] + fn rejects_truncated_header() -> Result<(), SaveError> { + let bytes = encode(&sample(), 0)?; + // A buffer shorter than the fixed header cannot yield a full record. + assert!(matches!( + decode(&bytes[..HEADER_LEN - 1]), + Err(SaveError::Truncated { .. }) + )); + Ok(()) + } + + #[test] + fn rejects_truncated_payload() -> Result<(), SaveError> { + let bytes = encode(&sample(), 0)?; + // Keep the whole header but cut the compressed payload short. + assert!(matches!( + decode(&bytes[..=HEADER_LEN]), + Err(SaveError::Truncated { .. }) + )); + Ok(()) + } + + #[test] + fn detects_declared_length_mismatch() -> Result<(), SaveError> { + let mut bytes = encode(&sample(), 0)?; + // The uncompressed-length field is the u32 at offset 16 (after magic, version, + // flags, and the timestamp). Overwriting it with a value the payload cannot + // decompress to must be caught by the post-decompression length check. + bytes[16..20].copy_from_slice(&1u32.to_le_bytes()); + assert!(matches!( + decode(&bytes), + Err(SaveError::LengthMismatch { .. }) + )); + Ok(()) + } +}