diff --git a/Cargo.lock b/Cargo.lock index d7734df..1991352 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,6 +143,15 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -238,7 +247,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde", - "spin", + "spin 0.10.0", "wasm-bindgen", "wasm-bindgen-futures", "windows-sys 0.61.2", @@ -304,7 +313,7 @@ dependencies = [ "crossbeam-queue", "derive_more", "futures-lite", - "heapless", + "heapless 0.9.3", "web-task", ] @@ -454,6 +463,15 @@ dependencies = [ "winit", ] +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "cocoa" version = "0.25.0" @@ -651,6 +669,18 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76" +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "equivalent" version = "1.0.2" @@ -848,6 +878,15 @@ dependencies = [ "windows", ] +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + [[package]] name = "hash32" version = "0.3.1" @@ -874,13 +913,27 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin 0.9.8", + "stable_deref_trait", +] + [[package]] name = "heapless" version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4" dependencies = [ - "hash32", + "hash32 0.3.1", "portable-atomic", "stable_deref_trait", ] @@ -1033,6 +1086,15 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.29" @@ -1505,6 +1567,19 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + [[package]] name = "presser" version = "0.3.1" @@ -1706,6 +1781,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "scripting" version = "0.1.0" @@ -1782,6 +1863,7 @@ dependencies = [ "glam 0.33.2", "serde_json", "shared", + "tempfile", "tracing", "tracing-subscriber", ] @@ -1803,7 +1885,10 @@ dependencies = [ "fastrand", "glam 0.33.2", "noise", + "postcard", "serde", + "thiserror 2.0.18", + "zstd", ] [[package]] @@ -1883,6 +1968,15 @@ dependencies = [ "serde", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "spin" version = "0.10.0" @@ -1915,6 +2009,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -2734,3 +2841,31 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index e76c546..d1cecfa 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -17,3 +17,6 @@ serde_json = { workspace = true } shared = { version = "0.1.0", path = "../shared" } tracing = { workspace = true } tracing-subscriber = { workspace = true } + +[dev-dependencies] +tempfile = "3.27.0" diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 5752e95..e87a33a 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -7,6 +7,8 @@ /// Entity components describing players and other world-streaming anchors. pub mod player; +/// On-disk persistence: region files and the atomic durability layer. +pub mod save; /// Authoritative chunk storage and generation logic for the server. pub mod world_server; diff --git a/crates/server/src/save.rs b/crates/server/src/save.rs new file mode 100644 index 0000000..92ce02d --- /dev/null +++ b/crates/server/src/save.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//! Server-side persistence: the durability layer over the `shared` save format. +//! +//! `shared::save` owns the pure, in-memory framing (the `SYNR` index and `SYNC` records). This +//! module owns the filesystem side: reading a `.region` file into memory, mutating its chunks, and +//! flushing it back to disk crash-safely. The write strategy is a whole-file atomic rewrite +//! (`.tmp` + fsync + rename); the on-disk format is unchanged, so a later slice can switch to an +//! append-in-place strategy without a format change. + +mod region_file; + +pub use region_file::{REGION_SIZE, RegionFile, region_coords, region_path}; diff --git a/crates/server/src/save/region_file.rs b/crates/server/src/save/region_file.rs new file mode 100644 index 0000000..26e01ad --- /dev/null +++ b/crates/server/src/save/region_file.rs @@ -0,0 +1,353 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//! The durability layer for a single region file. + +use std::collections::BTreeMap; +use std::ffi::OsString; +use std::fs::{self, File}; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use shared::save::SaveError; +use shared::save::record; +use shared::save::region::{HeaderEntry, RegionIndex}; +use shared::world::{ChunkData, ChunkPos}; + +/// The side length, in chunk columns, of the square footprint one region file covers. +pub const REGION_SIZE: i32 = 32; + +/// Maps a chunk column `(cx, cz)` to the coordinates `(rx, rz)` of the region that contains it. +#[must_use] +pub fn region_coords(cx: i32, cz: i32) -> (i32, i32) { + (cx.div_euclid(REGION_SIZE), cz.div_euclid(REGION_SIZE)) +} + +/// Builds the on-disk path of the region file containing chunk column `(cx, cz)` within `dir`. +#[must_use] +pub fn region_path(dir: &Path, cx: i32, cz: i32) -> PathBuf { + let (rx, rz) = region_coords(cx, cz); + dir.join(format!("r.{rx}.{rz}.region")) +} + +/// An open region file: its `SYNR` index, the resident chunk records, and its on-disk location. +pub struct RegionFile { + /// The region framing (header table, free list, stamp table) held in memory. + index: RegionIndex, + /// Each resident chunk's raw `SYNC` record bytes, decoded lazily on read. + records: BTreeMap>, + /// The path this region is read from and written back to. + path: PathBuf, + /// Whether an in-memory mutation is pending a flush to disk. + dirty: bool, +} + +impl RegionFile { + /// Opens the region file at `path`, or yields an empty region if the file does not yet exist. + pub fn open(path: PathBuf) -> Result { + if !path.exists() { + return Ok(Self { + index: RegionIndex::new(0), + records: BTreeMap::new(), + path, + dirty: false, + }); + } + + let bytes = fs::read(&path)?; + let index = RegionIndex::decode(&bytes)?; + + // Slice each record out of the file by the absolute (offset, length) the header table records. On-disk bytes are untrusted, so an out-of-range span is rejected rather than panicking on the slice. + let mut records = BTreeMap::new(); + for (pos, entry) in index.entries() { + // Widening u32 -> usize is lossless on every supported (64-bit) target; the u64 offset is range-checked by try_from, failing loudly on a 32-bit target rather than wrapping. + let start = usize::try_from(entry.offset).map_err(|_| SaveError::PayloadTooLarge { + len: entry.length as usize, + })?; + let end = + start + .checked_add(entry.length as usize) + .ok_or(SaveError::PayloadTooLarge { + len: entry.length as usize, + })?; + let record_bytes = bytes + .get(start..end) + .ok_or(SaveError::Truncated { + offset: start, + needed: entry.length as usize, + available: bytes.len().saturating_sub(start), + })? + .to_vec(); + records.insert(*pos, record_bytes); + } + + Ok(Self { + index, + records, + path, + dirty: false, + }) + } + + /// Returns the number of resident chunk records. + #[must_use] + pub fn len(&self) -> usize { + self.records.len() + } + + /// Returns whether the region holds no chunk records. + #[must_use] + pub fn is_empty(&self) -> bool { + self.records.is_empty() + } + + /// Whether an in-memory mutation is pending a flush to disk. + #[must_use] + pub fn is_dirty(&self) -> bool { + self.dirty + } + + /// Decodes and returns the chunk at `pos`, or `None` if the region holds no record for it. + pub fn read_chunk(&self, pos: ChunkPos) -> Result, SaveError> { + match self.records.get(&pos) { + Some(bytes) => { + let (_meta, data) = record::decode(bytes)?; + Ok(Some(data)) + } + None => Ok(None), + } + } + + /// Encodes `data` into a `SYNC` record stamped with `last_modified` and stores it under `pos`. + pub fn write_chunk( + &mut self, + pos: ChunkPos, + data: &ChunkData, + last_modified: u64, + ) -> Result<(), SaveError> { + let bytes = record::encode(data, last_modified)?; + let length = u32::try_from(bytes.len()) + .map_err(|_| SaveError::PayloadTooLarge { len: bytes.len() })?; + self.records.insert(pos, bytes); + self.index.insert( + pos, + HeaderEntry { + offset: 0, + length, + flags: 0, + }, + ); + self.dirty = true; + Ok(()) + } + + /// Removes the chunk at `pos` from the region, if present. + pub fn remove_chunk(&mut self, pos: ChunkPos) { + let removed = self.records.remove(&pos).is_some(); + self.index.remove(pos); + if removed { + self.dirty = true; + } + } + + /// Flushes the region to disk with a crash-safe whole-file atomic rewrite, clearing the dirty flag. + pub fn save(&mut self) -> Result<(), SaveError> { + let image = self.serialize()?; + atomic_write(&self.path, &image)?; + self.dirty = false; + Ok(()) + } + + /// Builds the complete on-disk file image: the encoded index followed by every record. + fn serialize(&mut self) -> Result, SaveError> { + let index_len = self.index.encode()?.len(); + + // Assign each record a contiguous offset in ascending position order (BTreeMap order), the same order the records are concatenated below. + let mut offset = index_len as u64; + for (pos, bytes) in &self.records { + let length = u32::try_from(bytes.len()) + .map_err(|_| SaveError::PayloadTooLarge { len: bytes.len() })?; + self.index.insert( + *pos, + HeaderEntry { + offset, + length, + flags: 0, + }, + ); + offset += bytes.len() as u64; + } + + let mut image = self.index.encode()?; + for bytes in self.records.values() { + image.extend_from_slice(bytes); + } + Ok(image) + } +} + +/// Writes `bytes` to `path` via the POSIX atomic-write pattern: `.tmp` + fsync + rename. +fn atomic_write(path: &Path, bytes: &[u8]) -> Result<(), SaveError> { + // The region directory is created on demand so the first write to a fresh world succeeds. + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + + // The temp path appends ".tmp" to the full filename (rather than replacing the extension) so it sits beside the target on the same filesystem, keeping the rename atomic. + let mut tmp_name: OsString = path.as_os_str().to_owned(); + tmp_name.push(".tmp"); + let tmp_path = PathBuf::from(tmp_name); + + let mut file = File::create(&tmp_path)?; + file.write_all(bytes)?; + // fsync the data to disk before the rename, so the rename cannot expose an unwritten file. + file.sync_all()?; + drop(file); + + fs::rename(&tmp_path, path)?; + + // fsync the parent directory so the rename itself is durable. Opening a directory for fsync is a Unix affordance; Windows does not expose a directory handle to sync, so the step is skipped there. + #[cfg(unix)] + if let Some(parent) = path.parent() { + File::open(parent)?.sync_all()?; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use shared::world::BlockId; + + /// Builds a representative modified chunk with a few edits spanning the local index range. + fn sample(pos: ChunkPos) -> ChunkData { + let mut data = ChunkData::new(pos, 7); + data.set(0, BlockId(4)); + data.set(1000, BlockId(9)); + data.set(32_767, BlockId(2)); + data + } + + #[test] + fn region_coords_floor_negative_columns() { + // Truncating division would map -1 to region 0; Euclidean flooring maps it to region -1. + assert_eq!(region_coords(0, 0), (0, 0)); + assert_eq!(region_coords(31, 31), (0, 0)); + assert_eq!(region_coords(-1, -1), (-1, -1)); + assert_eq!(region_coords(-32, -33), (-1, -2)); + } + + #[test] + fn region_path_names_the_region_file() { + let dir = Path::new("/saves/world/region"); + assert_eq!( + region_path(dir, -1, 5), + Path::new("/saves/world/region/r.-1.0.region") + ); + } + + #[test] + fn open_missing_file_is_empty() -> Result<(), SaveError> { + let dir = tempfile::tempdir()?; + let region = RegionFile::open(dir.path().join("r.0.0.region"))?; + assert!(region.is_empty()); + assert_eq!(region.read_chunk(ChunkPos::new(0, 0, 0))?, None); + Ok(()) + } + + #[test] + fn round_trips_chunks_through_disk() -> Result<(), SaveError> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("r.0.0.region"); + + let positions = [ + ChunkPos::new(0, 0, 0), + ChunkPos::new(1, 2, 3), + ChunkPos::new(-5, 10, -30), + ]; + + let mut region = RegionFile::open(path.clone())?; + for pos in positions { + region.write_chunk(pos, &sample(pos), 123)?; + } + assert!(region.is_dirty()); + region.save()?; + assert!(!region.is_dirty()); + + // Reopen from disk in a fresh instance and confirm every chunk decodes byte-identically. + let reopened = RegionFile::open(path)?; + assert_eq!(reopened.len(), positions.len()); + for pos in positions { + assert_eq!(reopened.read_chunk(pos)?, Some(sample(pos))); + } + // A position never written has no record. + assert_eq!(reopened.read_chunk(ChunkPos::new(9, 9, 9))?, None); + Ok(()) + } + + #[test] + fn record_offsets_are_valid_and_contiguous() -> Result<(), SaveError> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("r.0.0.region"); + + let mut region = RegionFile::open(path.clone())?; + for pos in [ + ChunkPos::new(0, 0, 0), + ChunkPos::new(2, 0, 1), + ChunkPos::new(-1, 4, -1), + ] { + region.write_chunk(pos, &sample(pos), 0)?; + } + region.save()?; + + let reopened = RegionFile::open(path)?; + let index_len = reopened.index.encode()?.len() as u64; + + // Records are packed contiguously immediately after the index, in ascending position order. + let mut expected_offset = index_len; + for (_pos, entry) in reopened.index.entries() { + assert_eq!(entry.offset, expected_offset); + expected_offset += u64::from(entry.length); + } + Ok(()) + } + + #[test] + fn remove_drops_only_the_named_chunk() -> Result<(), SaveError> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("r.0.0.region"); + let kept = ChunkPos::new(0, 0, 0); + let dropped = ChunkPos::new(1, 1, 1); + + let mut region = RegionFile::open(path.clone())?; + region.write_chunk(kept, &sample(kept), 0)?; + region.write_chunk(dropped, &sample(dropped), 0)?; + region.save()?; + + let mut region = RegionFile::open(path.clone())?; + region.remove_chunk(dropped); + region.save()?; + + let reopened = RegionFile::open(path)?; + assert_eq!(reopened.read_chunk(dropped)?, None); + assert_eq!(reopened.read_chunk(kept)?, Some(sample(kept))); + Ok(()) + } + + #[test] + fn stray_tmp_file_does_not_corrupt_reads() -> Result<(), SaveError> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("r.0.0.region"); + let pos = ChunkPos::new(0, 0, 0); + + let mut region = RegionFile::open(path.clone())?; + region.write_chunk(pos, &sample(pos), 0)?; + region.save()?; + + // A leftover .tmp from an interrupted save must be ignored: only the renamed target is read. + fs::write(dir.path().join("r.0.0.region.tmp"), b"garbage")?; + let reopened = RegionFile::open(path)?; + assert_eq!(reopened.read_chunk(pos)?, Some(sample(pos))); + Ok(()) + } +} diff --git a/crates/shared/Cargo.toml b/crates/shared/Cargo.toml index c0b416f..8785ddb 100644 --- a/crates/shared/Cargo.toml +++ b/crates/shared/Cargo.toml @@ -13,4 +13,7 @@ bytemuck = { workspace = true } fastrand = "2.4.1" glam = { workspace = true } noise = "0.9" +postcard = { version = "1.1.3", features = ["use-std"] } serde = { version = "1.0", features = ["derive"] } +thiserror = "2.0.18" +zstd = "0.13.3" diff --git a/crates/shared/src/lib.rs b/crates/shared/src/lib.rs index 380c9ed..9d0b525 100644 --- a/crates/shared/src/lib.rs +++ b/crates/shared/src/lib.rs @@ -6,4 +6,5 @@ //! the client and the server. pub mod generator; +pub mod save; pub mod world; diff --git a/crates/shared/src/save.rs b/crates/shared/src/save.rs new file mode 100644 index 0000000..f7304eb --- /dev/null +++ b/crates/shared/src/save.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//! On-disk save format: the framing that persists modified chunks to region files. +//! +//! The format is built bottom-up. The smallest unit is the `SYNC` per-chunk [`record`], which wraps one [`crate::world::ChunkData`] in a self-describing, compressed frame. Region-level framing (the `SYNR` file and its header table) is layered on top of it. All parsing treats on-disk bytes as untrusted and reports failures through [`SaveError`]. + +mod cursor; +mod error; +pub mod record; +pub mod region; + +pub use error::SaveError; diff --git a/crates/shared/src/save/cursor.rs b/crates/shared/src/save/cursor.rs new file mode 100644 index 0000000..eb6d480 --- /dev/null +++ b/crates/shared/src/save/cursor.rs @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//! A bounds-checked, forward-only cursor over untrusted save-format bytes. + +use super::error::SaveError; + +/// A forward-only reader over a byte slice that bounds-checks every read. +pub(crate) struct Reader<'a> { + /// The full buffer being read. + bytes: &'a [u8], + /// The offset of the next unread byte. + offset: usize, +} + +impl<'a> Reader<'a> { + pub(crate) fn new(bytes: &'a [u8]) -> Self { + Self { bytes, offset: 0 } + } + + /// Returns the next `n` bytes and advances the cursor, or [`SaveError::Truncated`] if fewer remain. + pub(crate) fn take(&mut self, n: usize) -> Result<&'a [u8], SaveError> { + let end = self.offset.checked_add(n).ok_or(SaveError::Truncated { + offset: self.offset, + needed: n, + available: self.bytes.len().saturating_sub(self.offset), + })?; + let slice = self + .bytes + .get(self.offset..end) + .ok_or(SaveError::Truncated { + offset: self.offset, + needed: n, + available: self.bytes.len().saturating_sub(self.offset), + })?; + self.offset = end; + Ok(slice) + } + + /// Returns the next `N` bytes as a fixed-size array and advances the cursor. + pub(crate) fn take_array(&mut self) -> Result<[u8; N], SaveError> { + let mut array = [0u8; N]; + array.copy_from_slice(self.take(N)?); + Ok(array) + } +} diff --git a/crates/shared/src/save/error.rs b/crates/shared/src/save/error.rs new file mode 100644 index 0000000..92522a2 --- /dev/null +++ b/crates/shared/src/save/error.rs @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//! Error types for reading and writing the on-disk save format. + +use thiserror::Error; + +/// A failure encountered while encoding or decoding a save-format record. +#[derive(Debug, Error)] +pub enum SaveError { + /// The leading magic bytes did not match the expected record tag. + #[error("record magic mismatch: expected {expected:?}, found {found:?}")] + BadMagic { + /// The magic the record was expected to begin with. + expected: [u8; 4], + /// The magic actually found at the start of the buffer. + found: [u8; 4], + }, + + /// The buffer ended before a field of the declared size could be read. + #[error("record truncated at offset {offset}: needed {needed} bytes, {available} available")] + Truncated { + /// The byte offset at which the read was attempted. + offset: usize, + /// The number of bytes the field required. + needed: usize, + /// The number of bytes actually remaining from `offset`. + available: usize, + }, + + /// The decompressed payload length did not match the length the header declared. + #[error("payload length mismatch: header declared {expected} bytes, decompressed {actual}")] + LengthMismatch { + /// The uncompressed length recorded in the header. + expected: usize, + /// The length actually produced by decompression. + actual: usize, + }, + + /// The payload was too large for its length to fit the 32-bit header field. + #[error("payload too large to frame: {len} bytes exceeds the u32 length field")] + PayloadTooLarge { + /// The oversized payload length in bytes. + len: usize, + }, + + /// The record or region declared a format version this build does not support. + #[error("unsupported format version {found}, expected {expected}")] + UnsupportedVersion { + /// The format version this build writes and can read. + expected: u32, + /// The format version actually found in the header. + found: u32, + }, + + /// The payload could not be (de)serialized by `postcard`. + #[error("payload serialization failed")] + Serialization(#[from] postcard::Error), + + /// Compression or decompression failed at the I/O layer (`zstd`). + #[error("payload compression failed")] + Compression(#[from] std::io::Error), +} diff --git a/crates/shared/src/save/record.rs b/crates/shared/src/save/record.rs new file mode 100644 index 0000000..5d43e48 --- /dev/null +++ b/crates/shared/src/save/record.rs @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//! The `SYNC` per-chunk record: the on-disk framing around one [`ChunkData`]. +//! +//! A record is a fixed-size header followed by a zstd-compressed, postcard-serialized [`ChunkData`] payload. All multi-byte integers are little-endian. The framing itself is never compressed, so a repair tool can read the header without decompressing. + +use super::cursor::Reader; +use super::error::SaveError; +use crate::world::ChunkData; + +/// The magic tag every chunk record begins with. +const MAGIC: [u8; 4] = *b"SYNC"; + +/// The current schema version of the `ChunkData` payload, written into every new record. +pub const CHUNK_FORMAT_VERSION: u16 = 1; + +/// The zstd compression level used for chunk payloads: level 3 favours speed, per the save-format design. +const ZSTD_LEVEL: i32 = 3; + +/// The size in bytes of the fixed record header: magic + version + flags + timestamp + two length fields. +const HEADER_LEN: usize = 4 + 2 + 2 + 8 + 4 + 4; + +/// The non-payload header fields of a decoded record. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RecordMeta { + /// The schema version the payload was written under, used to drive per-chunk migration on load. + pub chunk_format_version: u16, + /// Reserved record flags; currently always zero. + pub flags: u16, + /// The wall-clock time the chunk was last modified, in milliseconds since the Unix epoch. + pub last_modified: u64, +} + +/// Encodes `data` into a `SYNC` record, stamping it with `last_modified` (unix-ms). +pub fn encode(data: &ChunkData, last_modified: u64) -> Result, SaveError> { + let uncompressed = postcard::to_stdvec(data)?; + let compressed = zstd::encode_all(uncompressed.as_slice(), ZSTD_LEVEL)?; + + // Checked conversion so an oversized payload fails loudly instead of truncating. + let uncompressed_len = + u32::try_from(uncompressed.len()).map_err(|_| SaveError::PayloadTooLarge { + len: uncompressed.len(), + })?; + let compressed_len = + u32::try_from(compressed.len()).map_err(|_| SaveError::PayloadTooLarge { + len: compressed.len(), + })?; + + let mut out = Vec::with_capacity(HEADER_LEN + compressed.len()); + out.extend_from_slice(&MAGIC); + out.extend_from_slice(&CHUNK_FORMAT_VERSION.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&last_modified.to_le_bytes()); + out.extend_from_slice(&uncompressed_len.to_le_bytes()); + out.extend_from_slice(&compressed_len.to_le_bytes()); + out.extend_from_slice(&compressed); + Ok(out) +} + +/// Decodes a `SYNC` record, returning its header metadata and the reconstructed [`ChunkData`]. +/// +/// `bytes` is untrusted on-disk input, so every field is bounds-checked and the decompressed +/// payload length is validated against the header before deserialization is attempted. +pub fn decode(bytes: &[u8]) -> Result<(RecordMeta, ChunkData), SaveError> { + let mut reader = Reader::new(bytes); + + let magic = reader.take_array::<4>()?; + if magic != MAGIC { + return Err(SaveError::BadMagic { + expected: MAGIC, + found: magic, + }); + } + + let chunk_format_version = u16::from_le_bytes(reader.take_array()?); + let flags = u16::from_le_bytes(reader.take_array()?); + let last_modified = u64::from_le_bytes(reader.take_array()?); + // Widening u32 -> usize is lossless on every supported (64-bit) target. + let uncompressed_len = u32::from_le_bytes(reader.take_array()?) as usize; + let compressed_len = u32::from_le_bytes(reader.take_array()?) as usize; + + let payload = reader.take(compressed_len)?; + let decompressed = zstd::decode_all(payload)?; + if decompressed.len() != uncompressed_len { + return Err(SaveError::LengthMismatch { + expected: uncompressed_len, + actual: decompressed.len(), + }); + } + + let data: ChunkData = postcard::from_bytes(&decompressed)?; + let meta = RecordMeta { + chunk_format_version, + flags, + last_modified, + }; + Ok((meta, data)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::world::{BlockId, ChunkPos}; + + /// Builds a representative modified chunk with a few edits spanning the local index range. + fn sample() -> ChunkData { + let mut data = ChunkData::new(ChunkPos::new(1, -2, 3), 7); + data.set(0, BlockId(4)); + data.set(1000, BlockId(9)); + data.set(32_767, BlockId(2)); + data + } + + #[test] + fn round_trips_payload_and_metadata() -> Result<(), SaveError> { + let data = sample(); + let bytes = encode(&data, 123_456)?; + let (meta, decoded) = decode(&bytes)?; + + assert_eq!(decoded, data); + assert_eq!(meta.chunk_format_version, CHUNK_FORMAT_VERSION); + assert_eq!(meta.flags, 0); + assert_eq!(meta.last_modified, 123_456); + Ok(()) + } + + #[test] + fn rejects_bad_magic() -> Result<(), SaveError> { + let mut bytes = encode(&sample(), 0)?; + bytes[0] = b'X'; + assert!(matches!(decode(&bytes), Err(SaveError::BadMagic { .. }))); + Ok(()) + } + + #[test] + fn rejects_truncated_header() -> Result<(), SaveError> { + let bytes = encode(&sample(), 0)?; + // A buffer shorter than the fixed header cannot yield a full record. + assert!(matches!( + decode(&bytes[..HEADER_LEN - 1]), + Err(SaveError::Truncated { .. }) + )); + Ok(()) + } + + #[test] + fn rejects_truncated_payload() -> Result<(), SaveError> { + let bytes = encode(&sample(), 0)?; + // Keep the whole header but cut the compressed payload short. + assert!(matches!( + decode(&bytes[..=HEADER_LEN]), + Err(SaveError::Truncated { .. }) + )); + Ok(()) + } + + #[test] + fn detects_declared_length_mismatch() -> Result<(), SaveError> { + let mut bytes = encode(&sample(), 0)?; + // The uncompressed-length field is the u32 at offset 16 (after magic, version, + // flags, and the timestamp). Overwriting it with a value the payload cannot + // decompress to must be caught by the post-decompression length check. + bytes[16..20].copy_from_slice(&1u32.to_le_bytes()); + assert!(matches!( + decode(&bytes), + Err(SaveError::LengthMismatch { .. }) + )); + Ok(()) + } +} diff --git a/crates/shared/src/save/region.rs b/crates/shared/src/save/region.rs new file mode 100644 index 0000000..ae7b78c --- /dev/null +++ b/crates/shared/src/save/region.rs @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//! The `SYNR` region index: the header table and side tables that frame a region file. +//! +//! A region file is this fixed header and its three tables, followed by the `SYNC` chunk +//! records the header table points at. This module owns only the index; the record bytes +//! and their placement are managed by the durability layer. All integers are little-endian. + +use std::collections::BTreeMap; + +use super::cursor::Reader; +use super::error::SaveError; +use crate::world::ChunkPos; + +/// The magic tag every region file begins with. +const MAGIC: [u8; 4] = *b"SYNR"; + +/// The current region-file framing version, written on encode and checked on decode. +pub const REGION_FORMAT_VERSION: u32 = 1; + +/// The location and flags of one chunk record within the region file. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct HeaderEntry { + /// The byte offset of the record from the start of the region file. + pub offset: u64, + /// The length of the record in bytes. + pub length: u32, + /// Record flags; currently always zero. + pub flags: u32, +} + +/// A reclaimable span of free space left by a removed or shrunken record. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FreeSpan { + /// The byte offset of the free span from the start of the region file. + pub offset: u64, + /// The length of the free span in bytes. + pub length: u32, +} + +/// The in-memory index of a region file: where every resident chunk record lives, the free spans between them, and the worldgen-version exceptions. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RegionIndex { + /// Monotonic cache-coherence counter for derived LOD tiles; only ever increases. + region_tile_version: u32, + /// The worldgen version most chunks in the region are pinned to; the stamp table holds only exceptions. + base_worldgen_version: u32, + /// Location of every resident chunk record, keyed by chunk position. + header_table: BTreeMap, + /// Reclaimable holes in the file, in no particular order. + free_list: Vec, + /// Worldgen-version exceptions: chunks pinned to a version other than `base_worldgen_version`. + stamps: BTreeMap, +} + +impl RegionIndex { + /// Creates an empty index whose chunks default to `base_worldgen_version`. + #[must_use] + pub fn new(base_worldgen_version: u32) -> Self { + Self { + region_tile_version: 0, + base_worldgen_version, + header_table: BTreeMap::new(), + free_list: Vec::new(), + stamps: BTreeMap::new(), + } + } + + /// Returns the monotonic LOD-tile version. + #[must_use] + pub fn region_tile_version(&self) -> u32 { + self.region_tile_version + } + + /// Increments the LOD-tile version; called once per save that commits a rebuild-triggering change. + pub fn bump_tile_version(&mut self) { + self.region_tile_version = self.region_tile_version.saturating_add(1); + } + + /// Returns the region's base worldgen version. + #[must_use] + pub fn base_worldgen_version(&self) -> u32 { + self.base_worldgen_version + } + + /// Looks up the record location for `pos`, if the chunk is resident. + #[must_use] + pub fn entry(&self, pos: ChunkPos) -> Option<&HeaderEntry> { + self.header_table.get(&pos) + } + + /// Records the location of the chunk at `pos`, replacing any existing entry. + pub fn insert(&mut self, pos: ChunkPos, entry: HeaderEntry) { + self.header_table.insert(pos, entry); + } + + /// Removes the chunk at `pos` from the header table, returning its former location. + pub fn remove(&mut self, pos: ChunkPos) -> Option { + self.header_table.remove(&pos) + } + + /// Iterates the resident records in ascending position order. + pub fn entries(&self) -> impl Iterator { + self.header_table.iter() + } + + /// Returns the pinned worldgen version for `pos`: its stamp exception, or the region base. + #[must_use] + pub fn worldgen_version(&self, pos: ChunkPos) -> u32 { + self.stamps + .get(&pos) + .copied() + .unwrap_or(self.base_worldgen_version) + } + + /// Pins `pos` to `version`, recording it as a stamp exception only when it differs from the base. + pub fn set_worldgen_version(&mut self, pos: ChunkPos, version: u32) { + if version == self.base_worldgen_version { + self.stamps.remove(&pos); + } else { + self.stamps.insert(pos, version); + } + } + + /// Returns the reclaimable free spans. + #[must_use] + pub fn free_spans(&self) -> &[FreeSpan] { + &self.free_list + } + + /// Adds a reclaimable free span. + pub fn push_free(&mut self, span: FreeSpan) { + self.free_list.push(span); + } + + /// Serializes the index to its on-disk framing bytes. + pub fn encode(&self) -> Result, SaveError> { + let mut out = Vec::new(); + out.extend_from_slice(&MAGIC); + out.extend_from_slice(®ION_FORMAT_VERSION.to_le_bytes()); + out.extend_from_slice(&self.base_worldgen_version.to_le_bytes()); + out.extend_from_slice(&self.region_tile_version.to_le_bytes()); + + out.extend_from_slice(&len_u32(self.header_table.len())?.to_le_bytes()); + for (pos, entry) in &self.header_table { + out.extend_from_slice(&pos.x.to_le_bytes()); + out.extend_from_slice(&pos.y.to_le_bytes()); + out.extend_from_slice(&pos.z.to_le_bytes()); + out.extend_from_slice(&entry.offset.to_le_bytes()); + out.extend_from_slice(&entry.length.to_le_bytes()); + out.extend_from_slice(&entry.flags.to_le_bytes()); + } + + out.extend_from_slice(&len_u32(self.free_list.len())?.to_le_bytes()); + for span in &self.free_list { + out.extend_from_slice(&span.offset.to_le_bytes()); + out.extend_from_slice(&span.length.to_le_bytes()); + } + + // The stamp value is widened to u32 to match ChunkData's u32 worldgen version and avoid + // truncation; the spec's u16 stamp field (Save format.md) is treated as an oversight. + out.extend_from_slice(&len_u32(self.stamps.len())?.to_le_bytes()); + for (pos, version) in &self.stamps { + out.extend_from_slice(&pos.x.to_le_bytes()); + out.extend_from_slice(&pos.y.to_le_bytes()); + out.extend_from_slice(&pos.z.to_le_bytes()); + out.extend_from_slice(&version.to_le_bytes()); + } + + Ok(out) + } + + /// Parses a region index from its framing bytes, ignoring any chunk records that follow it. + pub fn decode(bytes: &[u8]) -> Result { + let mut reader = Reader::new(bytes); + + let magic = reader.take_array::<4>()?; + if magic != MAGIC { + return Err(SaveError::BadMagic { + expected: MAGIC, + found: magic, + }); + } + + let format_version = u32::from_le_bytes(reader.take_array()?); + if format_version != REGION_FORMAT_VERSION { + return Err(SaveError::UnsupportedVersion { + expected: REGION_FORMAT_VERSION, + found: format_version, + }); + } + + let base_worldgen_version = u32::from_le_bytes(reader.take_array()?); + let region_tile_version = u32::from_le_bytes(reader.take_array()?); + + let header_len = u32::from_le_bytes(reader.take_array()?); + let mut header_table = BTreeMap::new(); + for _ in 0..header_len { + let pos = read_pos(&mut reader)?; + let entry = HeaderEntry { + offset: u64::from_le_bytes(reader.take_array()?), + length: u32::from_le_bytes(reader.take_array()?), + flags: u32::from_le_bytes(reader.take_array()?), + }; + header_table.insert(pos, entry); + } + + let free_len = u32::from_le_bytes(reader.take_array()?); + let mut free_list = Vec::with_capacity(free_len as usize); + for _ in 0..free_len { + free_list.push(FreeSpan { + offset: u64::from_le_bytes(reader.take_array()?), + length: u32::from_le_bytes(reader.take_array()?), + }); + } + + let stamp_len = u32::from_le_bytes(reader.take_array()?); + let mut stamps = BTreeMap::new(); + for _ in 0..stamp_len { + let pos = read_pos(&mut reader)?; + stamps.insert(pos, u32::from_le_bytes(reader.take_array()?)); + } + + Ok(Self { + region_tile_version, + base_worldgen_version, + header_table, + free_list, + stamps, + }) + } +} + +/// Reads a chunk position as three little-endian `i32`s. +fn read_pos(reader: &mut Reader) -> Result { + let x = i32::from_le_bytes(reader.take_array()?); + let y = i32::from_le_bytes(reader.take_array()?); + let z = i32::from_le_bytes(reader.take_array()?); + Ok(ChunkPos::new(x, y, z)) +} + +/// Narrows a table length to the `u32` the framing uses, failing loudly rather than truncating. +fn len_u32(len: usize) -> Result { + u32::try_from(len).map_err(|_| SaveError::PayloadTooLarge { len }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Builds a populated index with entries at varied positions, a free span, and a stamp exception. + fn sample() -> RegionIndex { + let mut index = RegionIndex::new(3); + index.insert( + ChunkPos::new(0, 0, 0), + HeaderEntry { + offset: 4096, + length: 128, + flags: 0, + }, + ); + index.insert( + ChunkPos::new(-5, 12, -30), + HeaderEntry { + offset: 8192, + length: 256, + flags: 0, + }, + ); + index.push_free(FreeSpan { + offset: 512, + length: 64, + }); + index.set_worldgen_version(ChunkPos::new(0, 0, 0), 2); + index.bump_tile_version(); + index + } + + #[test] + fn round_trips_index() -> Result<(), SaveError> { + let index = sample(); + let decoded = RegionIndex::decode(&index.encode()?)?; + assert_eq!(decoded, index); + Ok(()) + } + + #[test] + fn preserves_worldgen_versions_above_u16_max() -> Result<(), SaveError> { + // Regression guard: base and stamp worldgen versions are u32, so a value that would not + // fit a u16 must survive encode -> decode without truncation. + let mut index = RegionIndex::new(70_000); + index.set_worldgen_version(ChunkPos::new(0, 0, 0), 100_000); + let decoded = RegionIndex::decode(&index.encode()?)?; + assert_eq!(decoded.base_worldgen_version(), 70_000); + assert_eq!(decoded.worldgen_version(ChunkPos::new(0, 0, 0)), 100_000); + Ok(()) + } + + #[test] + fn stamp_equal_to_base_is_not_recorded() { + let mut index = RegionIndex::new(7); + // A stamp equal to the base is redundant, so no exception entry is stored. + index.set_worldgen_version(ChunkPos::new(1, 1, 1), 7); + assert_eq!(index.worldgen_version(ChunkPos::new(1, 1, 1)), 7); + assert!(index.stamps.is_empty()); + } + + #[test] + fn rejects_bad_magic() -> Result<(), SaveError> { + let mut bytes = sample().encode()?; + bytes[0] = b'X'; + assert!(matches!( + RegionIndex::decode(&bytes), + Err(SaveError::BadMagic { .. }) + )); + Ok(()) + } + + #[test] + fn rejects_unsupported_version() -> Result<(), SaveError> { + let mut bytes = sample().encode()?; + // The format_version u32 sits just after the 4 magic bytes. + bytes[4..8].copy_from_slice(&999u32.to_le_bytes()); + assert!(matches!( + RegionIndex::decode(&bytes), + Err(SaveError::UnsupportedVersion { .. }) + )); + Ok(()) + } + + #[test] + fn rejects_truncated_table() -> Result<(), SaveError> { + let bytes = sample().encode()?; + // Cut the buffer mid-header-table so an entry read runs off the end. + assert!(matches!( + RegionIndex::decode(&bytes[..20]), + Err(SaveError::Truncated { .. }) + )); + Ok(()) + } +} diff --git a/crates/shared/src/world/mod.rs b/crates/shared/src/world.rs similarity index 100% rename from crates/shared/src/world/mod.rs rename to crates/shared/src/world.rs diff --git a/crates/shared/src/world/coords.rs b/crates/shared/src/world/coords.rs index afbeb87..e526da7 100644 --- a/crates/shared/src/world/coords.rs +++ b/crates/shared/src/world/coords.rs @@ -6,7 +6,7 @@ use super::CHUNK_SIZE; use serde::{Deserialize, Serialize}; /// The three-dimensional spatial coordinates of a chunk in the world. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub struct ChunkPos { /// The X coordinate of the chunk. pub x: i32,