CryoforgeNexus/Synvael
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3 from Cryoforge-Nexus/feat/region-io

Browse source 
feat(shared): region file save format and durability layer
This commit is contained in:
Serkyo 2026-07-10 00:59:18 +02:00 committed by GitHub
parent cf1adf7133 085f496898
commit 3a634c717d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 1144 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

141
Cargo.lock generated
View file

@ -143,6 +143,15 @@ dependencies = [
"portable-atomic",
]
[[package]]
name = "atomic-polyfill"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4"
dependencies = [
"critical-section",
]
[[package]]
name = "atomic-waker"
version = "1.1.2"
@ -238,7 +247,7 @@ dependencies = [
"portable-atomic",
"portable-atomic-util",
"serde",
"spin",
"spin 0.10.0",
"wasm-bindgen",
"wasm-bindgen-futures",
"windows-sys 0.61.2",
@ -304,7 +313,7 @@ dependencies = [
"crossbeam-queue",
"derive_more",
"futures-lite",
"heapless",
"heapless 0.9.3",
"web-task",
]
@ -454,6 +463,15 @@ dependencies = [
"winit",
]
[[package]]
name = "cobs"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1"
dependencies = [
"thiserror 2.0.18",
]
[[package]]
name = "cocoa"
version = "0.25.0"
@ -651,6 +669,18 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76"
[[package]]
name = "embedded-io"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced"
[[package]]
name = "embedded-io"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d"
[[package]]
name = "equivalent"
version = "1.0.2"
@ -848,6 +878,15 @@ dependencies = [
"windows",
]
[[package]]
name = "hash32"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67"
dependencies = [
"byteorder",
]
[[package]]
name = "hash32"
version = "0.3.1"
@ -874,13 +913,27 @@ version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51"
[[package]]
name = "heapless"
version = "0.7.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f"
dependencies = [
"atomic-polyfill",
"hash32 0.2.1",
"rustc_version",
"serde",
"spin 0.9.8",
"stable_deref_trait",
]
[[package]]
name = "heapless"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4"
dependencies = [
"hash32",
"hash32 0.3.1",
"portable-atomic",
"stable_deref_trait",
]
@ -1033,6 +1086,15 @@ version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
[[package]]
name = "lock_api"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
dependencies = [
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.29"
@ -1505,6 +1567,19 @@ dependencies = [
"portable-atomic",
]
[[package]]
name = "postcard"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24"
dependencies = [
"cobs",
"embedded-io 0.4.0",
"embedded-io 0.6.1",
"heapless 0.7.17",
"serde",
]
[[package]]
name = "presser"
version = "0.3.1"
@ -1706,6 +1781,12 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "scripting"
version = "0.1.0"
@ -1782,6 +1863,7 @@ dependencies = [
"glam 0.33.2",
"serde_json",
"shared",
"tempfile",
"tracing",
"tracing-subscriber",
]
@ -1803,7 +1885,10 @@ dependencies = [
"fastrand",
"glam 0.33.2",
"noise",
"postcard",
"serde",
"thiserror 2.0.18",
"zstd",
]
[[package]]
@ -1883,6 +1968,15 @@ dependencies = [
"serde",
]
[[package]]
name = "spin"
version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
dependencies = [
"lock_api",
]
[[package]]
name = "spin"
version = "0.10.0"
@ -1915,6 +2009,19 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "tempfile"
version = "3.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.4.3",
"once_cell",
"rustix 1.1.4",
"windows-sys 0.61.2",
]
[[package]]
name = "thiserror"
version = "1.0.69"
@ -2734,3 +2841,31 @@ name = "zmij"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
[[package]]
name = "zstd"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "7.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.16+zstd.1.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
dependencies = [
"cc",
"pkg-config",
]

3
crates/server/Cargo.toml
View file

@ -17,3 +17,6 @@ serde_json = { workspace = true }
shared = { version = "0.1.0", path = "../shared" }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
[dev-dependencies]
tempfile = "3.27.0"

2
crates/server/src/main.rs
View file

@ -7,6 +7,8 @@
/// Entity components describing players and other world-streaming anchors.
pub mod player;
/// On-disk persistence: region files and the atomic durability layer.
pub mod save;
/// Authoritative chunk storage and generation logic for the server.
pub mod world_server;

13
crates/server/src/save.rs Normal file
View file

@ -0,0 +1,13 @@
// SPDX-License-Identifier: AGPL-3.0-only
//! Server-side persistence: the durability layer over the `shared` save format.
//!
//! `shared::save` owns the pure, in-memory framing (the `SYNR` index and `SYNC` records). This
//! module owns the filesystem side: reading a `.region` file into memory, mutating its chunks, and
//! flushing it back to disk crash-safely. The write strategy is a whole-file atomic rewrite
//! (`.tmp` + fsync + rename); the on-disk format is unchanged, so a later slice can switch to an
//! append-in-place strategy without a format change.
mod region_file;
pub use region_file::{REGION_SIZE, RegionFile, region_coords, region_path};

353
crates/server/src/save/region_file.rs Normal file
View file

@ -0,0 +1,353 @@
// SPDX-License-Identifier: AGPL-3.0-only
//! The durability layer for a single region file.
use std::collections::BTreeMap;
use std::ffi::OsString;
use std::fs::{self, File};
use std::io::Write;
use std::path::{Path, PathBuf};
use shared::save::SaveError;
use shared::save::record;
use shared::save::region::{HeaderEntry, RegionIndex};
use shared::world::{ChunkData, ChunkPos};
/// The side length, in chunk columns, of the square footprint one region file covers.
pub const REGION_SIZE: i32 = 32;
/// Maps a chunk column `(cx, cz)` to the coordinates `(rx, rz)` of the region that contains it.
#[must_use]
pub fn region_coords(cx: i32, cz: i32) -> (i32, i32) {
(cx.div_euclid(REGION_SIZE), cz.div_euclid(REGION_SIZE))
}
/// Builds the on-disk path of the region file containing chunk column `(cx, cz)` within `dir`.
#[must_use]
pub fn region_path(dir: &Path, cx: i32, cz: i32) -> PathBuf {
let (rx, rz) = region_coords(cx, cz);
dir.join(format!("r.{rx}.{rz}.region"))
}
/// An open region file: its `SYNR` index, the resident chunk records, and its on-disk location.
pub struct RegionFile {
/// The region framing (header table, free list, stamp table) held in memory.
index: RegionIndex,
/// Each resident chunk's raw `SYNC` record bytes, decoded lazily on read.
records: BTreeMap<ChunkPos, Vec<u8>>,
/// The path this region is read from and written back to.
path: PathBuf,
/// Whether an in-memory mutation is pending a flush to disk.
dirty: bool,
}
impl RegionFile {
/// Opens the region file at `path`, or yields an empty region if the file does not yet exist.
pub fn open(path: PathBuf) -> Result<Self, SaveError> {
if !path.exists() {
return Ok(Self {
index: RegionIndex::new(0),
records: BTreeMap::new(),
path,
dirty: false,
});
}
let bytes = fs::read(&path)?;
let index = RegionIndex::decode(&bytes)?;
// Slice each record out of the file by the absolute (offset, length) the header table records. On-disk bytes are untrusted, so an out-of-range span is rejected rather than panicking on the slice.
let mut records = BTreeMap::new();
for (pos, entry) in index.entries() {
// Widening u32 -> usize is lossless on every supported (64-bit) target; the u64 offset is range-checked by try_from, failing loudly on a 32-bit target rather than wrapping.
let start = usize::try_from(entry.offset).map_err(|_| SaveError::PayloadTooLarge {
len: entry.length as usize,
})?;
let end =
start
.checked_add(entry.length as usize)
.ok_or(SaveError::PayloadTooLarge {
len: entry.length as usize,
})?;
let record_bytes = bytes
.get(start..end)
.ok_or(SaveError::Truncated {
offset: start,
needed: entry.length as usize,
available: bytes.len().saturating_sub(start),
})?
.to_vec();
records.insert(*pos, record_bytes);
}
Ok(Self {
index,
records,
path,
dirty: false,
})
}
/// Returns the number of resident chunk records.
#[must_use]
pub fn len(&self) -> usize {
self.records.len()
}
/// Returns whether the region holds no chunk records.
#[must_use]
pub fn is_empty(&self) -> bool {
self.records.is_empty()
}
/// Whether an in-memory mutation is pending a flush to disk.
#[must_use]
pub fn is_dirty(&self) -> bool {
self.dirty
}
/// Decodes and returns the chunk at `pos`, or `None` if the region holds no record for it.
pub fn read_chunk(&self, pos: ChunkPos) -> Result<Option<ChunkData>, SaveError> {
match self.records.get(&pos) {
Some(bytes) => {
let (_meta, data) = record::decode(bytes)?;
Ok(Some(data))
}
None => Ok(None),
}
}
/// Encodes `data` into a `SYNC` record stamped with `last_modified` and stores it under `pos`.
pub fn write_chunk(
&mut self,
pos: ChunkPos,
data: &ChunkData,
last_modified: u64,
) -> Result<(), SaveError> {
let bytes = record::encode(data, last_modified)?;
let length = u32::try_from(bytes.len())
.map_err(|_| SaveError::PayloadTooLarge { len: bytes.len() })?;
self.records.insert(pos, bytes);
self.index.insert(
pos,
HeaderEntry {
offset: 0,
length,
flags: 0,
},
);
self.dirty = true;
Ok(())
}
/// Removes the chunk at `pos` from the region, if present.
pub fn remove_chunk(&mut self, pos: ChunkPos) {
let removed = self.records.remove(&pos).is_some();
self.index.remove(pos);
if removed {
self.dirty = true;
}
}
/// Flushes the region to disk with a crash-safe whole-file atomic rewrite, clearing the dirty flag.
pub fn save(&mut self) -> Result<(), SaveError> {
let image = self.serialize()?;
atomic_write(&self.path, &image)?;
self.dirty = false;
Ok(())
}
/// Builds the complete on-disk file image: the encoded index followed by every record.
fn serialize(&mut self) -> Result<Vec<u8>, SaveError> {
let index_len = self.index.encode()?.len();
// Assign each record a contiguous offset in ascending position order (BTreeMap order), the same order the records are concatenated below.
let mut offset = index_len as u64;
for (pos, bytes) in &self.records {
let length = u32::try_from(bytes.len())
.map_err(|_| SaveError::PayloadTooLarge { len: bytes.len() })?;
self.index.insert(
*pos,
HeaderEntry {
offset,
length,
flags: 0,
},
);
offset += bytes.len() as u64;
}
let mut image = self.index.encode()?;
for bytes in self.records.values() {
image.extend_from_slice(bytes);
}
Ok(image)
}
}
/// Writes `bytes` to `path` via the POSIX atomic-write pattern: `.tmp` + fsync + rename.
fn atomic_write(path: &Path, bytes: &[u8]) -> Result<(), SaveError> {
// The region directory is created on demand so the first write to a fresh world succeeds.
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)?;
}
// The temp path appends ".tmp" to the full filename (rather than replacing the extension) so it sits beside the target on the same filesystem, keeping the rename atomic.
let mut tmp_name: OsString = path.as_os_str().to_owned();
tmp_name.push(".tmp");
let tmp_path = PathBuf::from(tmp_name);
let mut file = File::create(&tmp_path)?;
file.write_all(bytes)?;
// fsync the data to disk before the rename, so the rename cannot expose an unwritten file.
file.sync_all()?;
drop(file);
fs::rename(&tmp_path, path)?;
// fsync the parent directory so the rename itself is durable. Opening a directory for fsync is a Unix affordance; Windows does not expose a directory handle to sync, so the step is skipped there.
#[cfg(unix)]
if let Some(parent) = path.parent() {
File::open(parent)?.sync_all()?;
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use shared::world::BlockId;
/// Builds a representative modified chunk with a few edits spanning the local index range.
fn sample(pos: ChunkPos) -> ChunkData {
let mut data = ChunkData::new(pos, 7);
data.set(0, BlockId(4));
data.set(1000, BlockId(9));
data.set(32_767, BlockId(2));
data
}
#[test]
fn region_coords_floor_negative_columns() {
// Truncating division would map -1 to region 0; Euclidean flooring maps it to region -1.
assert_eq!(region_coords(0, 0), (0, 0));
assert_eq!(region_coords(31, 31), (0, 0));
assert_eq!(region_coords(-1, -1), (-1, -1));
assert_eq!(region_coords(-32, -33), (-1, -2));
}
#[test]
fn region_path_names_the_region_file() {
let dir = Path::new("/saves/world/region");
assert_eq!(
region_path(dir, -1, 5),
Path::new("/saves/world/region/r.-1.0.region")
);
}
#[test]
fn open_missing_file_is_empty() -> Result<(), SaveError> {
let dir = tempfile::tempdir()?;
let region = RegionFile::open(dir.path().join("r.0.0.region"))?;
assert!(region.is_empty());
assert_eq!(region.read_chunk(ChunkPos::new(0, 0, 0))?, None);
Ok(())
}
#[test]
fn round_trips_chunks_through_disk() -> Result<(), SaveError> {
let dir = tempfile::tempdir()?;
let path = dir.path().join("r.0.0.region");
let positions = [
ChunkPos::new(0, 0, 0),
ChunkPos::new(1, 2, 3),
ChunkPos::new(-5, 10, -30),
];
let mut region = RegionFile::open(path.clone())?;
for pos in positions {
region.write_chunk(pos, &sample(pos), 123)?;
}
assert!(region.is_dirty());
region.save()?;
assert!(!region.is_dirty());
// Reopen from disk in a fresh instance and confirm every chunk decodes byte-identically.
let reopened = RegionFile::open(path)?;
assert_eq!(reopened.len(), positions.len());
for pos in positions {
assert_eq!(reopened.read_chunk(pos)?, Some(sample(pos)));
}
// A position never written has no record.
assert_eq!(reopened.read_chunk(ChunkPos::new(9, 9, 9))?, None);
Ok(())
}
#[test]
fn record_offsets_are_valid_and_contiguous() -> Result<(), SaveError> {
let dir = tempfile::tempdir()?;
let path = dir.path().join("r.0.0.region");
let mut region = RegionFile::open(path.clone())?;
for pos in [
ChunkPos::new(0, 0, 0),
ChunkPos::new(2, 0, 1),
ChunkPos::new(-1, 4, -1),
] {
region.write_chunk(pos, &sample(pos), 0)?;
}
region.save()?;
let reopened = RegionFile::open(path)?;
let index_len = reopened.index.encode()?.len() as u64;
// Records are packed contiguously immediately after the index, in ascending position order.
let mut expected_offset = index_len;
for (_pos, entry) in reopened.index.entries() {
assert_eq!(entry.offset, expected_offset);
expected_offset += u64::from(entry.length);
}
Ok(())
}
#[test]
fn remove_drops_only_the_named_chunk() -> Result<(), SaveError> {
let dir = tempfile::tempdir()?;
let path = dir.path().join("r.0.0.region");
let kept = ChunkPos::new(0, 0, 0);
let dropped = ChunkPos::new(1, 1, 1);
let mut region = RegionFile::open(path.clone())?;
region.write_chunk(kept, &sample(kept), 0)?;
region.write_chunk(dropped, &sample(dropped), 0)?;
region.save()?;
let mut region = RegionFile::open(path.clone())?;
region.remove_chunk(dropped);
region.save()?;
let reopened = RegionFile::open(path)?;
assert_eq!(reopened.read_chunk(dropped)?, None);
assert_eq!(reopened.read_chunk(kept)?, Some(sample(kept)));
Ok(())
}
#[test]
fn stray_tmp_file_does_not_corrupt_reads() -> Result<(), SaveError> {
let dir = tempfile::tempdir()?;
let path = dir.path().join("r.0.0.region");
let pos = ChunkPos::new(0, 0, 0);
let mut region = RegionFile::open(path.clone())?;
region.write_chunk(pos, &sample(pos), 0)?;
region.save()?;
// A leftover .tmp from an interrupted save must be ignored: only the renamed target is read.
fs::write(dir.path().join("r.0.0.region.tmp"), b"garbage")?;
let reopened = RegionFile::open(path)?;
assert_eq!(reopened.read_chunk(pos)?, Some(sample(pos)));
Ok(())
}
}

3
crates/shared/Cargo.toml
View file

@ -13,4 +13,7 @@ bytemuck = { workspace = true }
fastrand = "2.4.1"
glam = { workspace = true }
noise = "0.9"
postcard = { version = "1.1.3", features = ["use-std"] }
serde = { version = "1.0", features = ["derive"] }
thiserror = "2.0.18"
zstd = "0.13.3"

1
crates/shared/src/lib.rs
View file

@ -6,4 +6,5 @@
//! the client and the server.
pub mod generator;
pub mod save;
pub mod world;

12
crates/shared/src/save.rs Normal file
View file

@ -0,0 +1,12 @@
// SPDX-License-Identifier: AGPL-3.0-only
//! On-disk save format: the framing that persists modified chunks to region files.
//!
//! The format is built bottom-up. The smallest unit is the `SYNC` per-chunk [`record`], which wraps one [`crate::world::ChunkData`] in a self-describing, compressed frame. Region-level framing (the `SYNR` file and its header table) is layered on top of it. All parsing treats on-disk bytes as untrusted and reports failures through [`SaveError`].
mod cursor;
mod error;
pub mod record;
pub mod region;
pub use error::SaveError;

45
crates/shared/src/save/cursor.rs Normal file
View file

@ -0,0 +1,45 @@
// SPDX-License-Identifier: AGPL-3.0-only
//! A bounds-checked, forward-only cursor over untrusted save-format bytes.
use super::error::SaveError;
/// A forward-only reader over a byte slice that bounds-checks every read.
pub(crate) struct Reader<'a> {
/// The full buffer being read.
bytes: &'a [u8],
/// The offset of the next unread byte.
offset: usize,
}
impl<'a> Reader<'a> {
pub(crate) fn new(bytes: &'a [u8]) -> Self {
Self { bytes, offset: 0 }
}
/// Returns the next `n` bytes and advances the cursor, or [`SaveError::Truncated`] if fewer remain.
pub(crate) fn take(&mut self, n: usize) -> Result<&'a [u8], SaveError> {
let end = self.offset.checked_add(n).ok_or(SaveError::Truncated {
offset: self.offset,
needed: n,
available: self.bytes.len().saturating_sub(self.offset),
})?;
let slice = self
.bytes
.get(self.offset..end)
.ok_or(SaveError::Truncated {
offset: self.offset,
needed: n,
available: self.bytes.len().saturating_sub(self.offset),
})?;
self.offset = end;
Ok(slice)
}
/// Returns the next `N` bytes as a fixed-size array and advances the cursor.
pub(crate) fn take_array<const N: usize>(&mut self) -> Result<[u8; N], SaveError> {
let mut array = [0u8; N];
array.copy_from_slice(self.take(N)?);
Ok(array)
}
}

62
crates/shared/src/save/error.rs Normal file
View file

@ -0,0 +1,62 @@
// SPDX-License-Identifier: AGPL-3.0-only
//! Error types for reading and writing the on-disk save format.
use thiserror::Error;
/// A failure encountered while encoding or decoding a save-format record.
#[derive(Debug, Error)]
pub enum SaveError {
/// The leading magic bytes did not match the expected record tag.
#[error("record magic mismatch: expected {expected:?}, found {found:?}")]
BadMagic {
/// The magic the record was expected to begin with.
expected: [u8; 4],
/// The magic actually found at the start of the buffer.
found: [u8; 4],
},
/// The buffer ended before a field of the declared size could be read.
#[error("record truncated at offset {offset}: needed {needed} bytes, {available} available")]
Truncated {
/// The byte offset at which the read was attempted.
offset: usize,
/// The number of bytes the field required.
needed: usize,
/// The number of bytes actually remaining from `offset`.
available: usize,
},
/// The decompressed payload length did not match the length the header declared.
#[error("payload length mismatch: header declared {expected} bytes, decompressed {actual}")]
LengthMismatch {
/// The uncompressed length recorded in the header.
expected: usize,
/// The length actually produced by decompression.
actual: usize,
},
/// The payload was too large for its length to fit the 32-bit header field.
#[error("payload too large to frame: {len} bytes exceeds the u32 length field")]
PayloadTooLarge {
/// The oversized payload length in bytes.
len: usize,
},
/// The record or region declared a format version this build does not support.
#[error("unsupported format version {found}, expected {expected}")]
UnsupportedVersion {
/// The format version this build writes and can read.
expected: u32,
/// The format version actually found in the header.
found: u32,
},
/// The payload could not be (de)serialized by `postcard`.
#[error("payload serialization failed")]
Serialization(#[from] postcard::Error),
/// Compression or decompression failed at the I/O layer (`zstd`).
#[error("payload compression failed")]
Compression(#[from] std::io::Error),
}

170
crates/shared/src/save/record.rs Normal file
View file

@ -0,0 +1,170 @@
// SPDX-License-Identifier: AGPL-3.0-only
//! The `SYNC` per-chunk record: the on-disk framing around one [`ChunkData`].
//!
//! A record is a fixed-size header followed by a zstd-compressed, postcard-serialized [`ChunkData`] payload. All multi-byte integers are little-endian. The framing itself is never compressed, so a repair tool can read the header without decompressing.
use super::cursor::Reader;
use super::error::SaveError;
use crate::world::ChunkData;
/// The magic tag every chunk record begins with.
const MAGIC: [u8; 4] = *b"SYNC";
/// The current schema version of the `ChunkData` payload, written into every new record.
pub const CHUNK_FORMAT_VERSION: u16 = 1;
/// The zstd compression level used for chunk payloads: level 3 favours speed, per the save-format design.
const ZSTD_LEVEL: i32 = 3;
/// The size in bytes of the fixed record header: magic + version + flags + timestamp + two length fields.
const HEADER_LEN: usize = 4 + 2 + 2 + 8 + 4 + 4;
/// The non-payload header fields of a decoded record.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct RecordMeta {
/// The schema version the payload was written under, used to drive per-chunk migration on load.
pub chunk_format_version: u16,
/// Reserved record flags; currently always zero.
pub flags: u16,
/// The wall-clock time the chunk was last modified, in milliseconds since the Unix epoch.
pub last_modified: u64,
}
/// Encodes `data` into a `SYNC` record, stamping it with `last_modified` (unix-ms).
pub fn encode(data: &ChunkData, last_modified: u64) -> Result<Vec<u8>, SaveError> {
let uncompressed = postcard::to_stdvec(data)?;
let compressed = zstd::encode_all(uncompressed.as_slice(), ZSTD_LEVEL)?;
// Checked conversion so an oversized payload fails loudly instead of truncating.
let uncompressed_len =
u32::try_from(uncompressed.len()).map_err(|_| SaveError::PayloadTooLarge {
len: uncompressed.len(),
})?;
let compressed_len =
u32::try_from(compressed.len()).map_err(|_| SaveError::PayloadTooLarge {
len: compressed.len(),
})?;
let mut out = Vec::with_capacity(HEADER_LEN + compressed.len());
out.extend_from_slice(&MAGIC);
out.extend_from_slice(&CHUNK_FORMAT_VERSION.to_le_bytes());
out.extend_from_slice(&0u16.to_le_bytes());
out.extend_from_slice(&last_modified.to_le_bytes());
out.extend_from_slice(&uncompressed_len.to_le_bytes());
out.extend_from_slice(&compressed_len.to_le_bytes());
out.extend_from_slice(&compressed);
Ok(out)
}
/// Decodes a `SYNC` record, returning its header metadata and the reconstructed [`ChunkData`].
///
/// `bytes` is untrusted on-disk input, so every field is bounds-checked and the decompressed
/// payload length is validated against the header before deserialization is attempted.
pub fn decode(bytes: &[u8]) -> Result<(RecordMeta, ChunkData), SaveError> {
let mut reader = Reader::new(bytes);
let magic = reader.take_array::<4>()?;
if magic != MAGIC {
return Err(SaveError::BadMagic {
expected: MAGIC,
found: magic,
});
}
let chunk_format_version = u16::from_le_bytes(reader.take_array()?);
let flags = u16::from_le_bytes(reader.take_array()?);
let last_modified = u64::from_le_bytes(reader.take_array()?);
// Widening u32 -> usize is lossless on every supported (64-bit) target.
let uncompressed_len = u32::from_le_bytes(reader.take_array()?) as usize;
let compressed_len = u32::from_le_bytes(reader.take_array()?) as usize;
let payload = reader.take(compressed_len)?;
let decompressed = zstd::decode_all(payload)?;
if decompressed.len() != uncompressed_len {
return Err(SaveError::LengthMismatch {
expected: uncompressed_len,
actual: decompressed.len(),
});
}
let data: ChunkData = postcard::from_bytes(&decompressed)?;
let meta = RecordMeta {
chunk_format_version,
flags,
last_modified,
};
Ok((meta, data))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::world::{BlockId, ChunkPos};
/// Builds a representative modified chunk with a few edits spanning the local index range.
fn sample() -> ChunkData {
let mut data = ChunkData::new(ChunkPos::new(1, -2, 3), 7);
data.set(0, BlockId(4));
data.set(1000, BlockId(9));
data.set(32_767, BlockId(2));
data
}
#[test]
fn round_trips_payload_and_metadata() -> Result<(), SaveError> {
let data = sample();
let bytes = encode(&data, 123_456)?;
let (meta, decoded) = decode(&bytes)?;
assert_eq!(decoded, data);
assert_eq!(meta.chunk_format_version, CHUNK_FORMAT_VERSION);
assert_eq!(meta.flags, 0);
assert_eq!(meta.last_modified, 123_456);
Ok(())
}
#[test]
fn rejects_bad_magic() -> Result<(), SaveError> {
let mut bytes = encode(&sample(), 0)?;
bytes[0] = b'X';
assert!(matches!(decode(&bytes), Err(SaveError::BadMagic { .. })));
Ok(())
}
#[test]
fn rejects_truncated_header() -> Result<(), SaveError> {
let bytes = encode(&sample(), 0)?;
// A buffer shorter than the fixed header cannot yield a full record.
assert!(matches!(
decode(&bytes[..HEADER_LEN - 1]),
Err(SaveError::Truncated { .. })
));
Ok(())
}
#[test]
fn rejects_truncated_payload() -> Result<(), SaveError> {
let bytes = encode(&sample(), 0)?;
// Keep the whole header but cut the compressed payload short.
assert!(matches!(
decode(&bytes[..=HEADER_LEN]),
Err(SaveError::Truncated { .. })
));
Ok(())
}
#[test]
fn detects_declared_length_mismatch() -> Result<(), SaveError> {
let mut bytes = encode(&sample(), 0)?;
// The uncompressed-length field is the u32 at offset 16 (after magic, version,
// flags, and the timestamp). Overwriting it with a value the payload cannot
// decompress to must be caught by the post-decompression length check.
bytes[16..20].copy_from_slice(&1u32.to_le_bytes());
assert!(matches!(
decode(&bytes),
Err(SaveError::LengthMismatch { .. })
));
Ok(())
}
}

341
crates/shared/src/save/region.rs Normal file
View file

@ -0,0 +1,341 @@
// SPDX-License-Identifier: AGPL-3.0-only
//! The `SYNR` region index: the header table and side tables that frame a region file.
//!
//! A region file is this fixed header and its three tables, followed by the `SYNC` chunk
//! records the header table points at. This module owns only the index; the record bytes
//! and their placement are managed by the durability layer. All integers are little-endian.
use std::collections::BTreeMap;
use super::cursor::Reader;
use super::error::SaveError;
use crate::world::ChunkPos;
/// The magic tag every region file begins with.
const MAGIC: [u8; 4] = *b"SYNR";
/// The current region-file framing version, written on encode and checked on decode.
pub const REGION_FORMAT_VERSION: u32 = 1;
/// The location and flags of one chunk record within the region file.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct HeaderEntry {
/// The byte offset of the record from the start of the region file.
pub offset: u64,
/// The length of the record in bytes.
pub length: u32,
/// Record flags; currently always zero.
pub flags: u32,
}
/// A reclaimable span of free space left by a removed or shrunken record.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct FreeSpan {
/// The byte offset of the free span from the start of the region file.
pub offset: u64,
/// The length of the free span in bytes.
pub length: u32,
}
/// The in-memory index of a region file: where every resident chunk record lives, the free spans between them, and the worldgen-version exceptions.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RegionIndex {
/// Monotonic cache-coherence counter for derived LOD tiles; only ever increases.
region_tile_version: u32,
/// The worldgen version most chunks in the region are pinned to; the stamp table holds only exceptions.
base_worldgen_version: u32,
/// Location of every resident chunk record, keyed by chunk position.
header_table: BTreeMap<ChunkPos, HeaderEntry>,
/// Reclaimable holes in the file, in no particular order.
free_list: Vec<FreeSpan>,
/// Worldgen-version exceptions: chunks pinned to a version other than `base_worldgen_version`.
stamps: BTreeMap<ChunkPos, u32>,
}
impl RegionIndex {
/// Creates an empty index whose chunks default to `base_worldgen_version`.
#[must_use]
pub fn new(base_worldgen_version: u32) -> Self {
Self {
region_tile_version: 0,
base_worldgen_version,
header_table: BTreeMap::new(),
free_list: Vec::new(),
stamps: BTreeMap::new(),
}
}
/// Returns the monotonic LOD-tile version.
#[must_use]
pub fn region_tile_version(&self) -> u32 {
self.region_tile_version
}
/// Increments the LOD-tile version; called once per save that commits a rebuild-triggering change.
pub fn bump_tile_version(&mut self) {
self.region_tile_version = self.region_tile_version.saturating_add(1);
}
/// Returns the region's base worldgen version.
#[must_use]
pub fn base_worldgen_version(&self) -> u32 {
self.base_worldgen_version
}
/// Looks up the record location for `pos`, if the chunk is resident.
#[must_use]
pub fn entry(&self, pos: ChunkPos) -> Option<&HeaderEntry> {
self.header_table.get(&pos)
}
/// Records the location of the chunk at `pos`, replacing any existing entry.
pub fn insert(&mut self, pos: ChunkPos, entry: HeaderEntry) {
self.header_table.insert(pos, entry);
}
/// Removes the chunk at `pos` from the header table, returning its former location.
pub fn remove(&mut self, pos: ChunkPos) -> Option<HeaderEntry> {
self.header_table.remove(&pos)
}
/// Iterates the resident records in ascending position order.
pub fn entries(&self) -> impl Iterator<Item = (&ChunkPos, &HeaderEntry)> {
self.header_table.iter()
}
/// Returns the pinned worldgen version for `pos`: its stamp exception, or the region base.
#[must_use]
pub fn worldgen_version(&self, pos: ChunkPos) -> u32 {
self.stamps
.get(&pos)
.copied()
.unwrap_or(self.base_worldgen_version)
}
/// Pins `pos` to `version`, recording it as a stamp exception only when it differs from the base.
pub fn set_worldgen_version(&mut self, pos: ChunkPos, version: u32) {
if version == self.base_worldgen_version {
self.stamps.remove(&pos);
} else {
self.stamps.insert(pos, version);
}
}
/// Returns the reclaimable free spans.
#[must_use]
pub fn free_spans(&self) -> &[FreeSpan] {
&self.free_list
}
/// Adds a reclaimable free span.
pub fn push_free(&mut self, span: FreeSpan) {
self.free_list.push(span);
}
/// Serializes the index to its on-disk framing bytes.
pub fn encode(&self) -> Result<Vec<u8>, SaveError> {
let mut out = Vec::new();
out.extend_from_slice(&MAGIC);
out.extend_from_slice(&REGION_FORMAT_VERSION.to_le_bytes());
out.extend_from_slice(&self.base_worldgen_version.to_le_bytes());
out.extend_from_slice(&self.region_tile_version.to_le_bytes());
out.extend_from_slice(&len_u32(self.header_table.len())?.to_le_bytes());
for (pos, entry) in &self.header_table {
out.extend_from_slice(&pos.x.to_le_bytes());
out.extend_from_slice(&pos.y.to_le_bytes());
out.extend_from_slice(&pos.z.to_le_bytes());
out.extend_from_slice(&entry.offset.to_le_bytes());
out.extend_from_slice(&entry.length.to_le_bytes());
out.extend_from_slice(&entry.flags.to_le_bytes());
}
out.extend_from_slice(&len_u32(self.free_list.len())?.to_le_bytes());
for span in &self.free_list {
out.extend_from_slice(&span.offset.to_le_bytes());
out.extend_from_slice(&span.length.to_le_bytes());
}
// The stamp value is widened to u32 to match ChunkData's u32 worldgen version and avoid
// truncation; the spec's u16 stamp field (Save format.md) is treated as an oversight.
out.extend_from_slice(&len_u32(self.stamps.len())?.to_le_bytes());
for (pos, version) in &self.stamps {
out.extend_from_slice(&pos.x.to_le_bytes());
out.extend_from_slice(&pos.y.to_le_bytes());
out.extend_from_slice(&pos.z.to_le_bytes());
out.extend_from_slice(&version.to_le_bytes());
}
Ok(out)
}
/// Parses a region index from its framing bytes, ignoring any chunk records that follow it.
pub fn decode(bytes: &[u8]) -> Result<Self, SaveError> {
let mut reader = Reader::new(bytes);
let magic = reader.take_array::<4>()?;
if magic != MAGIC {
return Err(SaveError::BadMagic {
expected: MAGIC,
found: magic,
});
}
let format_version = u32::from_le_bytes(reader.take_array()?);
if format_version != REGION_FORMAT_VERSION {
return Err(SaveError::UnsupportedVersion {
expected: REGION_FORMAT_VERSION,
found: format_version,
});
}
let base_worldgen_version = u32::from_le_bytes(reader.take_array()?);
let region_tile_version = u32::from_le_bytes(reader.take_array()?);
let header_len = u32::from_le_bytes(reader.take_array()?);
let mut header_table = BTreeMap::new();
for _ in 0..header_len {
let pos = read_pos(&mut reader)?;
let entry = HeaderEntry {
offset: u64::from_le_bytes(reader.take_array()?),
length: u32::from_le_bytes(reader.take_array()?),
flags: u32::from_le_bytes(reader.take_array()?),
};
header_table.insert(pos, entry);
}
let free_len = u32::from_le_bytes(reader.take_array()?);
let mut free_list = Vec::with_capacity(free_len as usize);
for _ in 0..free_len {
free_list.push(FreeSpan {
offset: u64::from_le_bytes(reader.take_array()?),
length: u32::from_le_bytes(reader.take_array()?),
});
}
let stamp_len = u32::from_le_bytes(reader.take_array()?);
let mut stamps = BTreeMap::new();
for _ in 0..stamp_len {
let pos = read_pos(&mut reader)?;
stamps.insert(pos, u32::from_le_bytes(reader.take_array()?));
}
Ok(Self {
region_tile_version,
base_worldgen_version,
header_table,
free_list,
stamps,
})
}
}
/// Reads a chunk position as three little-endian `i32`s.
fn read_pos(reader: &mut Reader) -> Result<ChunkPos, SaveError> {
let x = i32::from_le_bytes(reader.take_array()?);
let y = i32::from_le_bytes(reader.take_array()?);
let z = i32::from_le_bytes(reader.take_array()?);
Ok(ChunkPos::new(x, y, z))
}
/// Narrows a table length to the `u32` the framing uses, failing loudly rather than truncating.
fn len_u32(len: usize) -> Result<u32, SaveError> {
u32::try_from(len).map_err(|_| SaveError::PayloadTooLarge { len })
}
#[cfg(test)]
mod tests {
use super::*;
/// Builds a populated index with entries at varied positions, a free span, and a stamp exception.
fn sample() -> RegionIndex {
let mut index = RegionIndex::new(3);
index.insert(
ChunkPos::new(0, 0, 0),
HeaderEntry {
offset: 4096,
length: 128,
flags: 0,
},
);
index.insert(
ChunkPos::new(-5, 12, -30),
HeaderEntry {
offset: 8192,
length: 256,
flags: 0,
},
);
index.push_free(FreeSpan {
offset: 512,
length: 64,
});
index.set_worldgen_version(ChunkPos::new(0, 0, 0), 2);
index.bump_tile_version();
index
}
#[test]
fn round_trips_index() -> Result<(), SaveError> {
let index = sample();
let decoded = RegionIndex::decode(&index.encode()?)?;
assert_eq!(decoded, index);
Ok(())
}
#[test]
fn preserves_worldgen_versions_above_u16_max() -> Result<(), SaveError> {
// Regression guard: base and stamp worldgen versions are u32, so a value that would not
// fit a u16 must survive encode -> decode without truncation.
let mut index = RegionIndex::new(70_000);
index.set_worldgen_version(ChunkPos::new(0, 0, 0), 100_000);
let decoded = RegionIndex::decode(&index.encode()?)?;
assert_eq!(decoded.base_worldgen_version(), 70_000);
assert_eq!(decoded.worldgen_version(ChunkPos::new(0, 0, 0)), 100_000);
Ok(())
}
#[test]
fn stamp_equal_to_base_is_not_recorded() {
let mut index = RegionIndex::new(7);
// A stamp equal to the base is redundant, so no exception entry is stored.
index.set_worldgen_version(ChunkPos::new(1, 1, 1), 7);
assert_eq!(index.worldgen_version(ChunkPos::new(1, 1, 1)), 7);
assert!(index.stamps.is_empty());
}
#[test]
fn rejects_bad_magic() -> Result<(), SaveError> {
let mut bytes = sample().encode()?;
bytes[0] = b'X';
assert!(matches!(
RegionIndex::decode(&bytes),
Err(SaveError::BadMagic { .. })
));
Ok(())
}
#[test]
fn rejects_unsupported_version() -> Result<(), SaveError> {
let mut bytes = sample().encode()?;
// The format_version u32 sits just after the 4 magic bytes.
bytes[4..8].copy_from_slice(&999u32.to_le_bytes());
assert!(matches!(
RegionIndex::decode(&bytes),
Err(SaveError::UnsupportedVersion { .. })
));
Ok(())
}
#[test]
fn rejects_truncated_table() -> Result<(), SaveError> {
let bytes = sample().encode()?;
// Cut the buffer mid-header-table so an entry read runs off the end.
assert!(matches!(
RegionIndex::decode(&bytes[..20]),
Err(SaveError::Truncated { .. })
));
Ok(())
}
}

0
crates/shared/src/world/mod.rs → crates/shared/src/world.rs
View file

2
crates/shared/src/world/coords.rs
View file

@ -6,7 +6,7 @@ use super::CHUNK_SIZE;
use serde::{Deserialize, Serialize};
/// The three-dimensional spatial coordinates of a chunk in the world.
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct ChunkPos {
/// The X coordinate of the chunk.
pub x: i32,