CryoforgeNexus/Synvael
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat(shared): add SYNC per-chunk record encoding

Browse source
This commit is contained in:
Serkyo 2026-07-08 16:21:54 +02:00
parent fa7ec0e380
commit b676755633
6 changed files with 400 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

127
Cargo.lock generated
View file

@ -143,6 +143,15 @@ dependencies = [
"portable-atomic",
]
[[package]]
name = "atomic-polyfill"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4"
dependencies = [
"critical-section",
]
[[package]]
name = "atomic-waker"
version = "1.1.2"
@ -238,7 +247,7 @@ dependencies = [
"portable-atomic",
"portable-atomic-util",
"serde",
"spin",
"spin 0.10.0",
"wasm-bindgen",
"wasm-bindgen-futures",
"windows-sys 0.61.2",
@ -304,7 +313,7 @@ dependencies = [
"crossbeam-queue",
"derive_more",
"futures-lite",
"heapless",
"heapless 0.9.3",
"web-task",
]
@ -454,6 +463,15 @@ dependencies = [
"winit",
]
[[package]]
name = "cobs"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1"
dependencies = [
"thiserror 2.0.18",
]
[[package]]
name = "cocoa"
version = "0.25.0"
@ -651,6 +669,18 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76"
[[package]]
name = "embedded-io"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced"
[[package]]
name = "embedded-io"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d"
[[package]]
name = "equivalent"
version = "1.0.2"
@ -848,6 +878,15 @@ dependencies = [
"windows",
]
[[package]]
name = "hash32"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67"
dependencies = [
"byteorder",
]
[[package]]
name = "hash32"
version = "0.3.1"
@ -874,13 +913,27 @@ version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51"
[[package]]
name = "heapless"
version = "0.7.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f"
dependencies = [
"atomic-polyfill",
"hash32 0.2.1",
"rustc_version",
"serde",
"spin 0.9.8",
"stable_deref_trait",
]
[[package]]
name = "heapless"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4"
dependencies = [
"hash32",
"hash32 0.3.1",
"portable-atomic",
"stable_deref_trait",
]
@ -1033,6 +1086,15 @@ version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
[[package]]
name = "lock_api"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
dependencies = [
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.29"
@ -1505,6 +1567,19 @@ dependencies = [
"portable-atomic",
]
[[package]]
name = "postcard"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24"
dependencies = [
"cobs",
"embedded-io 0.4.0",
"embedded-io 0.6.1",
"heapless 0.7.17",
"serde",
]
[[package]]
name = "presser"
version = "0.3.1"
@ -1706,6 +1781,12 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "scripting"
version = "0.1.0"
@ -1803,7 +1884,10 @@ dependencies = [
"fastrand",
"glam 0.33.2",
"noise",
"postcard",
"serde",
"thiserror 2.0.18",
"zstd",
]
[[package]]
@ -1883,6 +1967,15 @@ dependencies = [
"serde",
]
[[package]]
name = "spin"
version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
dependencies = [
"lock_api",
]
[[package]]
name = "spin"
version = "0.10.0"
@ -2734,3 +2827,31 @@ name = "zmij"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
[[package]]
name = "zstd"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "7.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.16+zstd.1.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
dependencies = [
"cc",
"pkg-config",
]

3
crates/shared/Cargo.toml
View file

@ -13,4 +13,7 @@ bytemuck = { workspace = true }
fastrand = "2.4.1"
glam = { workspace = true }
noise = "0.9"
postcard = { version = "1.1.3", features = ["use-std"] }
serde = { version = "1.0", features = ["derive"] }
thiserror = "2.0.18"
zstd = "0.13.3"

1
crates/shared/src/lib.rs
View file

@ -7,3 +7,4 @@
pub mod generator;
pub mod world;
pub mod save;

53
crates/shared/src/save/error.rs Normal file
View file

@ -0,0 +1,53 @@
// SPDX-License-Identifier: AGPL-3.0-only
//! Error types for reading and writing the on-disk save format.
use thiserror::Error;
/// A failure encountered while encoding or decoding a save-format record.
#[derive(Debug, Error)]
pub enum SaveError {
/// The leading magic bytes did not match the expected record tag.
#[error("record magic mismatch: expected {expected:?}, found {found:?}")]
BadMagic {
/// The magic the record was expected to begin with.
expected: [u8; 4],
/// The magic actually found at the start of the buffer.
found: [u8; 4],
},
/// The buffer ended before a field of the declared size could be read.
#[error("record truncated at offset {offset}: needed {needed} bytes, {available} available")]
Truncated {
/// The byte offset at which the read was attempted.
offset: usize,
/// The number of bytes the field required.
needed: usize,
/// The number of bytes actually remaining from `offset`.
available: usize,
},
/// The decompressed payload length did not match the length the header declared.
#[error("payload length mismatch: header declared {expected} bytes, decompressed {actual}")]
LengthMismatch {
/// The uncompressed length recorded in the header.
expected: usize,
/// The length actually produced by decompression.
actual: usize,
},
/// The payload was too large for its length to fit the 32-bit header field.
#[error("payload too large to frame: {len} bytes exceeds the u32 length field")]
PayloadTooLarge {
/// The oversized payload length in bytes.
len: usize,
},
/// The payload could not be (de)serialized by `postcard`.
#[error("payload serialization failed")]
Serialization(#[from] postcard::Error),
/// Compression or decompression failed at the I/O layer (`zstd`).
#[error("payload compression failed")]
Compression(#[from] std::io::Error),
}

10
crates/shared/src/save/mod.rs Normal file
View file

@ -0,0 +1,10 @@
// SPDX-License-Identifier: AGPL-3.0-only
//! On-disk save format: the framing that persists modified chunks to region files.
//!
//! The format is built bottom-up. The smallest unit is the `SYNC` per-chunk [`record`], which wraps one [`crate::world::ChunkData`] in a self-describing, compressed frame. Region-level framing (the `SYNR` file and its header table) is layered on top of it. All parsing treats on-disk bytes as untrusted and reports failures through [`SaveError`].
mod error;
pub mod record;
pub use error::SaveError;

209
crates/shared/src/save/record.rs Normal file
View file

@ -0,0 +1,209 @@
// SPDX-License-Identifier: AGPL-3.0-only
//! The `SYNC` per-chunk record: the on-disk framing around one [`ChunkData`].
//!
//! A record is a fixed-size header followed by a zstd-compressed, postcard-serialized [`ChunkData`] payload. All multi-byte integers are little-endian. The framing itself is never compressed, so a repair tool can read the header without decompressing.
use super::error::SaveError;
use crate::world::ChunkData;
/// The magic tag every chunk record begins with.
const MAGIC: [u8; 4] = *b"SYNC";
/// The current schema version of the `ChunkData` payload, written into every new record.
pub const CHUNK_FORMAT_VERSION: u16 = 1;
/// The zstd compression level used for chunk payloads: level 3 favours speed, per the save-format design.
const ZSTD_LEVEL: i32 = 3;
/// The size in bytes of the fixed record header: magic + version + flags + timestamp + two length fields.
const HEADER_LEN: usize = 4 + 2 + 2 + 8 + 4 + 4;
/// The non-payload header fields of a decoded record.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct RecordMeta {
/// The schema version the payload was written under, used to drive per-chunk migration on load.
pub chunk_format_version: u16,
/// Reserved record flags; currently always zero.
pub flags: u16,
/// The wall-clock time the chunk was last modified, in milliseconds since the Unix epoch.
pub last_modified: u64,
}
/// Encodes `data` into a `SYNC` record, stamping it with `last_modified` (unix-ms).
pub fn encode(data: &ChunkData, last_modified: u64) -> Result<Vec<u8>, SaveError> {
let uncompressed = postcard::to_stdvec(data)?;
let compressed = zstd::encode_all(uncompressed.as_slice(), ZSTD_LEVEL)?;
// Checked conversion so an oversized payload fails loudly instead of truncating.
let uncompressed_len =
u32::try_from(uncompressed.len()).map_err(|_| SaveError::PayloadTooLarge {
len: uncompressed.len(),
})?;
let compressed_len = u32::try_from(compressed.len()).map_err(|_| SaveError::PayloadTooLarge {
len: compressed.len(),
})?;
let mut out = Vec::with_capacity(HEADER_LEN + compressed.len());
out.extend_from_slice(&MAGIC);
out.extend_from_slice(&CHUNK_FORMAT_VERSION.to_le_bytes());
out.extend_from_slice(&0u16.to_le_bytes());
out.extend_from_slice(&last_modified.to_le_bytes());
out.extend_from_slice(&uncompressed_len.to_le_bytes());
out.extend_from_slice(&compressed_len.to_le_bytes());
out.extend_from_slice(&compressed);
Ok(out)
}
/// Decodes a `SYNC` record, returning its header metadata and the reconstructed [`ChunkData`].
///
/// `bytes` is untrusted on-disk input, so every field is bounds-checked and the decompressed
/// payload length is validated against the header before deserialization is attempted.
pub fn decode(bytes: &[u8]) -> Result<(RecordMeta, ChunkData), SaveError> {
let mut reader = Reader::new(bytes);
let magic = reader.take_array::<4>()?;
if magic != MAGIC {
return Err(SaveError::BadMagic {
expected: MAGIC,
found: magic,
});
}
let chunk_format_version = u16::from_le_bytes(reader.take_array()?);
let flags = u16::from_le_bytes(reader.take_array()?);
let last_modified = u64::from_le_bytes(reader.take_array()?);
// Widening u32 -> usize is lossless on every supported (64-bit) target.
let uncompressed_len = u32::from_le_bytes(reader.take_array()?) as usize;
let compressed_len = u32::from_le_bytes(reader.take_array()?) as usize;
let payload = reader.take(compressed_len)?;
let decompressed = zstd::decode_all(payload)?;
if decompressed.len() != uncompressed_len {
return Err(SaveError::LengthMismatch {
expected: uncompressed_len,
actual: decompressed.len(),
});
}
let data: ChunkData = postcard::from_bytes(&decompressed)?;
let meta = RecordMeta {
chunk_format_version,
flags,
last_modified,
};
Ok((meta, data))
}
/// A forward-only cursor over a byte slice that bounds-checks every read.
struct Reader<'a> {
/// The full buffer being read.
bytes: &'a [u8],
/// The offset of the next unread byte.
offset: usize,
}
impl<'a> Reader<'a> {
/// Creates a reader positioned at the start of `bytes`.
fn new(bytes: &'a [u8]) -> Self {
Self { bytes, offset: 0 }
}
/// Returns the next `n` bytes and advances the cursor, or [`SaveError::Truncated`] if fewer remain.
fn take(&mut self, n: usize) -> Result<&'a [u8], SaveError> {
let end = self
.offset
.checked_add(n)
.ok_or(SaveError::Truncated {
offset: self.offset,
needed: n,
available: self.bytes.len().saturating_sub(self.offset),
})?;
let slice = self.bytes.get(self.offset..end).ok_or(SaveError::Truncated {
offset: self.offset,
needed: n,
available: self.bytes.len().saturating_sub(self.offset),
})?;
self.offset = end;
Ok(slice)
}
/// Returns the next `N` bytes as a fixed-size array and advances the cursor.
fn take_array<const N: usize>(&mut self) -> Result<[u8; N], SaveError> {
let mut array = [0u8; N];
array.copy_from_slice(self.take(N)?);
Ok(array)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::world::{BlockId, ChunkPos};
/// Builds a representative modified chunk with a few edits spanning the local index range.
fn sample() -> ChunkData {
let mut data = ChunkData::new(ChunkPos::new(1, -2, 3), 7);
data.set(0, BlockId(4));
data.set(1000, BlockId(9));
data.set(32_767, BlockId(2));
data
}
#[test]
fn round_trips_payload_and_metadata() -> Result<(), SaveError> {
let data = sample();
let bytes = encode(&data, 123_456)?;
let (meta, decoded) = decode(&bytes)?;
assert_eq!(decoded, data);
assert_eq!(meta.chunk_format_version, CHUNK_FORMAT_VERSION);
assert_eq!(meta.flags, 0);
assert_eq!(meta.last_modified, 123_456);
Ok(())
}
#[test]
fn rejects_bad_magic() -> Result<(), SaveError> {
let mut bytes = encode(&sample(), 0)?;
bytes[0] = b'X';
assert!(matches!(decode(&bytes), Err(SaveError::BadMagic { .. })));
Ok(())
}
#[test]
fn rejects_truncated_header() -> Result<(), SaveError> {
let bytes = encode(&sample(), 0)?;
// A buffer shorter than the fixed header cannot yield a full record.
assert!(matches!(
decode(&bytes[..HEADER_LEN - 1]),
Err(SaveError::Truncated { .. })
));
Ok(())
}
#[test]
fn rejects_truncated_payload() -> Result<(), SaveError> {
let bytes = encode(&sample(), 0)?;
// Keep the whole header but cut the compressed payload short.
assert!(matches!(
decode(&bytes[..=HEADER_LEN]),
Err(SaveError::Truncated { .. })
));
Ok(())
}
#[test]
fn detects_declared_length_mismatch() -> Result<(), SaveError> {
let mut bytes = encode(&sample(), 0)?;
// The uncompressed-length field is the u32 at offset 16 (after magic, version,
// flags, and the timestamp). Overwriting it with a value the payload cannot
// decompress to must be caught by the post-decompression length check.
bytes[16..20].copy_from_slice(&1u32.to_le_bytes());
assert!(matches!(
decode(&bytes),
Err(SaveError::LengthMismatch { .. })
));
Ok(())
}
}