feat(shared): add SYNC per-chunk record encoding
This commit is contained in:
parent
fa7ec0e380
commit
b676755633
127
Cargo.lock
generated
127
Cargo.lock
generated
|
|
@ -143,6 +143,15 @@ dependencies = [
|
|||
"portable-atomic",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "atomic-polyfill"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4"
|
||||
dependencies = [
|
||||
"critical-section",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "atomic-waker"
|
||||
version = "1.1.2"
|
||||
|
|
@ -238,7 +247,7 @@ dependencies = [
|
|||
"portable-atomic",
|
||||
"portable-atomic-util",
|
||||
"serde",
|
||||
"spin",
|
||||
"spin 0.10.0",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"windows-sys 0.61.2",
|
||||
|
|
@ -304,7 +313,7 @@ dependencies = [
|
|||
"crossbeam-queue",
|
||||
"derive_more",
|
||||
"futures-lite",
|
||||
"heapless",
|
||||
"heapless 0.9.3",
|
||||
"web-task",
|
||||
]
|
||||
|
||||
|
|
@ -454,6 +463,15 @@ dependencies = [
|
|||
"winit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cobs"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1"
|
||||
dependencies = [
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cocoa"
|
||||
version = "0.25.0"
|
||||
|
|
@ -651,6 +669,18 @@ version = "0.1.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76"
|
||||
|
||||
[[package]]
|
||||
name = "embedded-io"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced"
|
||||
|
||||
[[package]]
|
||||
name = "embedded-io"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d"
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.2"
|
||||
|
|
@ -848,6 +878,15 @@ dependencies = [
|
|||
"windows",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hash32"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hash32"
|
||||
version = "0.3.1"
|
||||
|
|
@ -874,13 +913,27 @@ version = "0.17.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51"
|
||||
|
||||
[[package]]
|
||||
name = "heapless"
|
||||
version = "0.7.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f"
|
||||
dependencies = [
|
||||
"atomic-polyfill",
|
||||
"hash32 0.2.1",
|
||||
"rustc_version",
|
||||
"serde",
|
||||
"spin 0.9.8",
|
||||
"stable_deref_trait",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heapless"
|
||||
version = "0.9.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4"
|
||||
dependencies = [
|
||||
"hash32",
|
||||
"hash32 0.3.1",
|
||||
"portable-atomic",
|
||||
"stable_deref_trait",
|
||||
]
|
||||
|
|
@ -1033,6 +1086,15 @@ version = "0.12.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
|
||||
dependencies = [
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.29"
|
||||
|
|
@ -1505,6 +1567,19 @@ dependencies = [
|
|||
"portable-atomic",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "postcard"
|
||||
version = "1.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24"
|
||||
dependencies = [
|
||||
"cobs",
|
||||
"embedded-io 0.4.0",
|
||||
"embedded-io 0.6.1",
|
||||
"heapless 0.7.17",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "presser"
|
||||
version = "0.3.1"
|
||||
|
|
@ -1706,6 +1781,12 @@ version = "1.0.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "scripting"
|
||||
version = "0.1.0"
|
||||
|
|
@ -1803,7 +1884,10 @@ dependencies = [
|
|||
"fastrand",
|
||||
"glam 0.33.2",
|
||||
"noise",
|
||||
"postcard",
|
||||
"serde",
|
||||
"thiserror 2.0.18",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1883,6 +1967,15 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spin"
|
||||
version = "0.9.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spin"
|
||||
version = "0.10.0"
|
||||
|
|
@ -2734,3 +2827,31 @@ name = "zmij"
|
|||
version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
|
||||
dependencies = [
|
||||
"zstd-safe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-safe"
|
||||
version = "7.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
|
||||
dependencies = [
|
||||
"zstd-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-sys"
|
||||
version = "2.0.16+zstd.1.5.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -13,4 +13,7 @@ bytemuck = { workspace = true }
|
|||
fastrand = "2.4.1"
|
||||
glam = { workspace = true }
|
||||
noise = "0.9"
|
||||
postcard = { version = "1.1.3", features = ["use-std"] }
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
thiserror = "2.0.18"
|
||||
zstd = "0.13.3"
|
||||
|
|
|
|||
|
|
@ -7,3 +7,4 @@
|
|||
|
||||
pub mod generator;
|
||||
pub mod world;
|
||||
pub mod save;
|
||||
|
|
|
|||
53
crates/shared/src/save/error.rs
Normal file
53
crates/shared/src/save/error.rs
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
//! Error types for reading and writing the on-disk save format.
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// A failure encountered while encoding or decoding a save-format record.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum SaveError {
|
||||
/// The leading magic bytes did not match the expected record tag.
|
||||
#[error("record magic mismatch: expected {expected:?}, found {found:?}")]
|
||||
BadMagic {
|
||||
/// The magic the record was expected to begin with.
|
||||
expected: [u8; 4],
|
||||
/// The magic actually found at the start of the buffer.
|
||||
found: [u8; 4],
|
||||
},
|
||||
|
||||
/// The buffer ended before a field of the declared size could be read.
|
||||
#[error("record truncated at offset {offset}: needed {needed} bytes, {available} available")]
|
||||
Truncated {
|
||||
/// The byte offset at which the read was attempted.
|
||||
offset: usize,
|
||||
/// The number of bytes the field required.
|
||||
needed: usize,
|
||||
/// The number of bytes actually remaining from `offset`.
|
||||
available: usize,
|
||||
},
|
||||
|
||||
/// The decompressed payload length did not match the length the header declared.
|
||||
#[error("payload length mismatch: header declared {expected} bytes, decompressed {actual}")]
|
||||
LengthMismatch {
|
||||
/// The uncompressed length recorded in the header.
|
||||
expected: usize,
|
||||
/// The length actually produced by decompression.
|
||||
actual: usize,
|
||||
},
|
||||
|
||||
/// The payload was too large for its length to fit the 32-bit header field.
|
||||
#[error("payload too large to frame: {len} bytes exceeds the u32 length field")]
|
||||
PayloadTooLarge {
|
||||
/// The oversized payload length in bytes.
|
||||
len: usize,
|
||||
},
|
||||
|
||||
/// The payload could not be (de)serialized by `postcard`.
|
||||
#[error("payload serialization failed")]
|
||||
Serialization(#[from] postcard::Error),
|
||||
|
||||
/// Compression or decompression failed at the I/O layer (`zstd`).
|
||||
#[error("payload compression failed")]
|
||||
Compression(#[from] std::io::Error),
|
||||
}
|
||||
10
crates/shared/src/save/mod.rs
Normal file
10
crates/shared/src/save/mod.rs
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
//! On-disk save format: the framing that persists modified chunks to region files.
|
||||
//!
|
||||
//! The format is built bottom-up. The smallest unit is the `SYNC` per-chunk [`record`], which wraps one [`crate::world::ChunkData`] in a self-describing, compressed frame. Region-level framing (the `SYNR` file and its header table) is layered on top of it. All parsing treats on-disk bytes as untrusted and reports failures through [`SaveError`].
|
||||
|
||||
mod error;
|
||||
pub mod record;
|
||||
|
||||
pub use error::SaveError;
|
||||
209
crates/shared/src/save/record.rs
Normal file
209
crates/shared/src/save/record.rs
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
//! The `SYNC` per-chunk record: the on-disk framing around one [`ChunkData`].
|
||||
//!
|
||||
//! A record is a fixed-size header followed by a zstd-compressed, postcard-serialized [`ChunkData`] payload. All multi-byte integers are little-endian. The framing itself is never compressed, so a repair tool can read the header without decompressing.
|
||||
|
||||
use super::error::SaveError;
|
||||
use crate::world::ChunkData;
|
||||
|
||||
/// The magic tag every chunk record begins with.
|
||||
const MAGIC: [u8; 4] = *b"SYNC";
|
||||
|
||||
/// The current schema version of the `ChunkData` payload, written into every new record.
|
||||
pub const CHUNK_FORMAT_VERSION: u16 = 1;
|
||||
|
||||
/// The zstd compression level used for chunk payloads: level 3 favours speed, per the save-format design.
|
||||
const ZSTD_LEVEL: i32 = 3;
|
||||
|
||||
/// The size in bytes of the fixed record header: magic + version + flags + timestamp + two length fields.
|
||||
const HEADER_LEN: usize = 4 + 2 + 2 + 8 + 4 + 4;
|
||||
|
||||
/// The non-payload header fields of a decoded record.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct RecordMeta {
|
||||
/// The schema version the payload was written under, used to drive per-chunk migration on load.
|
||||
pub chunk_format_version: u16,
|
||||
/// Reserved record flags; currently always zero.
|
||||
pub flags: u16,
|
||||
/// The wall-clock time the chunk was last modified, in milliseconds since the Unix epoch.
|
||||
pub last_modified: u64,
|
||||
}
|
||||
|
||||
/// Encodes `data` into a `SYNC` record, stamping it with `last_modified` (unix-ms).
|
||||
pub fn encode(data: &ChunkData, last_modified: u64) -> Result<Vec<u8>, SaveError> {
|
||||
let uncompressed = postcard::to_stdvec(data)?;
|
||||
let compressed = zstd::encode_all(uncompressed.as_slice(), ZSTD_LEVEL)?;
|
||||
|
||||
// Checked conversion so an oversized payload fails loudly instead of truncating.
|
||||
let uncompressed_len =
|
||||
u32::try_from(uncompressed.len()).map_err(|_| SaveError::PayloadTooLarge {
|
||||
len: uncompressed.len(),
|
||||
})?;
|
||||
let compressed_len = u32::try_from(compressed.len()).map_err(|_| SaveError::PayloadTooLarge {
|
||||
len: compressed.len(),
|
||||
})?;
|
||||
|
||||
let mut out = Vec::with_capacity(HEADER_LEN + compressed.len());
|
||||
out.extend_from_slice(&MAGIC);
|
||||
out.extend_from_slice(&CHUNK_FORMAT_VERSION.to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
out.extend_from_slice(&last_modified.to_le_bytes());
|
||||
out.extend_from_slice(&uncompressed_len.to_le_bytes());
|
||||
out.extend_from_slice(&compressed_len.to_le_bytes());
|
||||
out.extend_from_slice(&compressed);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Decodes a `SYNC` record, returning its header metadata and the reconstructed [`ChunkData`].
|
||||
///
|
||||
/// `bytes` is untrusted on-disk input, so every field is bounds-checked and the decompressed
|
||||
/// payload length is validated against the header before deserialization is attempted.
|
||||
pub fn decode(bytes: &[u8]) -> Result<(RecordMeta, ChunkData), SaveError> {
|
||||
let mut reader = Reader::new(bytes);
|
||||
|
||||
let magic = reader.take_array::<4>()?;
|
||||
if magic != MAGIC {
|
||||
return Err(SaveError::BadMagic {
|
||||
expected: MAGIC,
|
||||
found: magic,
|
||||
});
|
||||
}
|
||||
|
||||
let chunk_format_version = u16::from_le_bytes(reader.take_array()?);
|
||||
let flags = u16::from_le_bytes(reader.take_array()?);
|
||||
let last_modified = u64::from_le_bytes(reader.take_array()?);
|
||||
// Widening u32 -> usize is lossless on every supported (64-bit) target.
|
||||
let uncompressed_len = u32::from_le_bytes(reader.take_array()?) as usize;
|
||||
let compressed_len = u32::from_le_bytes(reader.take_array()?) as usize;
|
||||
|
||||
let payload = reader.take(compressed_len)?;
|
||||
let decompressed = zstd::decode_all(payload)?;
|
||||
if decompressed.len() != uncompressed_len {
|
||||
return Err(SaveError::LengthMismatch {
|
||||
expected: uncompressed_len,
|
||||
actual: decompressed.len(),
|
||||
});
|
||||
}
|
||||
|
||||
let data: ChunkData = postcard::from_bytes(&decompressed)?;
|
||||
let meta = RecordMeta {
|
||||
chunk_format_version,
|
||||
flags,
|
||||
last_modified,
|
||||
};
|
||||
Ok((meta, data))
|
||||
}
|
||||
|
||||
/// A forward-only cursor over a byte slice that bounds-checks every read.
|
||||
struct Reader<'a> {
|
||||
/// The full buffer being read.
|
||||
bytes: &'a [u8],
|
||||
/// The offset of the next unread byte.
|
||||
offset: usize,
|
||||
}
|
||||
|
||||
impl<'a> Reader<'a> {
|
||||
/// Creates a reader positioned at the start of `bytes`.
|
||||
fn new(bytes: &'a [u8]) -> Self {
|
||||
Self { bytes, offset: 0 }
|
||||
}
|
||||
|
||||
/// Returns the next `n` bytes and advances the cursor, or [`SaveError::Truncated`] if fewer remain.
|
||||
fn take(&mut self, n: usize) -> Result<&'a [u8], SaveError> {
|
||||
let end = self
|
||||
.offset
|
||||
.checked_add(n)
|
||||
.ok_or(SaveError::Truncated {
|
||||
offset: self.offset,
|
||||
needed: n,
|
||||
available: self.bytes.len().saturating_sub(self.offset),
|
||||
})?;
|
||||
let slice = self.bytes.get(self.offset..end).ok_or(SaveError::Truncated {
|
||||
offset: self.offset,
|
||||
needed: n,
|
||||
available: self.bytes.len().saturating_sub(self.offset),
|
||||
})?;
|
||||
self.offset = end;
|
||||
Ok(slice)
|
||||
}
|
||||
|
||||
/// Returns the next `N` bytes as a fixed-size array and advances the cursor.
|
||||
fn take_array<const N: usize>(&mut self) -> Result<[u8; N], SaveError> {
|
||||
let mut array = [0u8; N];
|
||||
array.copy_from_slice(self.take(N)?);
|
||||
Ok(array)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::world::{BlockId, ChunkPos};
|
||||
|
||||
/// Builds a representative modified chunk with a few edits spanning the local index range.
|
||||
fn sample() -> ChunkData {
|
||||
let mut data = ChunkData::new(ChunkPos::new(1, -2, 3), 7);
|
||||
data.set(0, BlockId(4));
|
||||
data.set(1000, BlockId(9));
|
||||
data.set(32_767, BlockId(2));
|
||||
data
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trips_payload_and_metadata() -> Result<(), SaveError> {
|
||||
let data = sample();
|
||||
let bytes = encode(&data, 123_456)?;
|
||||
let (meta, decoded) = decode(&bytes)?;
|
||||
|
||||
assert_eq!(decoded, data);
|
||||
assert_eq!(meta.chunk_format_version, CHUNK_FORMAT_VERSION);
|
||||
assert_eq!(meta.flags, 0);
|
||||
assert_eq!(meta.last_modified, 123_456);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_bad_magic() -> Result<(), SaveError> {
|
||||
let mut bytes = encode(&sample(), 0)?;
|
||||
bytes[0] = b'X';
|
||||
assert!(matches!(decode(&bytes), Err(SaveError::BadMagic { .. })));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_truncated_header() -> Result<(), SaveError> {
|
||||
let bytes = encode(&sample(), 0)?;
|
||||
// A buffer shorter than the fixed header cannot yield a full record.
|
||||
assert!(matches!(
|
||||
decode(&bytes[..HEADER_LEN - 1]),
|
||||
Err(SaveError::Truncated { .. })
|
||||
));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_truncated_payload() -> Result<(), SaveError> {
|
||||
let bytes = encode(&sample(), 0)?;
|
||||
// Keep the whole header but cut the compressed payload short.
|
||||
assert!(matches!(
|
||||
decode(&bytes[..=HEADER_LEN]),
|
||||
Err(SaveError::Truncated { .. })
|
||||
));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_declared_length_mismatch() -> Result<(), SaveError> {
|
||||
let mut bytes = encode(&sample(), 0)?;
|
||||
// The uncompressed-length field is the u32 at offset 16 (after magic, version,
|
||||
// flags, and the timestamp). Overwriting it with a value the payload cannot
|
||||
// decompress to must be caught by the post-decompression length check.
|
||||
bytes[16..20].copy_from_slice(&1u32.to_le_bytes());
|
||||
assert!(matches!(
|
||||
decode(&bytes),
|
||||
Err(SaveError::LengthMismatch { .. })
|
||||
));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue