606 lines
21 KiB
Rust
606 lines
21 KiB
Rust
// SPDX-License-Identifier: AGPL-3.0-only
|
|
|
|
//! Collection and formatting of the debug statistics panel.
|
|
|
|
use std::fmt::Write as _;
|
|
use std::net::SocketAddr;
|
|
use std::time::{Duration, Instant};
|
|
|
|
use glam::{IVec3, Vec3};
|
|
use renderer::{GpuInfo, MemoryUsage, RenderStats};
|
|
use shared::protocol::authority::ServerStats;
|
|
use shared::session::ServerKind;
|
|
use shared::world::{CHUNK_SIZE, ChunkPos};
|
|
use sysinfo::{Pid, ProcessRefreshKind, ProcessesToUpdate, RefreshKind, System};
|
|
|
|
use crate::chunks::ChunkStats;
|
|
|
|
/// Wall-clock cadence at which a measurement window closes and a panel is emitted.
|
|
pub const STATS_INTERVAL: Duration = Duration::from_secs(1);
|
|
|
|
/// Bytes in one mebibyte, the unit memory figures are reported in.
|
|
const BYTES_PER_MIB: f32 = 1024.0 * 1024.0;
|
|
|
|
/// Converts a byte count to mebibytes for display.
|
|
///
|
|
/// The precision loss is intentional: the result is a display figure rounded to one decimal place, not an accounting quantity.
|
|
#[must_use]
|
|
#[expect(
|
|
clippy::cast_precision_loss,
|
|
reason = "the result is a display figure, not an exact byte count"
|
|
)]
|
|
fn mib(bytes: u64) -> f32 {
|
|
bytes as f32 / BYTES_PER_MIB
|
|
}
|
|
|
|
/// Frame timing aggregated over one measurement window.
|
|
#[derive(Copy, Clone, Debug, PartialEq)]
|
|
pub struct FrameStats {
|
|
/// Frames drawn in the window, expressed per second.
|
|
pub average_fps: f32,
|
|
/// Mean time between frames in the window, in milliseconds.
|
|
pub mean_frame_ms: f32,
|
|
/// Shortest time between frames in the window, in milliseconds.
|
|
pub min_frame_ms: f32,
|
|
/// Longest time between frames in the window, in milliseconds. The figure that exposes stutter a mean conceals.
|
|
pub max_frame_ms: f32,
|
|
/// Frames counted in the window.
|
|
pub frames: u32,
|
|
}
|
|
|
|
/// Accumulates per-frame delta times and closes a measurement window on a fixed cadence.
|
|
#[derive(Debug)]
|
|
pub struct FrameAccumulator {
|
|
/// Instant the current window opened; the window closes once [`STATS_INTERVAL`] has elapsed from here.
|
|
window_start: Instant,
|
|
/// Frames recorded in the current window.
|
|
frames: u32,
|
|
/// Summed delta time of every frame in the current window, in seconds.
|
|
total: f32,
|
|
/// Shortest delta time in the current window, in seconds.
|
|
min: f32,
|
|
/// Longest delta time in the current window, in seconds.
|
|
max: f32,
|
|
}
|
|
|
|
impl FrameAccumulator {
|
|
/// Opens the first measurement window at `now`.
|
|
#[must_use]
|
|
pub fn new(now: Instant) -> Self {
|
|
Self {
|
|
window_start: now,
|
|
frames: 0,
|
|
total: 0.0,
|
|
min: f32::INFINITY,
|
|
max: 0.0,
|
|
}
|
|
}
|
|
|
|
/// Records one frame whose delta time was `dt` seconds.
|
|
pub fn record(&mut self, dt: f32) {
|
|
self.frames = self.frames.saturating_add(1);
|
|
self.total += dt;
|
|
self.min = self.min.min(dt);
|
|
self.max = self.max.max(dt);
|
|
}
|
|
|
|
/// Closes the window and returns its summary once [`STATS_INTERVAL`] has elapsed since it opened, otherwise returns [`None`].
|
|
///
|
|
/// On close the accumulators reset and a fresh window opens at `now`, so windows tile the timeline without gaps or overlap.
|
|
pub fn take_window(&mut self, now: Instant) -> Option<FrameStats> {
|
|
let elapsed = now.saturating_duration_since(self.window_start);
|
|
if elapsed < STATS_INTERVAL {
|
|
return None;
|
|
}
|
|
|
|
let stats = summarise_frames(self.frames, self.total, self.min, self.max, elapsed);
|
|
self.window_start = now;
|
|
self.frames = 0;
|
|
self.total = 0.0;
|
|
self.min = f32::INFINITY;
|
|
self.max = 0.0;
|
|
Some(stats)
|
|
}
|
|
}
|
|
|
|
/// Derives a frame-timing summary from a window's raw accumulators.
|
|
///
|
|
/// Split out from [`FrameAccumulator::take_window`] so the arithmetic is exercisable without driving a clock. A window containing no frames reports zeroes throughout rather than dividing by zero, and its minimum is reported as zero rather than the sentinel infinity the accumulator starts from.
|
|
fn summarise_frames(frames: u32, total: f32, min: f32, max: f32, elapsed: Duration) -> FrameStats {
|
|
if frames == 0 {
|
|
return FrameStats {
|
|
average_fps: 0.0,
|
|
mean_frame_ms: 0.0,
|
|
min_frame_ms: 0.0,
|
|
max_frame_ms: 0.0,
|
|
frames: 0,
|
|
};
|
|
}
|
|
|
|
// Frame counts within a one-second window stay far inside f32's exact-integer range.
|
|
#[expect(
|
|
clippy::cast_precision_loss,
|
|
reason = "frame counts per window stay well within f32's exact-integer range"
|
|
)]
|
|
let count = frames as f32;
|
|
let seconds = elapsed.as_secs_f32();
|
|
|
|
FrameStats {
|
|
// The rate is frames over wall clock, not over summed delta time: the two differ whenever a frame's measured delta excludes time the loop spent elsewhere, and wall clock is the honest denominator.
|
|
average_fps: if seconds > 0.0 { count / seconds } else { 0.0 },
|
|
mean_frame_ms: total / count * 1000.0,
|
|
min_frame_ms: min * 1000.0,
|
|
max_frame_ms: max * 1000.0,
|
|
frames,
|
|
}
|
|
}
|
|
|
|
/// Where the camera is and where it is pointing, in every frame of reference worth reading at once.
|
|
#[derive(Copy, Clone, Debug, PartialEq)]
|
|
pub struct CameraStats {
|
|
/// Continuous world position, in blocks.
|
|
pub position: Vec3,
|
|
/// The block the camera occupies, floored from `position`.
|
|
pub block: IVec3,
|
|
/// The chunk containing that block.
|
|
pub chunk: ChunkPos,
|
|
/// Position within the containing chunk, in the range `0..CHUNK_SIZE` on each axis.
|
|
pub local: IVec3,
|
|
/// Cardinal direction the camera faces, from the dominant horizontal component of its forward vector.
|
|
pub facing: &'static str,
|
|
/// Signed axis matching `facing`, for readers who think in axes rather than compass points.
|
|
pub facing_axis: &'static str,
|
|
/// Camera yaw, in degrees.
|
|
pub yaw_degrees: f32,
|
|
/// Camera pitch, in degrees.
|
|
pub pitch_degrees: f32,
|
|
/// Magnitude of the camera's movement over the last frame, in blocks per second.
|
|
pub speed: f32,
|
|
}
|
|
|
|
/// Maps a forward vector to the cardinal direction and signed axis it points along.
|
|
///
|
|
/// Only the horizontal components are considered; pitch does not change which way the camera faces on the compass.
|
|
#[must_use]
|
|
fn facing_for(forward: Vec3) -> (&'static str, &'static str) {
|
|
if forward.x.abs() > forward.z.abs() {
|
|
if forward.x > 0.0 {
|
|
("east", "+X")
|
|
} else {
|
|
("west", "-X")
|
|
}
|
|
} else if forward.z > 0.0 {
|
|
("south", "+Z")
|
|
} else {
|
|
("north", "-Z")
|
|
}
|
|
}
|
|
|
|
/// Derives the camera figures from a position, orientation, and the distance covered since the previous frame.
|
|
///
|
|
/// `dt` is the previous frame's delta time in seconds; a zero or negative value yields a reported speed of zero rather than a division by zero.
|
|
#[must_use]
|
|
pub fn camera_stats(
|
|
position: Vec3,
|
|
forward: Vec3,
|
|
yaw: f32,
|
|
pitch: f32,
|
|
travelled: Vec3,
|
|
dt: f32,
|
|
) -> CameraStats {
|
|
// Flooring rather than truncating: a position of -0.5 lies in block -1, and truncation would place it in block 0.
|
|
let block = position.floor().as_ivec3();
|
|
let chunk = ChunkPos::from_world(
|
|
f64::from(position.x),
|
|
f64::from(position.y),
|
|
f64::from(position.z),
|
|
);
|
|
// The chunk edge is a compile-time constant of 32, so the narrowing cast is exact.
|
|
#[expect(
|
|
clippy::cast_possible_truncation,
|
|
clippy::cast_possible_wrap,
|
|
reason = "CHUNK_SIZE is a small compile-time constant"
|
|
)]
|
|
let size = CHUNK_SIZE as i32;
|
|
let local = IVec3::new(
|
|
block.x.rem_euclid(size),
|
|
block.y.rem_euclid(size),
|
|
block.z.rem_euclid(size),
|
|
);
|
|
let (facing, facing_axis) = facing_for(forward);
|
|
|
|
CameraStats {
|
|
position,
|
|
block,
|
|
chunk,
|
|
local,
|
|
facing,
|
|
facing_axis,
|
|
yaw_degrees: yaw.to_degrees(),
|
|
pitch_degrees: pitch.to_degrees(),
|
|
speed: if dt > 0.0 {
|
|
travelled.length() / dt
|
|
} else {
|
|
0.0
|
|
},
|
|
}
|
|
}
|
|
|
|
/// Facts about the machine and process that do not change while the client runs.
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct HostInfo {
|
|
/// Brand string of the first CPU the system reports.
|
|
pub cpu_brand: String,
|
|
/// Logical cores visible to the process.
|
|
pub logical_cores: usize,
|
|
/// Operating system name and version.
|
|
pub os: String,
|
|
/// Kernel version string.
|
|
pub kernel: String,
|
|
/// Version of this client binary, from the crate manifest.
|
|
pub client_build: &'static str,
|
|
}
|
|
|
|
/// Host and process figures that change from window to window.
|
|
#[derive(Copy, Clone, Debug, PartialEq)]
|
|
pub struct HostUsage {
|
|
/// Share of one core's worth of time this process consumed, in percent. Exceeds 100 on a process using more than one core.
|
|
pub process_cpu_percent: f32,
|
|
/// Resident set size of this process, in bytes.
|
|
pub process_memory_bytes: u64,
|
|
/// Virtual address space reserved by this process, in bytes.
|
|
pub process_virtual_bytes: u64,
|
|
/// Total physical memory installed, in bytes.
|
|
pub system_total_bytes: u64,
|
|
/// Physical memory available for allocation, in bytes.
|
|
pub system_available_bytes: u64,
|
|
/// Current clock of the first CPU the system reports, in MHz.
|
|
pub cpu_frequency_mhz: u64,
|
|
}
|
|
|
|
/// Owns the `sysinfo` handle and reads host figures on the panel's cadence.
|
|
///
|
|
/// Construction is expensive and the per-window refresh is deliberately narrow: only this process's entry and the CPU are refreshed, never the full system enumeration. The handle is therefore built once and kept for the lifetime of the client.
|
|
pub struct HostMonitor {
|
|
/// The `sysinfo` view of the machine, refreshed selectively.
|
|
system: System,
|
|
/// Identifier of this process, resolved once at construction.
|
|
pid: Pid,
|
|
/// Immutable facts read once at construction.
|
|
info: HostInfo,
|
|
}
|
|
|
|
impl HostMonitor {
|
|
/// Builds the monitor, reading the immutable host facts once.
|
|
#[must_use]
|
|
pub fn new() -> Self {
|
|
let mut system = System::new_with_specifics(
|
|
RefreshKind::nothing()
|
|
.with_cpu(sysinfo::CpuRefreshKind::everything())
|
|
.with_memory(sysinfo::MemoryRefreshKind::everything()),
|
|
);
|
|
system.refresh_processes(ProcessesToUpdate::All, true);
|
|
|
|
let info = HostInfo {
|
|
cpu_brand: system
|
|
.cpus()
|
|
.first()
|
|
.map_or_else(|| "unknown".to_owned(), |cpu| cpu.brand().trim().to_owned()),
|
|
logical_cores: system.cpus().len(),
|
|
os: System::long_os_version().unwrap_or_else(|| "unknown".to_owned()),
|
|
kernel: System::kernel_version().unwrap_or_else(|| "unknown".to_owned()),
|
|
client_build: env!("CARGO_PKG_VERSION"),
|
|
};
|
|
|
|
Self {
|
|
system,
|
|
pid: sysinfo::get_current_pid().unwrap_or_else(|_| Pid::from(0)),
|
|
info,
|
|
}
|
|
}
|
|
|
|
/// Returns the immutable host facts.
|
|
#[must_use]
|
|
pub const fn info(&self) -> &HostInfo {
|
|
&self.info
|
|
}
|
|
|
|
/// Refreshes and returns the changing host and process figures.
|
|
pub fn usage(&mut self) -> HostUsage {
|
|
self.system.refresh_cpu_usage();
|
|
self.system.refresh_memory();
|
|
self.system.refresh_processes_specifics(
|
|
ProcessesToUpdate::Some(&[self.pid]),
|
|
true,
|
|
ProcessRefreshKind::nothing().with_cpu().with_memory(),
|
|
);
|
|
|
|
let process = self.system.process(self.pid);
|
|
HostUsage {
|
|
process_cpu_percent: process.map_or(0.0, sysinfo::Process::cpu_usage),
|
|
process_memory_bytes: process.map_or(0, sysinfo::Process::memory),
|
|
process_virtual_bytes: process.map_or(0, sysinfo::Process::virtual_memory),
|
|
system_total_bytes: self.system.total_memory(),
|
|
system_available_bytes: self.system.available_memory(),
|
|
cpu_frequency_mhz: self
|
|
.system
|
|
.cpus()
|
|
.first()
|
|
.map_or(0, sysinfo::Cpu::frequency),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for HostMonitor {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
/// Who the client is playing against, assembled from the address dialled and the handshake reply.
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct ServerIdentity {
|
|
/// Whether the server is integrated, local, or remote. Decided client-side; see [`ServerKind`].
|
|
pub kind: ServerKind,
|
|
/// The address the client dialled.
|
|
pub address: SocketAddr,
|
|
/// Build string the server reported in the handshake.
|
|
pub server_build: String,
|
|
/// Protocol version the two peers agreed on.
|
|
pub protocol_version: u32,
|
|
/// Nominal tick rate the server advertised, in Hz. Compare against the measured rate in [`ServerStats`].
|
|
pub tick_rate_hint: u16,
|
|
}
|
|
|
|
/// One window's worth of statistics from every source, ready to format.
|
|
///
|
|
/// Fields are [`Option`] wherever the source may not exist yet: before the renderer is initialised, before the handshake completes, or before the server has pushed its first report.
|
|
pub struct Snapshot<'a> {
|
|
/// Frame timing measured by the client over the window.
|
|
pub frame: FrameStats,
|
|
/// Camera position and orientation at the end of the window.
|
|
pub camera: CameraStats,
|
|
/// Chunk streaming state, owned by [`crate::chunks`].
|
|
pub chunks: Option<ChunkStats>,
|
|
/// What the renderer submitted on its most recent frame.
|
|
pub render: Option<RenderStats>,
|
|
/// The physical device the renderer selected.
|
|
pub gpu: Option<&'a GpuInfo>,
|
|
/// Live device memory figures.
|
|
pub memory: Option<MemoryUsage>,
|
|
/// Transport counters and QUIC path statistics.
|
|
pub net: Option<net::NetStats>,
|
|
/// The server's own most recent report, delivered over the authority stream.
|
|
pub server: Option<ServerStats>,
|
|
/// Session identity, present once the handshake has completed.
|
|
pub identity: Option<&'a ServerIdentity>,
|
|
/// Immutable host facts.
|
|
pub host: &'a HostInfo,
|
|
/// Host and process figures for this window.
|
|
pub usage: HostUsage,
|
|
}
|
|
|
|
/// Renders a snapshot as a multi-line panel.
|
|
///
|
|
/// Emission goes through a single `tracing` event rather than many, so the panel arrives as one cohesive block rather than interleaved with concurrent output from other threads. Sections whose source is absent are omitted entirely rather than printed as placeholders.
|
|
#[must_use]
|
|
#[expect(
|
|
clippy::too_many_lines,
|
|
reason = "a formatter is one statement per reported field; splitting it would only scatter the layout"
|
|
)]
|
|
pub fn format_panel(snapshot: &Snapshot) -> String {
|
|
let mut out = String::with_capacity(2048);
|
|
let camera = &snapshot.camera;
|
|
let frame = &snapshot.frame;
|
|
|
|
// `write!` into a String cannot fail, so the results are discarded rather than propagated.
|
|
let _ = writeln!(out, "── debug statistics ──");
|
|
let _ = writeln!(
|
|
out,
|
|
"frame {:.1} fps mean {:.2} ms min {:.2} ms max {:.2} ms ({} frames)",
|
|
frame.average_fps,
|
|
frame.mean_frame_ms,
|
|
frame.min_frame_ms,
|
|
frame.max_frame_ms,
|
|
frame.frames
|
|
);
|
|
let _ = writeln!(
|
|
out,
|
|
"pos {:.2} {:.2} {:.2} block {} {} {} speed {:.2} b/s",
|
|
camera.position.x,
|
|
camera.position.y,
|
|
camera.position.z,
|
|
camera.block.x,
|
|
camera.block.y,
|
|
camera.block.z,
|
|
camera.speed
|
|
);
|
|
let _ = writeln!(
|
|
out,
|
|
"chunk {} {} {} local {} {} {} facing {} ({}) yaw {:.1} pitch {:.1}",
|
|
camera.chunk.x,
|
|
camera.chunk.y,
|
|
camera.chunk.z,
|
|
camera.local.x,
|
|
camera.local.y,
|
|
camera.local.z,
|
|
camera.facing,
|
|
camera.facing_axis,
|
|
camera.yaw_degrees,
|
|
camera.pitch_degrees
|
|
);
|
|
|
|
if let Some(chunks) = snapshot.chunks {
|
|
let _ = writeln!(
|
|
out,
|
|
"chunks resident {} / desired {} radius {} remesh {} in-flight {} {:.1} MiB",
|
|
chunks.resident,
|
|
chunks.desired,
|
|
chunks.load_radius,
|
|
chunks.pending_remesh,
|
|
chunks.in_flight,
|
|
mib(chunks.resident_bytes)
|
|
);
|
|
let _ = writeln!(
|
|
out,
|
|
" loaded {} dropped {} evicted {} dispatched {} applied {} workers {}",
|
|
chunks.loaded_total,
|
|
chunks.dropped_total,
|
|
chunks.evicted_total,
|
|
chunks.dispatched_total,
|
|
chunks.applied_total,
|
|
chunks.mesh_workers
|
|
);
|
|
}
|
|
|
|
if let Some(render) = snapshot.render {
|
|
let _ = writeln!(
|
|
out,
|
|
"render {:?} meshes {} uploaded / {} visible / {} culled ({:.1}%) draws {}",
|
|
render.render_mode,
|
|
render.uploaded_meshes,
|
|
render.visible_meshes,
|
|
render.culled_meshes,
|
|
render.cull_ratio_percent(),
|
|
render.draw_calls
|
|
);
|
|
let _ = writeln!(
|
|
out,
|
|
" tris {} verts {} buffers {:.1} MiB vtx / {:.1} MiB idx presented {} skipped {}",
|
|
render.triangles,
|
|
render.vertices,
|
|
mib(render.vertex_bytes),
|
|
mib(render.index_bytes),
|
|
render.frames_presented,
|
|
render.frames_skipped
|
|
);
|
|
let _ = writeln!(
|
|
out,
|
|
" swapchain {}x{} x{} {} fov {:.1} near {} far {} aspect {:.3}",
|
|
render.swapchain.width,
|
|
render.swapchain.height,
|
|
render.swapchain.image_count,
|
|
render.swapchain.present_mode,
|
|
render.projection.fov_y_radians.to_degrees(),
|
|
render.projection.near,
|
|
render.projection.far,
|
|
render.projection.aspect
|
|
);
|
|
}
|
|
|
|
if let Some(gpu) = snapshot.gpu {
|
|
let _ = writeln!(
|
|
out,
|
|
"gpu {} ({}) vendor {:#06x} device {:#06x}",
|
|
gpu.device_name, gpu.device_type, gpu.vendor_id, gpu.device_id
|
|
);
|
|
let _ = writeln!(
|
|
out,
|
|
" driver {} vulkan {} vram {:.0} MiB",
|
|
gpu.driver_version,
|
|
gpu.api_version,
|
|
mib(gpu.vram_total_bytes)
|
|
);
|
|
}
|
|
|
|
if let Some(memory) = snapshot.memory {
|
|
let heap = match (memory.heap_usage_bytes, memory.heap_budget_bytes) {
|
|
(Some(used), Some(budget)) => {
|
|
format!("heap {:.0} / {:.0} MiB", mib(used), mib(budget))
|
|
}
|
|
// The extension is absent, so the driver publishes no figure to report.
|
|
_ => "heap unavailable".to_owned(),
|
|
};
|
|
let _ = writeln!(
|
|
out,
|
|
"vram {heap} allocator {:.1} / {:.1} MiB",
|
|
mib(memory.allocator_allocated_bytes),
|
|
mib(memory.allocator_capacity_bytes)
|
|
);
|
|
}
|
|
|
|
if let Some(identity) = snapshot.identity {
|
|
let _ = writeln!(
|
|
out,
|
|
"server {} {} build {} protocol {} nominal {} Hz",
|
|
identity.kind.label(),
|
|
identity.address,
|
|
identity.server_build,
|
|
identity.protocol_version,
|
|
identity.tick_rate_hint
|
|
);
|
|
}
|
|
|
|
if let Some(net) = snapshot.net {
|
|
let _ = writeln!(
|
|
out,
|
|
"net {} rtt {:.1} ms cwnd {} lost {} mtu {}",
|
|
if net.connected {
|
|
"connected"
|
|
} else {
|
|
"disconnected"
|
|
},
|
|
net.rtt_ms,
|
|
net.congestion_window,
|
|
net.lost_packets,
|
|
net.path_mtu
|
|
);
|
|
let _ = writeln!(
|
|
out,
|
|
" tx {:.2} MiB / {} dgram rx {:.2} MiB / {} dgram chunks {} drops {} subs {}",
|
|
mib(net.bytes_sent),
|
|
net.datagrams_sent,
|
|
mib(net.bytes_received),
|
|
net.datagrams_received,
|
|
net.chunks_received,
|
|
net.drops_received,
|
|
net.subscribes_sent
|
|
);
|
|
}
|
|
|
|
if let Some(server) = snapshot.server {
|
|
let _ = writeln!(
|
|
out,
|
|
"tick {:.1} tps mean {:.2} ms max {:.2} ms budget {:.0}% uptime {} s",
|
|
server.measured_tps,
|
|
server.mean_tick_ms,
|
|
server.max_tick_ms,
|
|
server.tick_budget_percent,
|
|
server.uptime_secs
|
|
);
|
|
let _ = writeln!(
|
|
out,
|
|
"world chunks {} resident / {} in flight clients {} entities {} players {}",
|
|
server.loaded_chunks,
|
|
server.chunks_in_flight,
|
|
server.connected_clients,
|
|
server.entities,
|
|
server.players
|
|
);
|
|
}
|
|
|
|
let host = snapshot.host;
|
|
let usage = snapshot.usage;
|
|
let _ = writeln!(
|
|
out,
|
|
"host {} x{} @ {} MHz {} kernel {}",
|
|
host.cpu_brand, host.logical_cores, usage.cpu_frequency_mhz, host.os, host.kernel
|
|
);
|
|
let _ = write!(
|
|
out,
|
|
"proc build {} cpu {:.1}% rss {:.1} MiB virt {:.1} MiB system {:.0} / {:.0} MiB free",
|
|
host.client_build,
|
|
usage.process_cpu_percent,
|
|
mib(usage.process_memory_bytes),
|
|
mib(usage.process_virtual_bytes),
|
|
mib(usage.system_available_bytes),
|
|
mib(usage.system_total_bytes)
|
|
);
|
|
|
|
out
|
|
}
|
|
|
|
#[cfg(test)]
|
|
#[path = "tests/stats.rs"]
|
|
mod tests;
|