Pumpkin-MC/Pumpkin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved chunk deserializing performance

Browse Source 
doing `let _ = io::copy(&mut self.reader.by_ref().take(count), &mut io::sink())` was just crazy slow when skipping bytes in NBT, we now instead use `seek`

The Palette was also changed and does not use a HashMap anymore
This commit is contained in:
Alexander Medvedev 2025-08-15 14:57:43 +02:00
parent ea2678a235
commit fbbade23dc
10 changed files with 241 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
pumpkin-nbt/src/compound.rs
View File

@ -5,7 +5,7 @@ use crate::deserializer::NbtReadHelper;
use crate::serializer::WriteAdaptor; use crate::serializer::WriteAdaptor;
use crate::tag::NbtTag; use crate::tag::NbtTag;
use crate::{END_ID, Error, Nbt, get_nbt_string}; use crate::{END_ID, Error, Nbt, get_nbt_string};
use std::io::{ErrorKind, Read, Write}; use std::io::{ErrorKind, Read, Seek, Write};
use std::vec::IntoIter; use std::vec::IntoIter;
#[derive(Clone, Debug, Default, PartialEq, PartialOrd)] #[derive(Clone, Debug, Default, PartialEq, PartialOrd)]
@ -20,7 +20,7 @@ impl NbtCompound {
} }
} }
pub fn skip_content<R: Read>(reader: &mut NbtReadHelper<R>) -> Result<(), Error> { pub fn skip_content<R: Read + Seek>(reader: &mut NbtReadHelper<R>) -> Result<(), Error> {
loop { loop {
let tag_id = match reader.get_u8_be() { let tag_id = match reader.get_u8_be() {
Ok(id) => id, Ok(id) => id,
@ -43,7 +43,7 @@ impl NbtCompound {
} }
let len = reader.get_u16_be()?; let len = reader.get_u16_be()?;
reader.skip_bytes(len as u64)?; reader.skip_bytes(len as i64)?;
NbtTag::skip_data(reader, tag_id)?; NbtTag::skip_data(reader, tag_id)?;
} }
@ -51,7 +51,7 @@ impl NbtCompound {
Ok(()) Ok(())
} }
pub fn deserialize_content<R: Read>( pub fn deserialize_content<R: Read + Seek>(
reader: &mut NbtReadHelper<R>, reader: &mut NbtReadHelper<R>,
) -> Result<NbtCompound, Error> { ) -> Result<NbtCompound, Error> {
let mut compound = NbtCompound::new(); let mut compound = NbtCompound::new();

89
pumpkin-nbt/src/deserializer.rs
View File

@ -1,3 +1,5 @@
use std::io::{Seek, SeekFrom};
use crate::*; use crate::*;
use io::Read; use io::Read;
use serde::de::{self, DeserializeSeed, IntoDeserializer, MapAccess, SeqAccess, Visitor}; use serde::de::{self, DeserializeSeed, IntoDeserializer, MapAccess, SeqAccess, Visitor};
@ -6,11 +8,11 @@ use serde::{Deserialize, forward_to_deserialize_any};
pub type Result<T> = std::result::Result<T, Error>; pub type Result<T> = std::result::Result<T, Error>;
#[derive(Debug)] #[derive(Debug)]
pub struct NbtReadHelper<R: Read> { pub struct NbtReadHelper<R: Read + Seek> {
reader: R, reader: R,
} }
impl<R: Read> NbtReadHelper<R> { impl<R: Read + Seek> NbtReadHelper<R> {
pub fn new(r: R) -> Self { pub fn new(r: R) -> Self {
Self { reader: r } Self { reader: r }
} }
@ -29,9 +31,11 @@ macro_rules! define_get_number_be {
}; };
} }
impl<R: Read> NbtReadHelper<R> { impl<R: Read + Seek> NbtReadHelper<R> {
pub fn skip_bytes(&mut self, count: u64) -> Result<()> { pub fn skip_bytes(&mut self, count: i64) -> Result<()> {
let _ = io::copy(&mut self.reader.by_ref().take(count), &mut io::sink()) self.reader
.by_ref()
.seek(SeekFrom::Current(count))
.map_err(Error::Incomplete)?; .map_err(Error::Incomplete)?;
Ok(()) Ok(())
} }
@ -58,41 +62,38 @@ impl<R: Read> NbtReadHelper<R> {
} }
#[derive(Debug)] #[derive(Debug)]
pub struct Deserializer<R: Read> { pub struct Deserializer<R: Read + Seek> {
input: NbtReadHelper<R>, input: NbtReadHelper<R>,
tag_to_deserialize_stack: Vec<u8>, tag_to_deserialize_stack: Option<u8>,
// Yes, this breaks with recursion. Just an attempt at a sanity check // Yes, this breaks with recursion. Just an attempt at a sanity check
in_list: bool, in_list: bool,
is_named: bool, is_named: bool,
// For debugging
key_stack: Vec<String>,
} }
impl<R: Read> Deserializer<R> { impl<R: Read + Seek> Deserializer<R> {
pub fn new(input: R, is_named: bool) -> Self { pub fn new(input: R, is_named: bool) -> Self {
Deserializer { Deserializer {
input: NbtReadHelper { reader: input }, input: NbtReadHelper { reader: input },
tag_to_deserialize_stack: Vec::new(), tag_to_deserialize_stack: None,
in_list: false, in_list: false,
is_named, is_named,
key_stack: Vec::new(),
} }
} }
} }
/// Deserializes struct using Serde Deserializer from normal NBT /// Deserializes struct using Serde Deserializer from normal NBT
pub fn from_bytes<'a, T: Deserialize<'a>>(r: impl Read) -> Result<T> { pub fn from_bytes<'a, T: Deserialize<'a>>(r: impl Read + Seek) -> Result<T> {
let mut deserializer = Deserializer::new(r, true); let mut deserializer = Deserializer::new(r, true);
T::deserialize(&mut deserializer) T::deserialize(&mut deserializer)
} }
/// Deserializes struct using Serde Deserializer from network NBT /// Deserializes struct using Serde Deserializer from network NBT
pub fn from_bytes_unnamed<'a, T: Deserialize<'a>>(r: impl Read) -> Result<T> { pub fn from_bytes_unnamed<'a, T: Deserialize<'a>>(r: impl Read + Seek) -> Result<T> {
let mut deserializer = Deserializer::new(r, false); let mut deserializer = Deserializer::new(r, false);
T::deserialize(&mut deserializer) T::deserialize(&mut deserializer)
} }
impl<'de, R: Read> de::Deserializer<'de> for &mut Deserializer<R> { impl<'de, R: Read + Seek> de::Deserializer<'de> for &mut Deserializer<R> {
type Error = Error; type Error = Error;
forward_to_deserialize_any! { forward_to_deserialize_any! {
@ -101,7 +102,7 @@ impl<'de, R: Read> de::Deserializer<'de> for &mut Deserializer<R> {
} }
fn deserialize_ignored_any<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value> { fn deserialize_ignored_any<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value> {
let Some(tag) = self.tag_to_deserialize_stack.pop() else { let Some(tag) = self.tag_to_deserialize_stack else {
return Err(Error::SerdeError("Ignoring nothing!".to_string())); return Err(Error::SerdeError("Ignoring nothing!".to_string()));
}; };
@ -110,7 +111,7 @@ impl<'de, R: Read> de::Deserializer<'de> for &mut Deserializer<R> {
} }
fn deserialize_any<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value> { fn deserialize_any<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value> {
let Some(tag_to_deserialize) = self.tag_to_deserialize_stack.pop() else { let Some(tag_to_deserialize) = self.tag_to_deserialize_stack else {
return Err(Error::SerdeError( return Err(Error::SerdeError(
"The top level must be a component (e.g. a struct)".to_string(), "The top level must be a component (e.g. a struct)".to_string(),
)); ));
@ -188,26 +189,14 @@ impl<'de, R: Read> de::Deserializer<'de> for &mut Deserializer<R> {
} }
fn deserialize_bool<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value> { fn deserialize_bool<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value> {
if let Some(tag_id) = self.tag_to_deserialize_stack.last() { if self.tag_to_deserialize_stack.unwrap() == BYTE_ID {
if *tag_id == BYTE_ID {
let value = self.input.get_u8_be()?; let value = self.input.get_u8_be()?;
if value != 0 { if value != 0 {
visitor.visit_bool(true) return visitor.visit_bool(true);
} else { }
}
visitor.visit_bool(false) visitor.visit_bool(false)
} }
} else {
Err(Error::UnsupportedType(format!(
"Non-byte bool (found type {tag_id})"
)))
}
} else {
Err(Error::SerdeError(
"Wanted to deserialize a bool, but there was no type hint on the stack!"
.to_string(),
))
}
}
fn deserialize_enum<V: Visitor<'de>>( fn deserialize_enum<V: Visitor<'de>>(
self, self,
@ -225,14 +214,10 @@ impl<'de, R: Read> de::Deserializer<'de> for &mut Deserializer<R> {
} }
fn deserialize_map<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value> { fn deserialize_map<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value> {
if let Some(tag_id) = self.tag_to_deserialize_stack.pop() { if let Some(tag_id) = self.tag_to_deserialize_stack {
if tag_id != COMPOUND_ID { if tag_id != COMPOUND_ID {
return Err(Error::SerdeError(format!( return Err(Error::SerdeError(format!(
"Trying to deserialize a map without a compound ID ({} with id {})", "Trying to deserialize a map without a compound ID (id {})",
self.key_stack
.last()
.cloned()
.unwrap_or_else(|| "compound root".to_string()),
tag_id tag_id
))); )));
} }
@ -243,8 +228,9 @@ impl<'de, R: Read> de::Deserializer<'de> for &mut Deserializer<R> {
} }
if self.is_named { if self.is_named {
// Consume struct name // Consume struct name, similar to get_nbt_string but without cesu8::from_java_cesu8
let _ = get_nbt_string(&mut self.input)?; let length = self.input.get_u16_be()? as usize;
let _ = self.input.read_boxed_slice(length)?;
} }
} }
@ -271,16 +257,16 @@ impl<'de, R: Read> de::Deserializer<'de> for &mut Deserializer<R> {
} }
} }
struct CompoundAccess<'a, R: Read> { struct CompoundAccess<'a, R: Read + Seek> {
de: &'a mut Deserializer<R>, de: &'a mut Deserializer<R>,
} }
impl<'de, R: Read> MapAccess<'de> for CompoundAccess<'_, R> { impl<'de, R: Read + Seek> MapAccess<'de> for CompoundAccess<'_, R> {
type Error = Error; type Error = Error;
fn next_key_seed<K: DeserializeSeed<'de>>(&mut self, seed: K) -> Result<Option<K::Value>> { fn next_key_seed<K: DeserializeSeed<'de>>(&mut self, seed: K) -> Result<Option<K::Value>> {
let tag = self.de.input.get_u8_be()?; let tag = self.de.input.get_u8_be()?;
self.de.tag_to_deserialize_stack.push(tag); self.de.tag_to_deserialize_stack = Some(tag);
if tag == END_ID { if tag == END_ID {
return Ok(None); return Ok(None);
@ -290,22 +276,19 @@ impl<'de, R: Read> MapAccess<'de> for CompoundAccess<'_, R> {
} }
fn next_value_seed<V: DeserializeSeed<'de>>(&mut self, seed: V) -> Result<V::Value> { fn next_value_seed<V: DeserializeSeed<'de>>(&mut self, seed: V) -> Result<V::Value> {
let result = seed.deserialize(&mut *self.de); seed.deserialize(&mut *self.de)
self.de.key_stack.pop();
result
} }
} }
struct MapKey<'a, R: Read> { struct MapKey<'a, R: Read + Seek> {
de: &'a mut Deserializer<R>, de: &'a mut Deserializer<R>,
} }
impl<'de, R: Read> de::Deserializer<'de> for MapKey<'_, R> { impl<'de, R: Read + Seek> de::Deserializer<'de> for MapKey<'_, R> {
type Error = Error; type Error = Error;
fn deserialize_any<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value> { fn deserialize_any<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value> {
let key = get_nbt_string(&mut self.de.input)?; let key = get_nbt_string(&mut self.de.input)?;
self.de.key_stack.push(key.clone());
visitor.visit_string(key) visitor.visit_string(key)
} }
@ -315,13 +298,13 @@ impl<'de, R: Read> de::Deserializer<'de> for MapKey<'_, R> {
} }
} }
struct ListAccess<'a, R: Read> { struct ListAccess<'a, R: Read + Seek> {
de: &'a mut Deserializer<R>, de: &'a mut Deserializer<R>,
remaining_values: usize, remaining_values: usize,
list_type: u8, list_type: u8,
} }
impl<'de, R: Read> SeqAccess<'de> for ListAccess<'_, R> { impl<'de, R: Read + Seek> SeqAccess<'de> for ListAccess<'_, R> {
type Error = Error; type Error = Error;
fn size_hint(&self) -> Option<usize> { fn size_hint(&self) -> Option<usize> {
@ -334,7 +317,7 @@ impl<'de, R: Read> SeqAccess<'de> for ListAccess<'_, R> {
} }
self.remaining_values -= 1; self.remaining_values -= 1;
self.de.tag_to_deserialize_stack.push(self.list_type); self.de.tag_to_deserialize_stack = Some(self.list_type);
self.de.in_list = true; self.de.in_list = true;
let result = seed.deserialize(&mut *self.de).map(Some); let result = seed.deserialize(&mut *self.de).map(Some);
self.de.in_list = false; self.de.in_list = false;

24
pumpkin-nbt/src/lib.rs
View File

@ -1,6 +1,6 @@
use std::{ use std::{
fmt::Display, fmt::Display,
io::{self, Read, Write}, io::{self, Read, Seek, Write},
ops::Deref, ops::Deref,
}; };
@ -83,7 +83,7 @@ impl Nbt {
} }
} }
pub fn read<R: Read>(reader: &mut NbtReadHelper<R>) -> Result<Nbt, Error> { pub fn read<R: Read + Seek>(reader: &mut NbtReadHelper<R>) -> Result<Nbt, Error> {
let tag_type_id = reader.get_u8_be()?; let tag_type_id = reader.get_u8_be()?;
if tag_type_id != COMPOUND_ID { if tag_type_id != COMPOUND_ID {
@ -97,7 +97,7 @@ impl Nbt {
} }
/// Reads an NBT tag that doesn't contain the name of the root `Compound`. /// Reads an NBT tag that doesn't contain the name of the root `Compound`.
pub fn read_unnamed<R: Read>(reader: &mut NbtReadHelper<R>) -> Result<Nbt, Error> { pub fn read_unnamed<R: Read + Seek>(reader: &mut NbtReadHelper<R>) -> Result<Nbt, Error> {
let tag_type_id = reader.get_u8_be()?; let tag_type_id = reader.get_u8_be()?;
if tag_type_id != COMPOUND_ID { if tag_type_id != COMPOUND_ID {
@ -174,7 +174,7 @@ impl AsMut<NbtCompound> for Nbt {
} }
} }
pub fn get_nbt_string<R: Read>(bytes: &mut NbtReadHelper<R>) -> Result<String, Error> { pub fn get_nbt_string<R: Read + Seek>(bytes: &mut NbtReadHelper<R>) -> Result<String, Error> {
let len = bytes.get_u16_be()? as usize; let len = bytes.get_u16_be()? as usize;
let string_bytes = bytes.read_boxed_slice(len)?; let string_bytes = bytes.read_boxed_slice(len)?;
let string = cesu8::from_java_cesu8(&string_bytes).map_err(|_| Error::Cesu8DecodingError)?; let string = cesu8::from_java_cesu8(&string_bytes).map_err(|_| Error::Cesu8DecodingError)?;
@ -205,6 +205,8 @@ impl_array!(nbt_byte_array, NBT_BYTE_ARRAY_TAG);
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use std::io::Cursor;
use crate::Error; use crate::Error;
use crate::deserializer::from_bytes; use crate::deserializer::from_bytes;
use crate::nbt_byte_array; use crate::nbt_byte_array;
@ -238,7 +240,7 @@ mod test {
let mut bytes = Vec::new(); let mut bytes = Vec::new();
to_bytes_unnamed(&test, &mut bytes).unwrap(); to_bytes_unnamed(&test, &mut bytes).unwrap();
let recreated_struct: Test = from_bytes_unnamed(&bytes[..]).unwrap(); let recreated_struct: Test = from_bytes_unnamed(Cursor::new(bytes)).unwrap();
assert_eq!(test, recreated_struct); assert_eq!(test, recreated_struct);
} }
@ -263,7 +265,7 @@ mod test {
let mut bytes = Vec::new(); let mut bytes = Vec::new();
to_bytes_unnamed(&test, &mut bytes).unwrap(); to_bytes_unnamed(&test, &mut bytes).unwrap();
let recreated_struct: TestArray = from_bytes_unnamed(&bytes[..]).unwrap(); let recreated_struct: TestArray = from_bytes_unnamed(Cursor::new(bytes)).unwrap();
assert_eq!(test, recreated_struct); assert_eq!(test, recreated_struct);
} }
@ -282,7 +284,7 @@ mod test {
let mut bytes = Vec::new(); let mut bytes = Vec::new();
to_bytes_named(&test, name, &mut bytes).unwrap(); to_bytes_named(&test, name, &mut bytes).unwrap();
let recreated_struct: Test = from_bytes(&bytes[..]).unwrap(); let recreated_struct: Test = from_bytes(Cursor::new(bytes)).unwrap();
assert_eq!(test, recreated_struct); assert_eq!(test, recreated_struct);
} }
@ -298,7 +300,7 @@ mod test {
let mut bytes = Vec::new(); let mut bytes = Vec::new();
to_bytes_named(&test, name, &mut bytes).unwrap(); to_bytes_named(&test, name, &mut bytes).unwrap();
let recreated_struct: TestArray = from_bytes(&bytes[..]).unwrap(); let recreated_struct: TestArray = from_bytes(Cursor::new(bytes)).unwrap();
assert_eq!(test, recreated_struct); assert_eq!(test, recreated_struct);
} }
@ -358,7 +360,7 @@ mod test {
let mut bytes = Vec::new(); let mut bytes = Vec::new();
to_bytes_unnamed(&list_compound, &mut bytes).unwrap(); to_bytes_unnamed(&list_compound, &mut bytes).unwrap();
let recreated_struct: TestList = from_bytes_unnamed(&bytes[..]).unwrap(); let recreated_struct: TestList = from_bytes_unnamed(Cursor::new(bytes)).unwrap();
assert_eq!(list_compound, recreated_struct); assert_eq!(list_compound, recreated_struct);
} }
@ -396,7 +398,7 @@ mod test {
let mut bytes = Vec::new(); let mut bytes = Vec::new();
to_bytes_named(&list_compound, "a".to_string(), &mut bytes).unwrap(); to_bytes_named(&list_compound, "a".to_string(), &mut bytes).unwrap();
let recreated_struct: TestList = from_bytes(&bytes[..]).unwrap(); let recreated_struct: TestList = from_bytes(Cursor::new(bytes)).unwrap();
assert_eq!(list_compound, recreated_struct); assert_eq!(list_compound, recreated_struct);
} }
@ -511,7 +513,7 @@ mod test {
let mut bytes = Vec::new(); let mut bytes = Vec::new();
to_bytes(&value, &mut bytes).unwrap(); to_bytes(&value, &mut bytes).unwrap();
let reconstructed = from_bytes(&bytes[..]).unwrap(); let reconstructed = from_bytes(Cursor::new(bytes)).unwrap();
assert_eq!(value, reconstructed); assert_eq!(value, reconstructed);
} }

16
pumpkin-nbt/src/nbt_compress.rs
View File

@ -1,7 +1,7 @@
use crate::deserializer::NbtReadHelper; use crate::deserializer::NbtReadHelper;
use crate::{Error, Nbt, NbtCompound, deserializer, serializer}; use crate::{Error, Nbt, NbtCompound, deserializer, serializer};
use flate2::{Compression, read::GzDecoder, write::GzEncoder}; use flate2::{Compression, read::GzDecoder, write::GzEncoder};
use std::io::{Read, Write}; use std::io::{Cursor, Read, Seek, Write};
/// Reads a GZipped NBT compound tag from any reader. /// Reads a GZipped NBT compound tag from any reader.
/// ///
@ -12,10 +12,12 @@ use std::io::{Read, Write};
/// # Returns /// # Returns
/// ///
/// A Result containing either the parsed NbtCompound or an Error /// A Result containing either the parsed NbtCompound or an Error
pub fn read_gzip_compound_tag(input: impl Read) -> Result<NbtCompound, Error> { pub fn read_gzip_compound_tag(input: impl Read + Seek) -> Result<NbtCompound, Error> {
// Create a GZip decoder and directly chain it to the NBT reader // Create a GZip decoder and directly chain it to the NBT reader
let decoder = GzDecoder::new(input); let mut decoder = GzDecoder::new(input);
let mut reader = NbtReadHelper::new(decoder); let mut buf = Vec::new();
decoder.read_to_end(&mut buf).unwrap();
let mut reader = NbtReadHelper::new(Cursor::new(buf));
// Read the NBT data directly from the decoder stream // Read the NBT data directly from the decoder stream
let nbt = Nbt::read(&mut reader)?; let nbt = Nbt::read(&mut reader)?;
@ -67,8 +69,10 @@ pub fn write_gzip_compound_tag_to_bytes(compound: &NbtCompound) -> Result<Vec<u8
/// A Result containing either the deserialized type or an Error /// A Result containing either the deserialized type or an Error
pub fn from_gzip_bytes<'a, T: serde::Deserialize<'a>, R: Read>(input: R) -> Result<T, Error> { pub fn from_gzip_bytes<'a, T: serde::Deserialize<'a>, R: Read>(input: R) -> Result<T, Error> {
// Create a GZip decoder and directly use it for deserialization // Create a GZip decoder and directly use it for deserialization
let decoder = GzDecoder::new(input); let mut decoder = GzDecoder::new(input);
deserializer::from_bytes(decoder) let mut buf = Vec::new();
decoder.read_to_end(&mut buf).unwrap();
deserializer::from_bytes(Cursor::new(buf))
} }
/// Writes a Rust type as GZipped NBT to any writer. /// Writes a Rust type as GZipped NBT to any writer.

17
pumpkin-nbt/src/tag.rs
View File

@ -108,12 +108,15 @@ impl NbtTag {
Ok(()) Ok(())
} }
pub fn deserialize<R: Read>(reader: &mut NbtReadHelper<R>) -> Result<NbtTag, Error> { pub fn deserialize<R: Read + Seek>(reader: &mut NbtReadHelper<R>) -> Result<NbtTag, Error> {
let tag_id = reader.get_u8_be()?; let tag_id = reader.get_u8_be()?;
Self::deserialize_data(reader, tag_id) Self::deserialize_data(reader, tag_id)
} }
pub fn skip_data<R: Read>(reader: &mut NbtReadHelper<R>, tag_id: u8) -> Result<(), Error> { pub fn skip_data<R: Read + Seek>(
reader: &mut NbtReadHelper<R>,
tag_id: u8,
) -> Result<(), Error> {
match tag_id { match tag_id {
END_ID => Ok(()), END_ID => Ok(()),
BYTE_ID => reader.skip_bytes(1), BYTE_ID => reader.skip_bytes(1),
@ -127,11 +130,11 @@ impl NbtTag {
if len < 0 { if len < 0 {
return Err(Error::NegativeLength(len)); return Err(Error::NegativeLength(len));
} }
reader.skip_bytes(len as u64) reader.skip_bytes(len as i64)
} }
STRING_ID => { STRING_ID => {
let len = reader.get_u16_be()?; let len = reader.get_u16_be()?;
reader.skip_bytes(len as u64) reader.skip_bytes(len as i64)
} }
LIST_ID => { LIST_ID => {
let tag_type_id = reader.get_u8_be()?; let tag_type_id = reader.get_u8_be()?;
@ -153,7 +156,7 @@ impl NbtTag {
return Err(Error::NegativeLength(len)); return Err(Error::NegativeLength(len));
} }
reader.skip_bytes(len as u64 * 4) reader.skip_bytes(len as i64 * 4)
} }
LONG_ARRAY_ID => { LONG_ARRAY_ID => {
let len = reader.get_i32_be()?; let len = reader.get_i32_be()?;
@ -161,13 +164,13 @@ impl NbtTag {
return Err(Error::NegativeLength(len)); return Err(Error::NegativeLength(len));
} }
reader.skip_bytes(len as u64 * 8) reader.skip_bytes(len as i64 * 8)
} }
_ => Err(Error::UnknownTagId(tag_id)), _ => Err(Error::UnknownTagId(tag_id)),
} }
} }
pub fn deserialize_data<R: Read>( pub fn deserialize_data<R: Read + Seek>(
reader: &mut NbtReadHelper<R>, reader: &mut NbtReadHelper<R>,
tag_id: u8, tag_id: u8,
) -> Result<NbtTag, Error> { ) -> Result<NbtTag, Error> {

8
pumpkin-world/src/chunk/format/mod.rs
View File

@ -1,4 +1,4 @@
use std::{collections::HashMap, path::PathBuf}; use std::{collections::HashMap, io::Cursor, path::PathBuf};
use async_trait::async_trait; use async_trait::async_trait;
use bytes::Bytes; use bytes::Bytes;
@ -80,7 +80,7 @@ impl ChunkData {
position: Vector2<i32>, position: Vector2<i32>,
) -> Result<Self, ChunkParsingError> { ) -> Result<Self, ChunkParsingError> {
// TODO: Implement chunk stages? // TODO: Implement chunk stages?
if from_bytes::<ChunkStatusWrapper>(chunk_data) if from_bytes::<ChunkStatusWrapper>(Cursor::new(chunk_data))
.map_err(ChunkParsingError::FailedReadStatus)? .map_err(ChunkParsingError::FailedReadStatus)?
.status .status
!= ChunkStatus::Full != ChunkStatus::Full
@ -88,7 +88,7 @@ impl ChunkData {
return Err(ChunkParsingError::ChunkNotGenerated); return Err(ChunkParsingError::ChunkNotGenerated);
} }
let chunk_data = from_bytes::<ChunkNbt>(chunk_data) let chunk_data = from_bytes::<ChunkNbt>(Cursor::new(chunk_data))
.map_err(|e| ChunkParsingError::ErrorDeserializingChunk(e.to_string()))?; .map_err(|e| ChunkParsingError::ErrorDeserializingChunk(e.to_string()))?;
if chunk_data.light_correct { if chunk_data.light_correct {
@ -298,7 +298,7 @@ impl ChunkEntityData {
chunk_data: &[u8], chunk_data: &[u8],
position: Vector2<i32>, position: Vector2<i32>,
) -> Result<Self, ChunkParsingError> { ) -> Result<Self, ChunkParsingError> {
let chunk_entity_data = pumpkin_nbt::from_bytes::<EntityNbt>(chunk_data) let chunk_entity_data = pumpkin_nbt::from_bytes::<EntityNbt>(Cursor::new(chunk_data))
.map_err(|e| ChunkParsingError::ErrorDeserializingChunk(e.to_string()))?; .map_err(|e| ChunkParsingError::ErrorDeserializingChunk(e.to_string()))?;
if chunk_entity_data.position[0] != position.x if chunk_entity_data.position[0] != position.x

228
pumpkin-world/src/chunk/palette.rs
View File

@ -1,8 +1,4 @@
use std::{ use std::{collections::HashMap, hash::Hash};
cmp::Ordering,
collections::{HashMap, hash_map::Entry},
hash::Hash,
};
use pumpkin_data::{Block, BlockState, chunk::Biome}; use pumpkin_data::{Block, BlockState, chunk::Biome};
use pumpkin_util::encompassing_bits; use pumpkin_util::encompassing_bits;
@ -17,7 +13,8 @@ type AbstractCube<T, const DIM: usize> = [[[T; DIM]; DIM]; DIM];
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct HeterogeneousPaletteData<V: Hash + Eq + Copy, const DIM: usize> { pub struct HeterogeneousPaletteData<V: Hash + Eq + Copy, const DIM: usize> {
cube: Box<AbstractCube<V, DIM>>, cube: Box<AbstractCube<V, DIM>>,
counts: HashMap<V, u16>, palette: Vec<V>,
counts: Vec<u16>,
} }
impl<V: Hash + Eq + Copy, const DIM: usize> HeterogeneousPaletteData<V, DIM> { impl<V: Hash + Eq + Copy, const DIM: usize> HeterogeneousPaletteData<V, DIM> {
@ -36,19 +33,26 @@ impl<V: Hash + Eq + Copy, const DIM: usize> HeterogeneousPaletteData<V, DIM> {
debug_assert!(z < DIM); debug_assert!(z < DIM);
let original = self.cube[y][z][x]; let original = self.cube[y][z][x];
if let Entry::Occupied(mut entry) = self.counts.entry(original) { let original_index = self.palette.iter().position(|v| v == &original).unwrap();
let count = entry.get_mut(); self.counts[original_index] -= 1;
*count -= 1;
if *count == 0 { if self.counts[original_index] == 0 {
let _ = entry.remove(); // Remove from palette and counts Vecs if the count hits zero.
} self.palette.swap_remove(original_index);
self.counts.swap_remove(original_index);
} }
// Set the new value in the cube
self.cube[y][z][x] = value; self.cube[y][z][x] = value;
self.counts
.entry(value) // Find or add the new value to the palette.
.and_modify(|count| *count += 1) if let Some(new_index) = self.palette.iter().position(|v| v == &value) {
.or_insert(1); self.counts[new_index] += 1;
} else {
self.palette.push(value);
self.counts.push(1);
}
original original
} }
} }
@ -66,19 +70,31 @@ impl<V: Hash + Eq + Copy + Default, const DIM: usize> PalettedContainer<V, DIM>
pub const VOLUME: usize = DIM * DIM * DIM; pub const VOLUME: usize = DIM * DIM * DIM;
fn from_cube(cube: Box<AbstractCube<V, DIM>>) -> Self { fn from_cube(cube: Box<AbstractCube<V, DIM>>) -> Self {
let counts = let mut palette: Vec<V> = Vec::new();
cube.as_flattened() let mut counts: Vec<u16> = Vec::new();
.as_flattened()
.iter()
.fold(HashMap::new(), |mut acc, key| {
acc.entry(*key).and_modify(|count| *count += 1).or_insert(1);
acc
});
if counts.len() == 1 { // Iterate over the flattened cube to populate the palette and counts
Self::Homogeneous(*counts.keys().next().unwrap()) for val in cube.as_flattened().as_flattened().iter() {
if let Some(index) = palette.iter().position(|v| v == val) {
// Value already exists, increment its count
counts[index] += 1;
} else { } else {
Self::Heterogeneous(Box::new(HeterogeneousPaletteData { cube, counts })) // New value, add it to the palette and start its count
palette.push(*val);
counts.push(1);
}
}
if palette.len() == 1 {
// Fast path: the cube is homogeneous, so we can store just one value
Self::Homogeneous(palette[0])
} else {
// Heterogeneous cube, store the full data
Self::Heterogeneous(Box::new(HeterogeneousPaletteData {
cube,
palette,
counts,
}))
} }
} }
@ -96,7 +112,8 @@ impl<V: Hash + Eq + Copy + Default, const DIM: usize> PalettedContainer<V, DIM>
debug_assert!(bits_per_entry >= encompassing_bits(data.counts.len())); debug_assert!(bits_per_entry >= encompassing_bits(data.counts.len()));
debug_assert!(bits_per_entry <= 15); debug_assert!(bits_per_entry <= 15);
let palette: Box<[V]> = data.counts.keys().copied().collect(); let palette: Box<[V]> = data.palette.iter().copied().collect();
let key_to_index_map: HashMap<V, usize> = palette let key_to_index_map: HashMap<V, usize> = palette
.iter() .iter()
.enumerate() .enumerate()
@ -128,62 +145,79 @@ impl<V: Hash + Eq + Copy + Default, const DIM: usize> PalettedContainer<V, DIM>
} }
pub fn from_palette_and_packed_data( pub fn from_palette_and_packed_data(
palette: &[V], palette_slice: &[V],
packed_data: &[i64], packed_data: &[i64],
minimum_bits_per_entry: u8, minimum_bits_per_entry: u8,
) -> Self { ) -> Self {
if palette.is_empty() { if palette_slice.is_empty() {
log::warn!("No palette data! Defaulting..."); log::warn!("No palette data! Defaulting...");
Self::Homogeneous(V::default()) return Self::Homogeneous(V::default());
} else if palette.len() == 1 { }
Self::Homogeneous(palette[0])
} else { if palette_slice.len() == 1 {
let bits_per_key = encompassing_bits(palette.len()).max(minimum_bits_per_entry); return Self::Homogeneous(palette_slice[0]);
}
let bits_per_key = encompassing_bits(palette_slice.len()).max(minimum_bits_per_entry);
let index_mask = (1 << bits_per_key) - 1; let index_mask = (1 << bits_per_key) - 1;
let keys_per_i64 = 64 / bits_per_key; let keys_per_i64 = 64 / bits_per_key;
let expected_i64_count = Self::VOLUME.div_ceil(keys_per_i64 as usize); let mut decompressed_values = Vec::with_capacity(Self::VOLUME);
match packed_data.len().cmp(&expected_i64_count) { // We already have the palette from the input `palette_slice`.
Ordering::Greater => { // The counts will be created in the next step.
// Handled by the zip
log::warn!("Filled the section but there is still more data! Ignoring..."); let mut packed_data_iter = packed_data.iter();
} let mut current_packed_word = *packed_data_iter.next().unwrap_or(&0);
Ordering::Less => {
// Handled by the array initialization and zip for i in 0..Self::VOLUME {
log::warn!( let bit_index_in_word = i % keys_per_i64 as usize;
"Ran out of packed indices, but did not fill the section ({} vs {} for {}). Defaulting...",
packed_data.len() * keys_per_i64 as usize, if bit_index_in_word == 0 && i > 0 {
Self::VOLUME, current_packed_word = *packed_data_iter.next().unwrap_or(&0);
palette.len(), }
);
} let lookup_index = (current_packed_word as u64
// This is what we want! >> (bit_index_in_word as u64 * bits_per_key as u64))
Ordering::Equal => {} & index_mask;
let value = palette_slice
.get(lookup_index as usize)
.copied()
.unwrap_or_else(|| {
log::warn!("Lookup index out of bounds! Defaulting...");
V::default()
});
decompressed_values.push(value);
}
// Now, with all decompressed values, build the counts.
let mut counts = vec![0; palette_slice.len()];
for &value in &decompressed_values {
// This is the key optimization: find the index in the palette Vec
// and increment the corresponding count.
if let Some(index) = palette_slice.iter().position(|v| v == &value) {
counts[index] += 1;
} else {
// This case should ideally not happen if the palette is complete.
log::warn!("Decompressed value not found in palette!");
}
} }
// TODO: Can we do this all with an `array::from_fn` or something?
let mut cube = Box::new([[[V::default(); DIM]; DIM]; DIM]); let mut cube = Box::new([[[V::default(); DIM]; DIM]; DIM]);
cube.as_flattened_mut() cube.as_flattened_mut()
.as_flattened_mut() .as_flattened_mut()
.chunks_mut(keys_per_i64 as usize) .copy_from_slice(&decompressed_values);
.zip(packed_data)
.for_each(|(values, packed)| {
values.iter_mut().enumerate().for_each(|(index, value)| {
let lookup_index =
(*packed as u64 >> (index as u64 * bits_per_key as u64)) & index_mask;
if let Some(v) = palette.get(lookup_index as usize) { let palette_vec: Vec<V> = palette_slice.to_vec();
*value = *v;
} else {
// The cube is already initialized to the default
log::warn!("Lookup index out of bounds! Defaulting...");
}
});
});
Self::from_cube(cube) Self::Heterogeneous(Box::new(HeterogeneousPaletteData {
} cube,
palette: palette_vec,
counts,
}))
} }
pub fn get(&self, x: usize, y: usize, z: usize) -> V { pub fn get(&self, x: usize, y: usize, z: usize) -> V {
@ -211,7 +245,7 @@ impl<V: Hash + Eq + Copy + Default, const DIM: usize> PalettedContainer<V, DIM>
Self::Heterogeneous(data) => { Self::Heterogeneous(data) => {
let original = data.set(x, y, z, value); let original = data.set(x, y, z, value);
if data.counts.len() == 1 { if data.counts.len() == 1 {
*self = Self::Homogeneous(*data.counts.keys().next().unwrap()); *self = Self::Homogeneous(data.palette[0]);
} }
original original
} }
@ -384,51 +418,48 @@ impl BlockPalette {
packed_data: Box::new([]), packed_data: Box::new([]),
}, },
Self::Heterogeneous(data) => { Self::Heterogeneous(data) => {
let bits_per_entry = encompassing_bits(data.counts.len()); let bits_per_entry = encompassing_bits(data.palette.len());
let palette: Box<[u16]> = data.counts.keys().copied().collect();
let key_to_index_map: HashMap<_, usize> = palette let key_to_index_map: HashMap<_, usize> = data
.palette
.iter() .iter()
.enumerate() .enumerate()
.map(|(index, key)| (*key, index)) .map(|(index, key)| (*key, index))
.collect(); .collect();
let blocks_per_word = 32 / bits_per_entry; let blocks_per_word = 32 / bits_per_entry;
let expected_word_count = Self::VOLUME.div_ceil(blocks_per_word as usize);
let mut packed_data = Vec::with_capacity(expected_word_count);
// Direktes Verarbeiten in der Reihenfolge [x][y][z] ohne Kopie let mut current_word: u32 = 0;
let mut packed_data = Vec::new(); let mut current_index_in_word = 0;
let mut current_word = 0;
let mut current_index = 0;
for x in 0..16 { for key in data.cube.as_flattened().as_flattened().iter() {
for y in 0..16 { let key_index = key_to_index_map.get(key).unwrap();
for z in 0..16 {
let key = data.cube[z][y][x]; // Zugriff in [x][y][z]-Reihenfolge
let key_index = key_to_index_map.get(&key).unwrap();
debug_assert!((1 << bits_per_entry) > *key_index); debug_assert!((1 << bits_per_entry) > *key_index);
let packed_offset_index = (*key_index as u32) current_word |=
<< (bits_per_entry as u32 * current_index as u32); (*key_index as u32) << (bits_per_entry as u32 * current_index_in_word);
current_word |= packed_offset_index; current_index_in_word += 1;
current_index += 1; if current_index_in_word == blocks_per_word as u32 {
if current_index == blocks_per_word {
packed_data.push(current_word); packed_data.push(current_word);
current_word = 0; current_word = 0;
current_index = 0; current_index_in_word = 0;
} }
} }
}
} // Push any remaining bits if the volume isn't a multiple of blocks_per_word
if current_index > 0 { if current_index_in_word > 0 {
packed_data.push(current_word); packed_data.push(current_word);
} }
BeNetworkSerialization { BeNetworkSerialization {
bits_per_entry, bits_per_entry,
palette: NetworkPalette::Indirect( palette: NetworkPalette::Indirect(
palette data.palette
.into_iter() .iter()
.map(BlockState::to_be_network_id) .map(|&id| BlockState::to_be_network_id(id))
.collect(), .collect(),
), ),
packed_data: packed_data.into_boxed_slice(), packed_data: packed_data.into_boxed_slice(),
@ -447,13 +478,14 @@ impl BlockPalette {
} }
} }
Self::Heterogeneous(data) => data Self::Heterogeneous(data) => data
.counts .palette
.iter() .iter()
.map(|(registry_id, count)| { .zip(data.counts.iter())
.filter_map(|(registry_id, count)| {
if !BlockState::from_id(*registry_id).is_air() { if !BlockState::from_id(*registry_id).is_air() {
*count Some(*count)
} else { } else {
0 None
} }
}) })
.sum(), .sum(),

11
pumpkin-world/src/generation/biome.rs
View File

@ -1,4 +1,4 @@
use pumpkin_util::math::{floor_mod, square, vector3::Vector3}; use pumpkin_util::math::{square, vector3::Vector3};
use super::biome_coords; use super::biome_coords;
@ -134,7 +134,14 @@ fn score_permutation(
#[inline] #[inline]
fn scale_mix(l: i64) -> f64 { fn scale_mix(l: i64) -> f64 {
let d = floor_mod(l >> 24, 1024i32 as i64) as i32 as f64 / 1024.0; const RECIPROCAL_1024: f64 = 1.0 / 1024.0;
// Use a bitwise AND for a faster modulus by a power of two
let scaled_l = (l >> 24) & 1023;
// Use a multiplication instead of a division
let d = scaled_l as f64 * RECIPROCAL_1024;
(d - 0.5) * 0.9 (d - 0.5) * 0.9
} }

23
pumpkin-world/src/generation/surface/mod.rs
View File

@ -1,6 +1,3 @@
use std::{cell::RefCell, num::NonZeroUsize};
use lru::LruCache;
use pumpkin_data::chunk::Biome; use pumpkin_data::chunk::Biome;
use pumpkin_util::{ use pumpkin_util::{
math::{lerp2, vector2::Vector2, vector3::Vector3, vertical_surface_type::VerticalSurfaceType}, math::{lerp2, vector2::Vector2, vector3::Vector3, vertical_surface_type::VerticalSurfaceType},
@ -398,7 +395,7 @@ pub struct VerticalGradientMaterialCondition {
true_at_and_below: YOffset, true_at_and_below: YOffset,
false_at_and_above: YOffset, false_at_and_above: YOffset,
#[serde(skip)] #[serde(skip)]
random_deriver: ThreadLocal<RefCell<LruCache<usize, RandomDeriver>>>, random_deriver: ThreadLocal<RandomDeriver>,
} }
impl VerticalGradientMaterialCondition { impl VerticalGradientMaterialCondition {
@ -406,21 +403,9 @@ impl VerticalGradientMaterialCondition {
let true_at = self.true_at_and_below.get_y(context.min_y, context.height); let true_at = self.true_at_and_below.get_y(context.min_y, context.height);
let false_at = self.false_at_and_above.get_y(context.min_y, context.height); let false_at = self.false_at_and_above.get_y(context.min_y, context.height);
let context_pointer: *const RandomDeriver = context.random_deriver; let splitter = self.random_deriver.get_or(|| {
let key = context_pointer.addr(); let random_deriver_clone = context.random_deriver;
random_deriver_clone
let mut cache = self
.random_deriver
.get_or(|| {
let cache_size = NonZeroUsize::new(128).unwrap();
let cache = LruCache::new(cache_size);
RefCell::new(cache)
})
.borrow_mut();
let splitter = cache.get_or_insert(key, || {
context
.random_deriver
.split_string(&self.random_name) .split_string(&self.random_name)
.next_splitter() .next_splitter()
}); });

22
pumpkin-world/src/world_info/anvil.rs
View File

@ -1,6 +1,6 @@
use std::{ use std::{
fs::OpenOptions, fs::OpenOptions,
io::Read, io::{Cursor, Read},
path::Path, path::Path,
time::{SystemTime, UNIX_EPOCH}, time::{SystemTime, UNIX_EPOCH},
}; };
@ -38,7 +38,7 @@ fn check_file_data_version(raw_nbt: &[u8]) -> Result<(), WorldInfoError> {
data: LevelData, data: LevelData,
} }
let info: LevelDat = pumpkin_nbt::from_bytes(raw_nbt) let info: LevelDat = pumpkin_nbt::from_bytes(Cursor::new(raw_nbt))
.map_err(|e|{ .map_err(|e|{
log::error!("The level.dat file does not have a data version! This means it is either corrupt or very old (read unsupported)"); log::error!("The level.dat file does not have a data version! This means it is either corrupt or very old (read unsupported)");
WorldInfoError::DeserializationError(e.to_string())})?; WorldInfoError::DeserializationError(e.to_string())})?;
@ -65,7 +65,7 @@ fn check_file_level_version(raw_nbt: &[u8]) -> Result<(), WorldInfoError> {
data: LevelData, data: LevelData,
} }
let info: LevelDat = pumpkin_nbt::from_bytes(raw_nbt) let info: LevelDat = pumpkin_nbt::from_bytes(Cursor::new(raw_nbt))
.map_err(|e|{ .map_err(|e|{
log::error!("The level.dat file does not have a level version! This means it is either corrupt or very old (read unsupported)"); log::error!("The level.dat file does not have a level version! This means it is either corrupt or very old (read unsupported)");
WorldInfoError::DeserializationError(e.to_string())})?; WorldInfoError::DeserializationError(e.to_string())})?;
@ -91,7 +91,7 @@ impl WorldInfoReader for AnvilLevelInfo {
check_file_data_version(&buf)?; check_file_data_version(&buf)?;
check_file_level_version(&buf)?; check_file_level_version(&buf)?;
let info = pumpkin_nbt::from_bytes::<LevelDat>(&buf[..]) let info = pumpkin_nbt::from_bytes::<LevelDat>(Cursor::new(buf))
.map_err(|e| WorldInfoError::DeserializationError(e.to_string()))?; .map_err(|e| WorldInfoError::DeserializationError(e.to_string()))?;
// TODO: check version // TODO: check version
@ -143,7 +143,11 @@ pub struct LevelDat {
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use std::{fs, sync::LazyLock}; use std::{
fs,
io::{Cursor, Read},
sync::LazyLock,
};
use flate2::read::GzDecoder; use flate2::read::GzDecoder;
use pumpkin_data::game_rules::GameRuleRegistry; use pumpkin_data::game_rules::GameRuleRegistry;
@ -281,8 +285,10 @@ mod test {
let raw_compressed_nbt = fs::read("assets/level_1_21_4.dat").unwrap(); let raw_compressed_nbt = fs::read("assets/level_1_21_4.dat").unwrap();
assert!(!raw_compressed_nbt.is_empty()); assert!(!raw_compressed_nbt.is_empty());
let decoder = GzDecoder::new(&raw_compressed_nbt[..]); let mut decoder = GzDecoder::new(&raw_compressed_nbt[..]);
let level_dat: LevelDat = from_bytes(decoder).expect("Failed to decode from file"); let mut buf = Vec::new();
decoder.read_to_end(&mut buf).unwrap();
let level_dat: LevelDat = from_bytes(Cursor::new(buf)).expect("Failed to decode from file");
assert_eq!(level_dat, *LEVEL_DAT); assert_eq!(level_dat, *LEVEL_DAT);
} }
@ -295,7 +301,7 @@ mod test {
assert!(!serialized.is_empty()); assert!(!serialized.is_empty());
let level_dat_again: LevelDat = let level_dat_again: LevelDat =
from_bytes(&serialized[..]).expect("Failed to decode from bytes"); from_bytes(Cursor::new(serialized)).expect("Failed to decode from bytes");
assert_eq!(level_dat_again, *LEVEL_DAT); assert_eq!(level_dat_again, *LEVEL_DAT);
} }