Block registry: read facts.json as it streams
The game and the assets step each parsed facts.json into a JSON tree, then copied each state into an NBT tree, before building the registry. With a pack's 214,000 states each step peaked at 816 MB for a 139 MB registry. The registry now deserializes the rows it needs straight from the file (each row's state and its air and fluid columns, and the defaults): registry peak 816 -> 153 MB, assets peak 816 -> 235 MB, RSS after the assets step 1,344 -> 442 MB. Vanilla: 138 -> 27 MB. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -2,14 +2,13 @@ use crate::Result;
|
||||
use horizon_world::{
|
||||
DATA_VERSION,
|
||||
heightmap::HeightmapState,
|
||||
nbt::Tag,
|
||||
registry::{BlockRegistry, BlockStateId},
|
||||
worldgen::{tags::TagDefinitions, terrain::TerrainBlockStates},
|
||||
};
|
||||
use serde_json::Value;
|
||||
use std::{
|
||||
collections::BTreeSet,
|
||||
fs::{self, File},
|
||||
fs::File,
|
||||
io::{self, BufReader},
|
||||
path::Path,
|
||||
sync::Arc,
|
||||
@@ -74,24 +73,6 @@ pub struct Assets {
|
||||
fn invalid(message: &str) -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidData, message)
|
||||
}
|
||||
fn tag(value: &Value) -> Result<Tag> {
|
||||
Ok(match value {
|
||||
Value::String(value) => Tag::String(value.as_str().into()),
|
||||
Value::Number(value) => Tag::Int(i32::try_from(
|
||||
value
|
||||
.as_i64()
|
||||
.ok_or_else(|| invalid("invalid registry number"))?,
|
||||
)?),
|
||||
Value::Array(values) => Tag::List(values.iter().map(tag).collect::<Result<_>>()?),
|
||||
Value::Object(values) => Tag::Compound(
|
||||
values
|
||||
.iter()
|
||||
.map(|(name, value)| Ok((name.as_str().into(), tag(value)?)))
|
||||
.collect::<Result<_>>()?,
|
||||
),
|
||||
_ => return Err(invalid("unsupported registry value").into()),
|
||||
})
|
||||
}
|
||||
impl Assets {
|
||||
pub fn load(root: &Path) -> Result<Self> {
|
||||
let manifest: Value = serde_json::from_reader(BufReader::new(File::open(
|
||||
@@ -100,23 +81,8 @@ impl Assets {
|
||||
if manifest["data_version"].as_i64() != Some(i64::from(DATA_VERSION)) {
|
||||
return Err(invalid("resources do not match Horizon's pinned data version").into());
|
||||
}
|
||||
let facts: Value = serde_json::from_slice(&fs::read(root.join("facts.json"))?)?;
|
||||
let states = facts["states"]
|
||||
.as_array()
|
||||
.ok_or_else(|| invalid("missing registry state facts"))?;
|
||||
let registry = BlockRegistry::from_nbt(&Tag::compound([
|
||||
("DataVersion", Tag::Int(DATA_VERSION)),
|
||||
(
|
||||
"states",
|
||||
Tag::List(
|
||||
states
|
||||
.iter()
|
||||
.map(|state| tag(&state[4]))
|
||||
.collect::<Result<_>>()?,
|
||||
),
|
||||
),
|
||||
("defaults", tag(&facts["defaults"])?),
|
||||
]))?;
|
||||
let (registry, flags) =
|
||||
BlockRegistry::from_facts_json_with_flags(File::open(root.join("facts.json"))?)?;
|
||||
let bundle: Value =
|
||||
serde_json::from_reader(BufReader::new(File::open(root.join("block-tags.json"))?))?;
|
||||
let mut definitions = TagDefinitions::default();
|
||||
@@ -132,7 +98,7 @@ impl Assets {
|
||||
)?;
|
||||
}
|
||||
let built =
|
||||
definitions.build(|name, _| facts["defaults"].get(name).map(|_| name.to_owned()))?;
|
||||
definitions.build(|name, _| registry.default_state(name).map(|_| name.to_owned()))?;
|
||||
if !built.missing.is_empty() {
|
||||
return Err(invalid("unresolved block tags").into());
|
||||
}
|
||||
@@ -158,27 +124,20 @@ impl Assets {
|
||||
.cloned()
|
||||
.collect();
|
||||
let properties = Properties(Arc::new(
|
||||
states
|
||||
flags
|
||||
.iter()
|
||||
.map(|state| {
|
||||
.enumerate()
|
||||
.map(|(i, &(air, fluid))| {
|
||||
let name = ®istry
|
||||
.state(BlockStateId(i as u32))
|
||||
.ok_or_else(|| invalid("missing registry state"))?
|
||||
.name;
|
||||
Ok(StateFacts {
|
||||
air: state[1]
|
||||
.as_bool()
|
||||
.ok_or_else(|| invalid("missing air fact"))?,
|
||||
fluid: state[2]
|
||||
.as_bool()
|
||||
.ok_or_else(|| invalid("missing fluid fact"))?,
|
||||
air_block: state[0].as_str() == Some("minecraft:air"),
|
||||
motion: motion.contains(
|
||||
state[0]
|
||||
.as_str()
|
||||
.ok_or_else(|| invalid("missing state name"))?,
|
||||
),
|
||||
no_leaves: no_leaves.contains(
|
||||
state[0]
|
||||
.as_str()
|
||||
.ok_or_else(|| invalid("missing state name"))?,
|
||||
),
|
||||
air,
|
||||
fluid,
|
||||
air_block: name == "minecraft:air",
|
||||
motion: motion.contains(name.as_str()),
|
||||
no_leaves: no_leaves.contains(name.as_str()),
|
||||
})
|
||||
})
|
||||
.collect::<Result<_>>()?,
|
||||
|
||||
@@ -613,45 +613,9 @@ fn set_type(value: &Value, cache: &mut HashMap<String, Arc<str>>) -> Option<SetT
|
||||
}
|
||||
|
||||
/// Build the block-state registry from the resources' `facts.json` (the
|
||||
/// same source the native client uses).
|
||||
/// same source the native client uses), streamed.
|
||||
pub fn load_registry(root: &Path) -> io::Result<BlockRegistry> {
|
||||
use crate::{DATA_VERSION, nbt::Tag};
|
||||
fn tag(value: &Value) -> io::Result<Tag> {
|
||||
Ok(match value {
|
||||
Value::String(value) => Tag::String(value.as_str().into()),
|
||||
Value::Number(value) => Tag::Int(
|
||||
value
|
||||
.as_i64()
|
||||
.and_then(|v| i32::try_from(v).ok())
|
||||
.ok_or_else(|| fail("invalid registry number"))?,
|
||||
),
|
||||
Value::Array(values) => Tag::List(values.iter().map(tag).collect::<io::Result<_>>()?),
|
||||
Value::Object(values) => Tag::Compound(
|
||||
values
|
||||
.iter()
|
||||
.map(|(name, value)| Ok((name.as_str().into(), tag(value)?)))
|
||||
.collect::<io::Result<_>>()?,
|
||||
),
|
||||
_ => return Err(fail("unsupported registry value")),
|
||||
})
|
||||
}
|
||||
let facts = read_json(&root.join("facts.json"))?;
|
||||
let states = facts["states"]
|
||||
.as_array()
|
||||
.ok_or_else(|| fail("missing registry state facts"))?;
|
||||
BlockRegistry::from_nbt(&Tag::compound([
|
||||
("DataVersion", Tag::Int(DATA_VERSION)),
|
||||
(
|
||||
"states",
|
||||
Tag::List(
|
||||
states
|
||||
.iter()
|
||||
.map(|state| tag(&state[4]))
|
||||
.collect::<io::Result<_>>()?,
|
||||
),
|
||||
),
|
||||
("defaults", tag(&facts["defaults"])?),
|
||||
]))
|
||||
BlockRegistry::from_facts_json(File::open(root.join("facts.json"))?)
|
||||
}
|
||||
|
||||
pub(super) fn load(registry: &BlockRegistry, root: &Path) -> io::Result<Behaviours> {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
//! Runtime block-state IDs imported from a version-pinned registry data export.
|
||||
//! This supplies identities and default properties, not block behavior.
|
||||
mod facts;
|
||||
mod json;
|
||||
use crate::{DATA_VERSION, chunk::BlockState, compression::invalid, nbt::Tag};
|
||||
pub use json::BlockStateJsonCodec;
|
||||
@@ -45,17 +46,53 @@ impl BlockRegistry {
|
||||
let Some(Tag::Compound(defaults)) = tag.get("defaults") else {
|
||||
return Err(invalid("missing registry defaults"));
|
||||
};
|
||||
if entries.is_empty() || entries.len() > i32::MAX as usize {
|
||||
let states = entries
|
||||
.iter()
|
||||
.map(BlockState::from_nbt)
|
||||
.collect::<io::Result<_>>()?;
|
||||
let defaults = defaults
|
||||
.iter()
|
||||
.map(|(name, id)| {
|
||||
let name = name.to_string().map_err(|e| invalid(e.to_string()))?;
|
||||
let Tag::Int(id) = id else {
|
||||
return Err(invalid("default state ID is not an int"));
|
||||
};
|
||||
Ok((name, i64::from(*id)))
|
||||
})
|
||||
.collect::<io::Result<_>>()?;
|
||||
Self::from_states(states, defaults)
|
||||
}
|
||||
|
||||
/// The registry of a resources `facts.json` (`{"states": [[..., ..., ...,
|
||||
/// ..., state], ...], "defaults": {name: id}}`), read as it streams: no
|
||||
/// JSON or NBT tree of the whole file is built. For a pack of 190,000
|
||||
/// states the two trees were several hundred megabytes at startup.
|
||||
pub fn from_facts_json(reader: impl io::Read) -> io::Result<Self> {
|
||||
Ok(Self::from_facts_json_with_flags(reader)?.0)
|
||||
}
|
||||
|
||||
/// [`Self::from_facts_json`], with each state's `(air, fluid)` columns.
|
||||
pub fn from_facts_json_with_flags(
|
||||
reader: impl io::Read,
|
||||
) -> io::Result<(Self, Vec<(bool, bool)>)> {
|
||||
let facts: facts::Facts = serde_json::from_reader(io::BufReader::new(reader))
|
||||
.map_err(|e| invalid(e.to_string()))?;
|
||||
let flags = facts.states.iter().map(|r| (r.air, r.fluid)).collect();
|
||||
let states = facts.states.into_iter().map(|r| r.state).collect();
|
||||
Ok((Self::from_states(states, facts.defaults)?, flags))
|
||||
}
|
||||
|
||||
fn from_states(states: Vec<BlockState>, defaults: BTreeMap<String, i64>) -> io::Result<Self> {
|
||||
if states.is_empty() || states.len() > i32::MAX as usize {
|
||||
return Err(invalid("invalid registry size"));
|
||||
}
|
||||
let mut result = Self {
|
||||
states: Vec::with_capacity(entries.len()),
|
||||
ids: HashMap::with_capacity(entries.len()),
|
||||
states: Vec::with_capacity(states.len()),
|
||||
ids: HashMap::with_capacity(states.len()),
|
||||
collided: BTreeMap::new(),
|
||||
defaults: BTreeMap::new(),
|
||||
};
|
||||
for (i, entry) in entries.iter().enumerate() {
|
||||
let state = BlockState::from_nbt(entry)?;
|
||||
for (i, state) in states.into_iter().enumerate() {
|
||||
if result.id(&state).is_some() {
|
||||
return Err(invalid("duplicate block registry state"));
|
||||
}
|
||||
@@ -72,19 +109,16 @@ impl BlockRegistry {
|
||||
result.states.push(state);
|
||||
}
|
||||
for (name, id) in defaults {
|
||||
let name = name.to_string().map_err(|e| invalid(e.to_string()))?;
|
||||
let Tag::Int(id) = id else {
|
||||
return Err(invalid("default state ID is not an int"));
|
||||
};
|
||||
if *id < 0
|
||||
if id < 0
|
||||
|| id > i64::from(i32::MAX)
|
||||
|| result
|
||||
.states
|
||||
.get(*id as usize)
|
||||
.get(id as usize)
|
||||
.is_none_or(|s| s.name != name)
|
||||
{
|
||||
return Err(invalid("invalid default block state ID"));
|
||||
}
|
||||
result.defaults.insert(name, BlockStateId(*id as u32));
|
||||
result.defaults.insert(name, BlockStateId(id as u32));
|
||||
}
|
||||
let names: BTreeSet<_> = result.states.iter().map(|s| &s.name).collect();
|
||||
if names.len() != result.defaults.len()
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
//! The part of `facts.json` the block registry reads, deserialized as it
|
||||
//! streams: each row's state (its fifth column) and the defaults.
|
||||
use crate::chunk::BlockState;
|
||||
use serde::{
|
||||
Deserialize, Deserializer,
|
||||
de::{self, IgnoredAny, SeqAccess, Visitor},
|
||||
};
|
||||
use std::{collections::BTreeMap, fmt};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(super) struct Facts {
|
||||
pub states: Vec<Row>,
|
||||
pub defaults: BTreeMap<String, i64>,
|
||||
}
|
||||
|
||||
/// One `states` row: `[name, air, fluid, description, state, ...]`, keeping
|
||||
/// the two flags and the state.
|
||||
pub(super) struct Row {
|
||||
pub air: bool,
|
||||
pub fluid: bool,
|
||||
pub state: BlockState,
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for Row {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct RowVisitor;
|
||||
impl<'de> Visitor<'de> for RowVisitor {
|
||||
type Value = Row;
|
||||
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
f.write_str("a registry state row")
|
||||
}
|
||||
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Row, A::Error> {
|
||||
let length = |i| de::Error::invalid_length(i, &self);
|
||||
seq.next_element::<IgnoredAny>()?.ok_or_else(|| length(0))?;
|
||||
let air = seq.next_element::<bool>()?.ok_or_else(|| length(1))?;
|
||||
let fluid = seq.next_element::<bool>()?.ok_or_else(|| length(2))?;
|
||||
seq.next_element::<IgnoredAny>()?.ok_or_else(|| length(3))?;
|
||||
let state = seq.next_element::<State>()?.ok_or_else(|| length(4))?;
|
||||
while seq.next_element::<IgnoredAny>()?.is_some() {}
|
||||
Ok(Row {
|
||||
air,
|
||||
fluid,
|
||||
state: state.0,
|
||||
})
|
||||
}
|
||||
}
|
||||
deserializer.deserialize_seq(RowVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
/// A state as BlockState::from_nbt reads it: a name, or `{"id", "properties"}`.
|
||||
struct State(BlockState);
|
||||
|
||||
impl<'de> Deserialize<'de> for State {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum Shape {
|
||||
Name(String),
|
||||
Full {
|
||||
id: String,
|
||||
#[serde(default)]
|
||||
properties: BTreeMap<String, String>,
|
||||
},
|
||||
}
|
||||
Ok(State(match Shape::deserialize(deserializer)? {
|
||||
Shape::Name(name) => BlockState {
|
||||
name,
|
||||
properties: BTreeMap::new(),
|
||||
},
|
||||
Shape::Full { id, properties } => BlockState {
|
||||
name: id,
|
||||
properties,
|
||||
},
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::{
|
||||
DATA_VERSION,
|
||||
chunk::BlockState,
|
||||
nbt::Tag,
|
||||
registry::{BlockRegistry, BlockStateId},
|
||||
};
|
||||
use serde_json::Value;
|
||||
use std::path::Path;
|
||||
|
||||
#[test]
|
||||
fn reads_states_and_defaults() {
|
||||
let json = br#"{"defaults": {"minecraft:air": 0, "minecraft:oak_slab": 2},
|
||||
"states": [
|
||||
["minecraft:air", true, false, "Block{minecraft:air}", "minecraft:air"],
|
||||
["minecraft:oak_slab", false, false, "x", {"id": "minecraft:oak_slab", "properties": {"type": "top"}}, "extra"],
|
||||
["minecraft:oak_slab", false, true, "x", {"id": "minecraft:oak_slab", "properties": {"type": "bottom"}}]
|
||||
], "other": [1, 2]}"#;
|
||||
let (registry, flags) = BlockRegistry::from_facts_json_with_flags(&json[..]).unwrap();
|
||||
assert_eq!(registry.len(), 3);
|
||||
assert_eq!(flags, [(true, false), (false, false), (false, true)]);
|
||||
assert_eq!(
|
||||
registry.default_state("minecraft:oak_slab"),
|
||||
Some(BlockStateId(2))
|
||||
);
|
||||
let top = BlockState {
|
||||
name: "minecraft:oak_slab".into(),
|
||||
properties: [("type".into(), "top".into())].into(),
|
||||
};
|
||||
assert_eq!(registry.id(&top), Some(BlockStateId(1)));
|
||||
// A default naming another block, a short row, a missing default.
|
||||
for bad in [
|
||||
&br#"{"defaults": {"minecraft:air": 1}, "states": [["minecraft:air", true, false, "", "minecraft:air"], ["minecraft:stone", false, false, "", "minecraft:stone"]]}"#[..],
|
||||
br#"{"defaults": {"minecraft:air": 0}, "states": [["minecraft:air", true, false, ""]]}"#,
|
||||
br#"{"defaults": {}, "states": [["minecraft:air", true, false, "", "minecraft:air"]]}"#,
|
||||
] {
|
||||
assert!(BlockRegistry::from_facts_json(bad).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
/// The streamed registry is the one the JSON and NBT trees built.
|
||||
#[test]
|
||||
fn matches_the_tree_built_registry() {
|
||||
let path = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../.local/horizon-client/26.3/facts.json");
|
||||
let Ok(bytes) = std::fs::read(&path) else {
|
||||
eprintln!("skipped: no prepared facts.json");
|
||||
return;
|
||||
};
|
||||
fn tag(v: &Value) -> Tag {
|
||||
match v {
|
||||
Value::String(s) => Tag::String(s.as_str().into()),
|
||||
Value::Number(n) => Tag::Int(n.as_i64().unwrap() as i32),
|
||||
Value::Object(o) => {
|
||||
Tag::Compound(o.iter().map(|(k, v)| (k.as_str().into(), tag(v))).collect())
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
let facts: Value = serde_json::from_slice(&bytes).unwrap();
|
||||
let rows = facts["states"].as_array().unwrap();
|
||||
let tree = BlockRegistry::from_nbt(&Tag::compound([
|
||||
("DataVersion", Tag::Int(DATA_VERSION)),
|
||||
(
|
||||
"states",
|
||||
Tag::List(rows.iter().map(|r| tag(&r[4])).collect()),
|
||||
),
|
||||
("defaults", tag(&facts["defaults"])),
|
||||
]))
|
||||
.unwrap();
|
||||
let (streamed, flags) = BlockRegistry::from_facts_json_with_flags(&bytes[..]).unwrap();
|
||||
assert_eq!(streamed.len(), tree.len());
|
||||
for (i, row) in rows.iter().enumerate() {
|
||||
let id = BlockStateId(i as u32);
|
||||
assert_eq!(streamed.state(id), tree.state(id));
|
||||
assert_eq!(
|
||||
flags[i],
|
||||
(row[1].as_bool().unwrap(), row[2].as_bool().unwrap())
|
||||
);
|
||||
}
|
||||
for name in facts["defaults"].as_object().unwrap().keys() {
|
||||
assert_eq!(streamed.default_state(name), tree.default_state(name));
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user