Block registry: read facts.json as it streams

The game and the assets step each parsed facts.json into a JSON tree, then
copied each state into an NBT tree, before building the registry. With a
pack's 214,000 states each step peaked at 816 MB for a 139 MB registry.
The registry now deserializes the rows it needs straight from the file
(each row's state and its air and fluid columns, and the defaults):
registry peak 816 -> 153 MB, assets peak 816 -> 235 MB, RSS after the
assets step 1,344 -> 442 MB. Vanilla: 138 -> 27 MB.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ludovic Marie
2026-10-18 07:35:27 +02:00
parent 5b62d771bb
commit 9190d14b7f
4 changed files with 228 additions and 107 deletions
+16 -57
View File
@@ -2,14 +2,13 @@ use crate::Result;
use horizon_world::{
DATA_VERSION,
heightmap::HeightmapState,
nbt::Tag,
registry::{BlockRegistry, BlockStateId},
worldgen::{tags::TagDefinitions, terrain::TerrainBlockStates},
};
use serde_json::Value;
use std::{
collections::BTreeSet,
fs::{self, File},
fs::File,
io::{self, BufReader},
path::Path,
sync::Arc,
@@ -74,24 +73,6 @@ pub struct Assets {
fn invalid(message: &str) -> io::Error {
io::Error::new(io::ErrorKind::InvalidData, message)
}
fn tag(value: &Value) -> Result<Tag> {
Ok(match value {
Value::String(value) => Tag::String(value.as_str().into()),
Value::Number(value) => Tag::Int(i32::try_from(
value
.as_i64()
.ok_or_else(|| invalid("invalid registry number"))?,
)?),
Value::Array(values) => Tag::List(values.iter().map(tag).collect::<Result<_>>()?),
Value::Object(values) => Tag::Compound(
values
.iter()
.map(|(name, value)| Ok((name.as_str().into(), tag(value)?)))
.collect::<Result<_>>()?,
),
_ => return Err(invalid("unsupported registry value").into()),
})
}
impl Assets {
pub fn load(root: &Path) -> Result<Self> {
let manifest: Value = serde_json::from_reader(BufReader::new(File::open(
@@ -100,23 +81,8 @@ impl Assets {
if manifest["data_version"].as_i64() != Some(i64::from(DATA_VERSION)) {
return Err(invalid("resources do not match Horizon's pinned data version").into());
}
let facts: Value = serde_json::from_slice(&fs::read(root.join("facts.json"))?)?;
let states = facts["states"]
.as_array()
.ok_or_else(|| invalid("missing registry state facts"))?;
let registry = BlockRegistry::from_nbt(&Tag::compound([
("DataVersion", Tag::Int(DATA_VERSION)),
(
"states",
Tag::List(
states
.iter()
.map(|state| tag(&state[4]))
.collect::<Result<_>>()?,
),
),
("defaults", tag(&facts["defaults"])?),
]))?;
let (registry, flags) =
BlockRegistry::from_facts_json_with_flags(File::open(root.join("facts.json"))?)?;
let bundle: Value =
serde_json::from_reader(BufReader::new(File::open(root.join("block-tags.json"))?))?;
let mut definitions = TagDefinitions::default();
@@ -132,7 +98,7 @@ impl Assets {
)?;
}
let built =
definitions.build(|name, _| facts["defaults"].get(name).map(|_| name.to_owned()))?;
definitions.build(|name, _| registry.default_state(name).map(|_| name.to_owned()))?;
if !built.missing.is_empty() {
return Err(invalid("unresolved block tags").into());
}
@@ -158,27 +124,20 @@ impl Assets {
.cloned()
.collect();
let properties = Properties(Arc::new(
states
flags
.iter()
.map(|state| {
.enumerate()
.map(|(i, &(air, fluid))| {
let name = &registry
.state(BlockStateId(i as u32))
.ok_or_else(|| invalid("missing registry state"))?
.name;
Ok(StateFacts {
air: state[1]
.as_bool()
.ok_or_else(|| invalid("missing air fact"))?,
fluid: state[2]
.as_bool()
.ok_or_else(|| invalid("missing fluid fact"))?,
air_block: state[0].as_str() == Some("minecraft:air"),
motion: motion.contains(
state[0]
.as_str()
.ok_or_else(|| invalid("missing state name"))?,
),
no_leaves: no_leaves.contains(
state[0]
.as_str()
.ok_or_else(|| invalid("missing state name"))?,
),
air,
fluid,
air_block: name == "minecraft:air",
motion: motion.contains(name.as_str()),
no_leaves: no_leaves.contains(name.as_str()),
})
})
.collect::<Result<_>>()?,
+2 -38
View File
@@ -613,45 +613,9 @@ fn set_type(value: &Value, cache: &mut HashMap<String, Arc<str>>) -> Option<SetT
}
/// Build the block-state registry from the resources' `facts.json` (the
/// same source the native client uses).
/// same source the native client uses), streamed.
pub fn load_registry(root: &Path) -> io::Result<BlockRegistry> {
use crate::{DATA_VERSION, nbt::Tag};
fn tag(value: &Value) -> io::Result<Tag> {
Ok(match value {
Value::String(value) => Tag::String(value.as_str().into()),
Value::Number(value) => Tag::Int(
value
.as_i64()
.and_then(|v| i32::try_from(v).ok())
.ok_or_else(|| fail("invalid registry number"))?,
),
Value::Array(values) => Tag::List(values.iter().map(tag).collect::<io::Result<_>>()?),
Value::Object(values) => Tag::Compound(
values
.iter()
.map(|(name, value)| Ok((name.as_str().into(), tag(value)?)))
.collect::<io::Result<_>>()?,
),
_ => return Err(fail("unsupported registry value")),
})
}
let facts = read_json(&root.join("facts.json"))?;
let states = facts["states"]
.as_array()
.ok_or_else(|| fail("missing registry state facts"))?;
BlockRegistry::from_nbt(&Tag::compound([
("DataVersion", Tag::Int(DATA_VERSION)),
(
"states",
Tag::List(
states
.iter()
.map(|state| tag(&state[4]))
.collect::<io::Result<_>>()?,
),
),
("defaults", tag(&facts["defaults"])?),
]))
BlockRegistry::from_facts_json(File::open(root.join("facts.json"))?)
}
pub(super) fn load(registry: &BlockRegistry, root: &Path) -> io::Result<Behaviours> {
+46 -12
View File
@@ -1,5 +1,6 @@
//! Runtime block-state IDs imported from a version-pinned registry data export.
//! This supplies identities and default properties, not block behavior.
mod facts;
mod json;
use crate::{DATA_VERSION, chunk::BlockState, compression::invalid, nbt::Tag};
pub use json::BlockStateJsonCodec;
@@ -45,17 +46,53 @@ impl BlockRegistry {
let Some(Tag::Compound(defaults)) = tag.get("defaults") else {
return Err(invalid("missing registry defaults"));
};
if entries.is_empty() || entries.len() > i32::MAX as usize {
let states = entries
.iter()
.map(BlockState::from_nbt)
.collect::<io::Result<_>>()?;
let defaults = defaults
.iter()
.map(|(name, id)| {
let name = name.to_string().map_err(|e| invalid(e.to_string()))?;
let Tag::Int(id) = id else {
return Err(invalid("default state ID is not an int"));
};
Ok((name, i64::from(*id)))
})
.collect::<io::Result<_>>()?;
Self::from_states(states, defaults)
}
/// The registry of a resources `facts.json` (`{"states": [[..., ..., ...,
/// ..., state], ...], "defaults": {name: id}}`), read as it streams: no
/// JSON or NBT tree of the whole file is built. For a pack of 190,000
/// states the two trees were several hundred megabytes at startup.
pub fn from_facts_json(reader: impl io::Read) -> io::Result<Self> {
Ok(Self::from_facts_json_with_flags(reader)?.0)
}
/// [`Self::from_facts_json`], with each state's `(air, fluid)` columns.
pub fn from_facts_json_with_flags(
reader: impl io::Read,
) -> io::Result<(Self, Vec<(bool, bool)>)> {
let facts: facts::Facts = serde_json::from_reader(io::BufReader::new(reader))
.map_err(|e| invalid(e.to_string()))?;
let flags = facts.states.iter().map(|r| (r.air, r.fluid)).collect();
let states = facts.states.into_iter().map(|r| r.state).collect();
Ok((Self::from_states(states, facts.defaults)?, flags))
}
fn from_states(states: Vec<BlockState>, defaults: BTreeMap<String, i64>) -> io::Result<Self> {
if states.is_empty() || states.len() > i32::MAX as usize {
return Err(invalid("invalid registry size"));
}
let mut result = Self {
states: Vec::with_capacity(entries.len()),
ids: HashMap::with_capacity(entries.len()),
states: Vec::with_capacity(states.len()),
ids: HashMap::with_capacity(states.len()),
collided: BTreeMap::new(),
defaults: BTreeMap::new(),
};
for (i, entry) in entries.iter().enumerate() {
let state = BlockState::from_nbt(entry)?;
for (i, state) in states.into_iter().enumerate() {
if result.id(&state).is_some() {
return Err(invalid("duplicate block registry state"));
}
@@ -72,19 +109,16 @@ impl BlockRegistry {
result.states.push(state);
}
for (name, id) in defaults {
let name = name.to_string().map_err(|e| invalid(e.to_string()))?;
let Tag::Int(id) = id else {
return Err(invalid("default state ID is not an int"));
};
if *id < 0
if id < 0
|| id > i64::from(i32::MAX)
|| result
.states
.get(*id as usize)
.get(id as usize)
.is_none_or(|s| s.name != name)
{
return Err(invalid("invalid default block state ID"));
}
result.defaults.insert(name, BlockStateId(*id as u32));
result.defaults.insert(name, BlockStateId(id as u32));
}
let names: BTreeSet<_> = result.states.iter().map(|s| &s.name).collect();
if names.len() != result.defaults.len()
+164
View File
@@ -0,0 +1,164 @@
//! The part of `facts.json` the block registry reads, deserialized as it
//! streams: each row's state (its fifth column) and the defaults.
use crate::chunk::BlockState;
use serde::{
Deserialize, Deserializer,
de::{self, IgnoredAny, SeqAccess, Visitor},
};
use std::{collections::BTreeMap, fmt};
#[derive(Deserialize)]
pub(super) struct Facts {
pub states: Vec<Row>,
pub defaults: BTreeMap<String, i64>,
}
/// One `states` row: `[name, air, fluid, description, state, ...]`, keeping
/// the two flags and the state.
pub(super) struct Row {
pub air: bool,
pub fluid: bool,
pub state: BlockState,
}
impl<'de> Deserialize<'de> for Row {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
struct RowVisitor;
impl<'de> Visitor<'de> for RowVisitor {
type Value = Row;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("a registry state row")
}
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Row, A::Error> {
let length = |i| de::Error::invalid_length(i, &self);
seq.next_element::<IgnoredAny>()?.ok_or_else(|| length(0))?;
let air = seq.next_element::<bool>()?.ok_or_else(|| length(1))?;
let fluid = seq.next_element::<bool>()?.ok_or_else(|| length(2))?;
seq.next_element::<IgnoredAny>()?.ok_or_else(|| length(3))?;
let state = seq.next_element::<State>()?.ok_or_else(|| length(4))?;
while seq.next_element::<IgnoredAny>()?.is_some() {}
Ok(Row {
air,
fluid,
state: state.0,
})
}
}
deserializer.deserialize_seq(RowVisitor)
}
}
/// A state as BlockState::from_nbt reads it: a name, or `{"id", "properties"}`.
struct State(BlockState);
impl<'de> Deserialize<'de> for State {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
#[derive(Deserialize)]
#[serde(untagged)]
enum Shape {
Name(String),
Full {
id: String,
#[serde(default)]
properties: BTreeMap<String, String>,
},
}
Ok(State(match Shape::deserialize(deserializer)? {
Shape::Name(name) => BlockState {
name,
properties: BTreeMap::new(),
},
Shape::Full { id, properties } => BlockState {
name: id,
properties,
},
}))
}
}
#[cfg(test)]
mod tests {
use crate::{
DATA_VERSION,
chunk::BlockState,
nbt::Tag,
registry::{BlockRegistry, BlockStateId},
};
use serde_json::Value;
use std::path::Path;
#[test]
fn reads_states_and_defaults() {
let json = br#"{"defaults": {"minecraft:air": 0, "minecraft:oak_slab": 2},
"states": [
["minecraft:air", true, false, "Block{minecraft:air}", "minecraft:air"],
["minecraft:oak_slab", false, false, "x", {"id": "minecraft:oak_slab", "properties": {"type": "top"}}, "extra"],
["minecraft:oak_slab", false, true, "x", {"id": "minecraft:oak_slab", "properties": {"type": "bottom"}}]
], "other": [1, 2]}"#;
let (registry, flags) = BlockRegistry::from_facts_json_with_flags(&json[..]).unwrap();
assert_eq!(registry.len(), 3);
assert_eq!(flags, [(true, false), (false, false), (false, true)]);
assert_eq!(
registry.default_state("minecraft:oak_slab"),
Some(BlockStateId(2))
);
let top = BlockState {
name: "minecraft:oak_slab".into(),
properties: [("type".into(), "top".into())].into(),
};
assert_eq!(registry.id(&top), Some(BlockStateId(1)));
// A default naming another block, a short row, a missing default.
for bad in [
&br#"{"defaults": {"minecraft:air": 1}, "states": [["minecraft:air", true, false, "", "minecraft:air"], ["minecraft:stone", false, false, "", "minecraft:stone"]]}"#[..],
br#"{"defaults": {"minecraft:air": 0}, "states": [["minecraft:air", true, false, ""]]}"#,
br#"{"defaults": {}, "states": [["minecraft:air", true, false, "", "minecraft:air"]]}"#,
] {
assert!(BlockRegistry::from_facts_json(bad).is_err());
}
}
/// The streamed registry is the one the JSON and NBT trees built.
#[test]
fn matches_the_tree_built_registry() {
let path = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../.local/horizon-client/26.3/facts.json");
let Ok(bytes) = std::fs::read(&path) else {
eprintln!("skipped: no prepared facts.json");
return;
};
fn tag(v: &Value) -> Tag {
match v {
Value::String(s) => Tag::String(s.as_str().into()),
Value::Number(n) => Tag::Int(n.as_i64().unwrap() as i32),
Value::Object(o) => {
Tag::Compound(o.iter().map(|(k, v)| (k.as_str().into(), tag(v))).collect())
}
_ => unreachable!(),
}
}
let facts: Value = serde_json::from_slice(&bytes).unwrap();
let rows = facts["states"].as_array().unwrap();
let tree = BlockRegistry::from_nbt(&Tag::compound([
("DataVersion", Tag::Int(DATA_VERSION)),
(
"states",
Tag::List(rows.iter().map(|r| tag(&r[4])).collect()),
),
("defaults", tag(&facts["defaults"])),
]))
.unwrap();
let (streamed, flags) = BlockRegistry::from_facts_json_with_flags(&bytes[..]).unwrap();
assert_eq!(streamed.len(), tree.len());
for (i, row) in rows.iter().enumerate() {
let id = BlockStateId(i as u32);
assert_eq!(streamed.state(id), tree.state(id));
assert_eq!(
flags[i],
(row[1].as_bool().unwrap(), row[2].as_bool().unwrap())
);
}
for name in facts["defaults"].as_object().unwrap().keys() {
assert_eq!(streamed.default_state(name), tree.default_state(name));
}
}
}