From 50a1649dea147ee3af1e2ee5cb6cf42635937fee Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Thu, 15 Oct 2026 04:06:42 +0200 Subject: [PATCH] Pack small palettes without hashing PalettedContainer::pack hashed all 4096 values with SipHash to build the palette. It now shares from_values' scan, which only switches to a map past 16 distinct values. Same palette order, same output. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-core/src/paletted.rs | 67 ++++++++++++++--------------- 1 file changed, 32 insertions(+), 35 deletions(-) diff --git a/crates/horizon-core/src/paletted.rs b/crates/horizon-core/src/paletted.rs index 9344d9392..f162404f5 100644 --- a/crates/horizon-core/src/paletted.rs +++ b/crates/horizon-core/src/paletted.rs @@ -151,6 +151,36 @@ fn write(words: &mut [u64], bits: u8, i: usize, value: u32) { let (word, shift) = locate(bits, i); words[word] = (words[word] & !(mask(bits) << shift)) | (u64::from(value) << shift); } +/// The distinct `values` in first-occurrence order and each value's index in +/// them. Up to 16 distinct values a scan finds them faster than hashing. +fn palette_ids(values: &[u32]) -> (Vec, Vec) { + let mut palette: Vec = Vec::new(); + let mut small = true; + let mut lookup: std::collections::HashMap = std::collections::HashMap::new(); + let mut ids = Vec::with_capacity(values.len()); + for &value in values { + let id = if small { + match palette.iter().position(|&v| v == value) { + Some(id) => id as u32, + None => { + palette.push(value); + if palette.len() > 16 { + small = false; + lookup.extend(palette.iter().enumerate().map(|(i, &v)| (v, i as u32))); + } + palette.len() as u32 - 1 + } + } + } else { + *lookup.entry(value).or_insert_with(|| { + palette.push(value); + palette.len() as u32 - 1 + }) + }; + ids.push(id); + } + (palette, ids) +} /// Words holding `ids` in `bits`-wide slots. fn pack_words(bits: u8, entries: usize, ids: impl Iterator) -> Vec { let mut words = vec![0u64; storage_len(bits, entries)]; @@ -403,31 +433,7 @@ impl PalettedContainer { /// container unpacked from disk. pub fn from_values(strategy: Strategy, values: &[u32]) -> Self { assert_eq!(values.len(), strategy.entries()); - let mut palette: Vec = Vec::new(); - let mut small = true; - let mut lookup: std::collections::HashMap = std::collections::HashMap::new(); - let mut ids = Vec::with_capacity(values.len()); - for &value in values { - let id = if small { - match palette.iter().position(|&v| v == value) { - Some(id) => id as u32, - None => { - palette.push(value); - if palette.len() > 16 { - small = false; - lookup.extend(palette.iter().enumerate().map(|(i, &v)| (v, i as u32))); - } - palette.len() as u32 - 1 - } - } - } else { - *lookup.entry(value).or_insert_with(|| { - palette.push(value); - palette.len() as u32 - 1 - }) - }; - ids.push(id); - } + let (palette, ids) = palette_ids(values); Self::build(strategy, palette, &ids) } /// A container over a distinct-value `palette` and an id per entry. @@ -530,18 +536,9 @@ impl PalettedContainer { /// re-encoded for the saved bit width. pub fn pack(&self) -> Packed { let entries = self.strategy.entries(); - let mut palette: Vec = Vec::new(); - let mut ids = Vec::with_capacity(entries); let mut values = vec![0u32; entries]; self.unpack(&mut values); - let mut lookup: std::collections::HashMap = std::collections::HashMap::new(); - for &value in &values { - let id = *lookup.entry(value).or_insert_with(|| { - palette.push(value); - palette.len() as u32 - 1 - }); - ids.push(id); - } + let (palette, ids) = palette_ids(&values); let bits = match self.strategy.config(ceil_log2(palette.len())) { Config::Single => 0, Config::List(bits) => bits,