From dbc246e40fff8bd984fc548865b8b645c37cee5d Mon Sep 17 00:00:00 2001 From: saik0 Date: Wed, 12 Jan 2022 04:30:20 -0800 Subject: [PATCH] create store module --- src/bitmap/container.rs | 7 +- src/bitmap/mod.rs | 2 - src/bitmap/serialization.rs | 13 +-- .../array_store.rs} | 96 +++++++++---------- .../{bitmap_8k.rs => store/bitmap_store.rs} | 54 +++++------ src/bitmap/{store.rs => store/mod.rs} | 23 ++++- 6 files changed, 100 insertions(+), 95 deletions(-) rename src/bitmap/{sorted_u16_vec.rs => store/array_store.rs} (85%) rename src/bitmap/{bitmap_8k.rs => store/bitmap_store.rs} (89%) rename src/bitmap/{store.rs => store/mod.rs} (97%) diff --git a/src/bitmap/container.rs b/src/bitmap/container.rs index af3a4e30..dcbb8c02 100644 --- a/src/bitmap/container.rs +++ b/src/bitmap/container.rs @@ -1,4 +1,3 @@ -use crate::bitmap::sorted_u16_vec::SortedU16Vec; use std::fmt; use std::ops::{ BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, RangeInclusive, Sub, SubAssign, @@ -22,7 +21,7 @@ pub struct Iter<'a> { impl Container { pub fn new(key: u16) -> Container { - Container { key, store: Store::Array(SortedU16Vec::new()) } + Container { key, store: Store::new() } } } @@ -97,12 +96,12 @@ impl Container { match &self.store { Store::Bitmap(ref bits) => { if bits.len() <= ARRAY_LIMIT { - self.store = bits.to_array_store() + self.store = Store::Array(bits.to_array_store()) } } Store::Array(ref vec) => { if vec.len() as u64 > ARRAY_LIMIT { - self.store = vec.to_bitmap_store() + self.store = Store::Bitmap(vec.to_bitmap_store()) } } }; diff --git a/src/bitmap/mod.rs b/src/bitmap/mod.rs index c875e5ae..e75cceb1 100644 --- a/src/bitmap/mod.rs +++ b/src/bitmap/mod.rs @@ -5,13 +5,11 @@ mod util; // Order of these modules matters as it determines the `impl` blocks order in // the docs -mod bitmap_8k; mod cmp; mod inherent; mod iter; mod ops; mod serialization; -mod sorted_u16_vec; use self::cmp::Pairs; pub use self::iter::IntoIter; diff --git a/src/bitmap/serialization.rs b/src/bitmap/serialization.rs index 86cb2dde..5fc9b3c8 100644 --- a/src/bitmap/serialization.rs +++ b/src/bitmap/serialization.rs @@ -1,12 +1,9 @@ use bytemuck::cast_slice_mut; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; -use std::convert::TryInto; use std::io; -use std::io::ErrorKind; use super::container::Container; -use super::store::Store; -use crate::bitmap::bitmap_8k::Bitmap8K; +use crate::bitmap::store::{ArrayStore, BitmapStore, Store}; use crate::RoaringBitmap; const SERIAL_COOKIE_NO_RUNCONTAINER: u32 = 12346; @@ -155,16 +152,12 @@ impl RoaringBitmap { let mut values = vec![0; len as usize]; reader.read_exact(cast_slice_mut(&mut values))?; values.iter_mut().for_each(|n| *n = u16::from_le(*n)); - let array = - values.try_into().map_err(|e| io::Error::new(ErrorKind::InvalidData, e))?; - Store::Array(array) + Store::Array(ArrayStore::from_vec_unchecked(values)) } else { let mut values = Box::new([0; 1024]); reader.read_exact(cast_slice_mut(&mut values[..]))?; values.iter_mut().for_each(|n| *n = u64::from_le(*n)); - let bitmap = Bitmap8K::try_from(len, values) - .map_err(|e| io::Error::new(ErrorKind::InvalidData, e))?; - Store::Bitmap(bitmap) + Store::Bitmap(BitmapStore::from_unchecked(len, values)) }; containers.push(Container { key, store }); diff --git a/src/bitmap/sorted_u16_vec.rs b/src/bitmap/store/array_store.rs similarity index 85% rename from src/bitmap/sorted_u16_vec.rs rename to src/bitmap/store/array_store.rs index 01c7710b..6e2b78d3 100644 --- a/src/bitmap/sorted_u16_vec.rs +++ b/src/bitmap/store/array_store.rs @@ -1,20 +1,19 @@ -use crate::bitmap::bitmap_8k::{bit, key, Bitmap8K, BITMAP_LENGTH}; -use crate::bitmap::store::Store; -use crate::bitmap::store::Store::Bitmap; use std::cmp::Ordering; use std::cmp::Ordering::*; use std::convert::{TryFrom, TryInto}; use std::fmt::{Display, Formatter}; use std::ops::{BitAnd, BitAndAssign, BitOr, BitXor, BitXorAssign, RangeInclusive, Sub, SubAssign}; +use super::bitmap_store::{bit, key, BitmapStore, BITMAP_LENGTH}; + #[derive(Clone, Eq, PartialEq)] -pub struct SortedU16Vec { +pub struct ArrayStore { vec: Vec, } -impl SortedU16Vec { - pub fn new() -> SortedU16Vec { - SortedU16Vec { vec: vec![] } +impl ArrayStore { + pub fn new() -> ArrayStore { + ArrayStore { vec: vec![] } } /// @@ -25,11 +24,11 @@ impl SortedU16Vec { /// # Panics /// /// When debug_assertions are enabled and the above invariants are not met - pub fn from_vec_unchecked(vec: Vec) -> SortedU16Vec { + pub fn from_vec_unchecked(vec: Vec) -> ArrayStore { if cfg!(debug_assertions) { vec.try_into().unwrap() } else { - SortedU16Vec { vec } + ArrayStore { vec } } } @@ -131,14 +130,14 @@ impl SortedU16Vec { } } - pub fn to_bitmap_store(&self) -> Store { + pub fn to_bitmap_store(&self) -> BitmapStore { let mut bits = Box::new([0; BITMAP_LENGTH]); let len = self.len() as u64; for &index in self.iter() { bits[key(index)] |= 1 << bit(index); } - Bitmap(Bitmap8K::from_unchecked(len, bits)) + BitmapStore::from_unchecked(len, bits) } pub fn len(&self) -> u64 { @@ -166,9 +165,9 @@ impl SortedU16Vec { } } -impl Default for SortedU16Vec { +impl Default for ArrayStore { fn default() -> Self { - SortedU16Vec::new() + ArrayStore::new() } } @@ -199,7 +198,7 @@ impl Display for Error { impl std::error::Error for Error {} -impl TryFrom> for SortedU16Vec { +impl TryFrom> for ArrayStore { type Error = Error; fn try_from(value: Vec) -> Result { @@ -215,12 +214,12 @@ impl TryFrom> for SortedU16Vec { } } - Ok(SortedU16Vec { vec }) + Ok(ArrayStore { vec: value }) } } -impl BitOr for &SortedU16Vec { - type Output = SortedU16Vec; +impl BitOr for &ArrayStore { + type Output = ArrayStore; fn bitor(self, rhs: Self) -> Self::Output { let mut vec = { @@ -255,12 +254,12 @@ impl BitOr for &SortedU16Vec { vec.extend_from_slice(&self.vec[i..]); vec.extend_from_slice(&rhs.vec[j..]); - SortedU16Vec { vec } + ArrayStore { vec } } } -impl BitAnd for &SortedU16Vec { - type Output = SortedU16Vec; +impl BitAnd for &ArrayStore { + type Output = ArrayStore; fn bitand(self, rhs: Self) -> Self::Output { let mut vec = Vec::new(); @@ -282,11 +281,11 @@ impl BitAnd for &SortedU16Vec { } } - SortedU16Vec { vec } + ArrayStore { vec } } } -impl BitAndAssign<&Self> for SortedU16Vec { +impl BitAndAssign<&Self> for ArrayStore { fn bitand_assign(&mut self, rhs: &Self) { let mut i = 0; self.vec.retain(|x| { @@ -296,14 +295,14 @@ impl BitAndAssign<&Self> for SortedU16Vec { } } -impl BitAndAssign<&Bitmap8K> for SortedU16Vec { - fn bitand_assign(&mut self, rhs: &Bitmap8K) { +impl BitAndAssign<&BitmapStore> for ArrayStore { + fn bitand_assign(&mut self, rhs: &BitmapStore) { self.vec.retain(|x| rhs.contains(*x)); } } -impl Sub for &SortedU16Vec { - type Output = SortedU16Vec; +impl Sub for &ArrayStore { + type Output = ArrayStore; fn sub(self, rhs: Self) -> Self::Output { let mut vec = Vec::new(); @@ -330,11 +329,11 @@ impl Sub for &SortedU16Vec { // Store remaining elements of the left array vec.extend_from_slice(&self.vec[i..]); - SortedU16Vec { vec } + ArrayStore { vec } } } -impl SubAssign<&Self> for SortedU16Vec { +impl SubAssign<&Self> for ArrayStore { fn sub_assign(&mut self, rhs: &Self) { let mut i = 0; self.vec.retain(|x| { @@ -344,14 +343,14 @@ impl SubAssign<&Self> for SortedU16Vec { } } -impl SubAssign<&Bitmap8K> for SortedU16Vec { - fn sub_assign(&mut self, rhs: &Bitmap8K) { +impl SubAssign<&BitmapStore> for ArrayStore { + fn sub_assign(&mut self, rhs: &BitmapStore) { self.vec.retain(|x| !rhs.contains(*x)); } } -impl BitXor for &SortedU16Vec { - type Output = SortedU16Vec; +impl BitXor for &ArrayStore { + type Output = ArrayStore; fn bitxor(self, rhs: Self) -> Self::Output { let mut vec = Vec::new(); @@ -382,11 +381,11 @@ impl BitXor for &SortedU16Vec { vec.extend_from_slice(&self.vec[i..]); vec.extend_from_slice(&rhs.vec[j..]); - SortedU16Vec { vec } + ArrayStore { vec } } } -impl BitXorAssign<&Self> for SortedU16Vec { +impl BitXorAssign<&Self> for ArrayStore { fn bitxor_assign(&mut self, rhs: &Self) { let mut i1 = 0usize; let mut iter2 = rhs.vec.iter(); @@ -418,25 +417,26 @@ impl BitXorAssign<&Self> for SortedU16Vec { #[cfg(test)] mod tests { use super::*; + use crate::bitmap::store::Store; fn into_vec(s: Store) -> Vec { match s { Store::Array(vec) => vec.vec, - Store::Bitmap(bits) => into_vec(bits.to_array_store()), + Store::Bitmap(bits) => bits.to_array_store().vec, } } fn into_bitmap_store(s: Store) -> Store { match s { - Store::Array(vec) => vec.to_bitmap_store(), - Bitmap(..) => s, + Store::Array(vec) => Store::Bitmap(vec.to_bitmap_store()), + Store::Bitmap(..) => s, } } #[test] #[allow(clippy::reversed_empty_ranges)] fn test_array_insert_invalid_range() { - let mut store = Store::Array(SortedU16Vec::from_vec_unchecked(vec![1, 2, 8, 9])); + let mut store = Store::Array(ArrayStore::from_vec_unchecked(vec![1, 2, 8, 9])); // Insert a range with start > end. let new = store.insert_range(6..=1); @@ -447,7 +447,7 @@ mod tests { #[test] fn test_array_insert_range() { - let mut store = Store::Array(SortedU16Vec::from_vec_unchecked(vec![1, 2, 8, 9])); + let mut store = Store::Array(ArrayStore::from_vec_unchecked(vec![1, 2, 8, 9])); let new = store.insert_range(4..=5); assert_eq!(new, 2); @@ -457,7 +457,7 @@ mod tests { #[test] fn test_array_insert_range_left_overlap() { - let mut store = Store::Array(SortedU16Vec::from_vec_unchecked(vec![1, 2, 8, 9])); + let mut store = Store::Array(ArrayStore::from_vec_unchecked(vec![1, 2, 8, 9])); let new = store.insert_range(2..=5); assert_eq!(new, 3); @@ -467,7 +467,7 @@ mod tests { #[test] fn test_array_insert_range_right_overlap() { - let mut store = Store::Array(SortedU16Vec::from_vec_unchecked(vec![1, 2, 8, 9])); + let mut store = Store::Array(ArrayStore::from_vec_unchecked(vec![1, 2, 8, 9])); let new = store.insert_range(4..=8); assert_eq!(new, 4); @@ -477,7 +477,7 @@ mod tests { #[test] fn test_array_insert_range_full_overlap() { - let mut store = Store::Array(SortedU16Vec::from_vec_unchecked(vec![1, 2, 8, 9])); + let mut store = Store::Array(ArrayStore::from_vec_unchecked(vec![1, 2, 8, 9])); let new = store.insert_range(1..=9); assert_eq!(new, 5); @@ -488,7 +488,7 @@ mod tests { #[test] #[allow(clippy::reversed_empty_ranges)] fn test_bitmap_insert_invalid_range() { - let store = Store::Array(SortedU16Vec::from_vec_unchecked(vec![1, 2, 8, 9])); + let store = Store::Array(ArrayStore::from_vec_unchecked(vec![1, 2, 8, 9])); let mut store = into_bitmap_store(store); // Insert a range with start > end. @@ -500,7 +500,7 @@ mod tests { #[test] fn test_bitmap_insert_same_key_overlap() { - let store = Store::Array(SortedU16Vec::from_vec_unchecked(vec![1, 2, 3, 62, 63])); + let store = Store::Array(ArrayStore::from_vec_unchecked(vec![1, 2, 3, 62, 63])); let mut store = into_bitmap_store(store); let new = store.insert_range(1..=62); @@ -511,7 +511,7 @@ mod tests { #[test] fn test_bitmap_insert_range() { - let store = Store::Array(SortedU16Vec::from_vec_unchecked(vec![1, 2, 130])); + let store = Store::Array(ArrayStore::from_vec_unchecked(vec![1, 2, 130])); let mut store = into_bitmap_store(store); let new = store.insert_range(4..=128); @@ -526,7 +526,7 @@ mod tests { #[test] fn test_bitmap_insert_range_left_overlap() { - let store = Store::Array(SortedU16Vec::from_vec_unchecked(vec![1, 2, 130])); + let store = Store::Array(ArrayStore::from_vec_unchecked(vec![1, 2, 130])); let mut store = into_bitmap_store(store); let new = store.insert_range(1..=128); @@ -541,7 +541,7 @@ mod tests { #[test] fn test_bitmap_insert_range_right_overlap() { - let store = Store::Array(SortedU16Vec::from_vec_unchecked(vec![1, 2, 130])); + let store = Store::Array(ArrayStore::from_vec_unchecked(vec![1, 2, 130])); let mut store = into_bitmap_store(store); let new = store.insert_range(4..=132); @@ -555,7 +555,7 @@ mod tests { #[test] fn test_bitmap_insert_range_full_overlap() { - let store = Store::Array(SortedU16Vec::from_vec_unchecked(vec![1, 2, 130])); + let store = Store::Array(ArrayStore::from_vec_unchecked(vec![1, 2, 130])); let mut store = into_bitmap_store(store); let new = store.insert_range(1..=134); diff --git a/src/bitmap/bitmap_8k.rs b/src/bitmap/store/bitmap_store.rs similarity index 89% rename from src/bitmap/bitmap_8k.rs rename to src/bitmap/store/bitmap_store.rs index ae858c29..4872de2e 100644 --- a/src/bitmap/bitmap_8k.rs +++ b/src/bitmap/store/bitmap_store.rs @@ -1,28 +1,28 @@ -use crate::bitmap::sorted_u16_vec::SortedU16Vec; -use crate::bitmap::store::Store; use std::borrow::Borrow; use std::fmt::{Display, Formatter}; use std::ops::{BitAndAssign, BitOrAssign, BitXorAssign, RangeInclusive, SubAssign}; +use super::ArrayStore; + pub const BITMAP_LENGTH: usize = 1024; #[derive(Clone)] -pub struct Bitmap8K { +pub struct BitmapStore { len: u64, bits: Box<[u64; BITMAP_LENGTH]>, } -impl Bitmap8K { - pub fn new() -> Bitmap8K { - Bitmap8K { len: 0, bits: Box::new([0; BITMAP_LENGTH]) } +impl BitmapStore { + pub fn new() -> BitmapStore { + BitmapStore { len: 0, bits: Box::new([0; BITMAP_LENGTH]) } } - pub fn try_from(len: u64, bits: Box<[u64; BITMAP_LENGTH]>) -> Result { + pub fn try_from(len: u64, bits: Box<[u64; BITMAP_LENGTH]>) -> Result { let actual_len = bits.iter().map(|v| v.count_ones() as u64).sum(); if len != actual_len { Err(Error { kind: ErrorKind::Cardinality { expected: len, actual: actual_len } }) } else { - Ok(Bitmap8K { len, bits }) + Ok(BitmapStore { len, bits }) } } @@ -34,11 +34,11 @@ impl Bitmap8K { /// # Panics /// /// When debug_assertions are enabled and the above invariant is not met - pub fn from_unchecked(len: u64, bits: Box<[u64; BITMAP_LENGTH]>) -> Bitmap8K { + pub fn from_unchecked(len: u64, bits: Box<[u64; BITMAP_LENGTH]>) -> BitmapStore { if cfg!(debug_assertions) { - Bitmap8K::try_from(len, bits).unwrap() + BitmapStore::try_from(len, bits).unwrap() } else { - Bitmap8K { len, bits } + BitmapStore { len, bits } } } @@ -161,7 +161,7 @@ impl Bitmap8K { self.bits[key(index)] & (1 << bit(index)) != 0 } - pub fn is_disjoint(&self, other: &Bitmap8K) -> bool { + pub fn is_disjoint(&self, other: &BitmapStore) -> bool { self.bits.iter().zip(other.bits.iter()).all(|(&i1, &i2)| (i1 & i2) == 0) } @@ -169,7 +169,7 @@ impl Bitmap8K { self.bits.iter().zip(other.bits.iter()).all(|(&i1, &i2)| (i1 & i2) == i1) } - pub fn to_array_store(&self) -> Store { + pub fn to_array_store(&self) -> ArrayStore { let mut vec = Vec::with_capacity(self.len as usize); for (index, mut bit) in self.bits.iter().cloned().enumerate() { while bit != 0 { @@ -177,7 +177,7 @@ impl Bitmap8K { bit &= bit - 1; } } - Store::Array(SortedU16Vec::from_vec_unchecked(vec)) + ArrayStore::from_vec_unchecked(vec) } pub fn len(&self) -> u64 { @@ -214,9 +214,9 @@ impl Bitmap8K { } } -impl Default for Bitmap8K { +impl Default for BitmapStore { fn default() -> Self { - Bitmap8K::new() + BitmapStore::new() } } @@ -289,7 +289,7 @@ pub fn bit(index: u16) -> usize { } #[inline] -fn op_bitmaps(bits1: &mut Bitmap8K, bits2: &Bitmap8K, op: impl Fn(&mut u64, u64)) { +fn op_bitmaps(bits1: &mut BitmapStore, bits2: &BitmapStore, op: impl Fn(&mut u64, u64)) { bits1.len = 0; for (index1, &index2) in bits1.bits.iter_mut().zip(bits2.bits.iter()) { op(index1, index2); @@ -297,14 +297,14 @@ fn op_bitmaps(bits1: &mut Bitmap8K, bits2: &Bitmap8K, op: impl Fn(&mut u64, u64) } } -impl BitOrAssign<&Self> for Bitmap8K { +impl BitOrAssign<&Self> for BitmapStore { fn bitor_assign(&mut self, rhs: &Self) { op_bitmaps(self, rhs, BitOrAssign::bitor_assign); } } -impl BitOrAssign<&SortedU16Vec> for Bitmap8K { - fn bitor_assign(&mut self, rhs: &SortedU16Vec) { +impl BitOrAssign<&ArrayStore> for BitmapStore { + fn bitor_assign(&mut self, rhs: &ArrayStore) { for &index in rhs.iter() { let (key, bit) = (key(index), bit(index)); let old_w = self.bits[key]; @@ -315,20 +315,20 @@ impl BitOrAssign<&SortedU16Vec> for Bitmap8K { } } -impl BitAndAssign<&Self> for Bitmap8K { +impl BitAndAssign<&Self> for BitmapStore { fn bitand_assign(&mut self, rhs: &Self) { op_bitmaps(self, rhs, BitAndAssign::bitand_assign); } } -impl SubAssign<&Self> for Bitmap8K { +impl SubAssign<&Self> for BitmapStore { fn sub_assign(&mut self, rhs: &Self) { op_bitmaps(self, rhs, |l, r| *l &= !r); } } -impl SubAssign<&SortedU16Vec> for Bitmap8K { - fn sub_assign(&mut self, rhs: &SortedU16Vec) { +impl SubAssign<&ArrayStore> for BitmapStore { + fn sub_assign(&mut self, rhs: &ArrayStore) { for &index in rhs.iter() { let (key, bit) = (key(index), bit(index)); let old_w = self.bits[key]; @@ -339,14 +339,14 @@ impl SubAssign<&SortedU16Vec> for Bitmap8K { } } -impl BitXorAssign<&Self> for Bitmap8K { +impl BitXorAssign<&Self> for BitmapStore { fn bitxor_assign(&mut self, rhs: &Self) { op_bitmaps(self, rhs, BitXorAssign::bitxor_assign); } } -impl BitXorAssign<&SortedU16Vec> for Bitmap8K { - fn bitxor_assign(&mut self, rhs: &SortedU16Vec) { +impl BitXorAssign<&ArrayStore> for BitmapStore { + fn bitxor_assign(&mut self, rhs: &ArrayStore) { let mut len = self.len as i64; for &index in rhs.iter() { let (key, bit) = (key(index), bit(index)); diff --git a/src/bitmap/store.rs b/src/bitmap/store/mod.rs similarity index 97% rename from src/bitmap/store.rs rename to src/bitmap/store/mod.rs index 149b0ad2..30c805b9 100644 --- a/src/bitmap/store.rs +++ b/src/bitmap/store/mod.rs @@ -1,17 +1,22 @@ -use crate::bitmap::bitmap_8k::{Bitmap8K, BitmapIter, BITMAP_LENGTH}; -use crate::bitmap::sorted_u16_vec::SortedU16Vec; +mod array_store; +mod bitmap_store; + use std::mem; use std::ops::{ BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, RangeInclusive, Sub, SubAssign, }; use std::{slice, vec}; +use self::bitmap_store::BITMAP_LENGTH; use self::Store::{Array, Bitmap}; +pub use self::array_store::ArrayStore; +pub use self::bitmap_store::{BitmapIter, BitmapStore}; + #[derive(Clone)] pub enum Store { - Array(SortedU16Vec), - Bitmap(Bitmap8K), + Array(ArrayStore), + Bitmap(BitmapStore), } pub enum Iter<'a> { @@ -22,6 +27,10 @@ pub enum Iter<'a> { } impl Store { + pub fn new() -> Store { + Store::Array(ArrayStore::new()) + } + pub fn insert(&mut self, index: u16) -> bool { match *self { Array(ref mut vec) => vec.insert(index), @@ -117,6 +126,12 @@ impl Store { } } +impl Default for Store { + fn default() -> Self { + Store::new() + } +} + impl BitOr<&Store> for &Store { type Output = Store;