From b27f60249f1275b685edaff3cb95fab417fa1150 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 12 Dec 2019 22:12:20 +0100 Subject: [PATCH 01/18] Replace ElasticArray with SmallVec --- kvdb-rocksdb/Cargo.toml | 2 +- kvdb-rocksdb/src/lib.rs | 7 ++--- kvdb/Cargo.toml | 2 +- kvdb/src/lib.rs | 25 ++++++++-------- parity-util-mem/Cargo.toml | 4 +-- parity-util-mem/src/impls.rs | 56 ++++++++++++++++++++---------------- 6 files changed, 51 insertions(+), 45 deletions(-) diff --git a/kvdb-rocksdb/Cargo.toml b/kvdb-rocksdb/Cargo.toml index 2049da4ba..b4954ae05 100644 --- a/kvdb-rocksdb/Cargo.toml +++ b/kvdb-rocksdb/Cargo.toml @@ -12,7 +12,7 @@ name = "bench_read_perf" harness = false [dependencies] -elastic-array = "0.10.2" +smallvec = "1.0.0" fs-swap = "0.2.4" interleaved-ordered = "0.1.1" kvdb = { path = "../kvdb", version = "0.1" } diff --git a/kvdb-rocksdb/src/lib.rs b/kvdb-rocksdb/src/lib.rs index df7b9f833..f94856c92 100644 --- a/kvdb-rocksdb/src/lib.rs +++ b/kvdb-rocksdb/src/lib.rs @@ -24,10 +24,9 @@ use rocksdb::{ }; use crate::iter::KeyValuePair; -use elastic_array::ElasticArray32; use fs_swap::{swap, swap_nonatomic}; use interleaved_ordered::interleave_ordered; -use kvdb::{DBOp, DBTransaction, DBValue, KeyValueDB}; +use kvdb::{DBOp, DBTransaction, DBValue, DBKey, KeyValueDB}; use log::{debug, warn}; #[cfg(target_os = "linux")] @@ -240,9 +239,9 @@ pub struct Database { read_opts: ReadOptions, block_opts: BlockBasedOptions, // Dirty values added with `write_buffered`. Cleaned on `flush`. - overlay: RwLock, KeyState>>>, + overlay: RwLock>>, // Values currently being flushed. Cleared when `flush` completes. - flushing: RwLock, KeyState>>>, + flushing: RwLock>>, // Prevents concurrent flushes. // Value indicates if a flush is in progress. flushing_lock: Mutex, diff --git a/kvdb/Cargo.toml b/kvdb/Cargo.toml index db7badb98..2f8db5964 100644 --- a/kvdb/Cargo.toml +++ b/kvdb/Cargo.toml @@ -8,5 +8,5 @@ license = "GPL-3.0" edition = "2018" [dependencies] -elastic-array = "0.10.2" +smallvec = "1.0.0" bytes = { package = "parity-bytes", version = "0.1", path = "../parity-bytes" } diff --git a/kvdb/src/lib.rs b/kvdb/src/lib.rs index 46c53e5f0..d16180d8f 100644 --- a/kvdb/src/lib.rs +++ b/kvdb/src/lib.rs @@ -17,7 +17,7 @@ //! Key-Value store abstraction with `RocksDB` backend. use bytes::Bytes; -use elastic_array::{ElasticArray128, ElasticArray32}; +use smallvec::SmallVec; use std::io; use std::path::Path; use std::sync::Arc; @@ -26,7 +26,9 @@ use std::sync::Arc; pub const PREFIX_LEN: usize = 12; /// Database value. -pub type DBValue = ElasticArray128; +pub type DBValue = SmallVec<[u8; 128]>; +// todo[dvdplm] is this type useful or harmful? +pub type DBKey = SmallVec<[u8; 32]>; /// Write transaction. Batches a sequence of put/delete operations for efficiency. #[derive(Default, Clone, PartialEq)] @@ -38,8 +40,8 @@ pub struct DBTransaction { /// Database operation. #[derive(Clone, PartialEq)] pub enum DBOp { - Insert { col: Option, key: ElasticArray32, value: DBValue }, - Delete { col: Option, key: ElasticArray32 }, + Insert { col: Option, key: DBKey, value: DBValue }, + Delete { col: Option, key: DBKey }, } impl DBOp { @@ -73,23 +75,20 @@ impl DBTransaction { /// Insert a key-value pair in the transaction. Any existing value will be overwritten upon write. pub fn put(&mut self, col: Option, key: &[u8], value: &[u8]) { - let mut ekey = ElasticArray32::new(); - ekey.append_slice(key); - self.ops.push(DBOp::Insert { col: col, key: ekey, value: DBValue::from_slice(value) }); + // todo[dvdplm] can avoid copy with `from_buf()` here? + self.ops.push(DBOp::Insert { col, key: DBKey::from_slice(key), value: DBValue::from_slice(value) }) } /// Insert a key-value pair in the transaction. Any existing value will be overwritten upon write. pub fn put_vec(&mut self, col: Option, key: &[u8], value: Bytes) { - let mut ekey = ElasticArray32::new(); - ekey.append_slice(key); - self.ops.push(DBOp::Insert { col: col, key: ekey, value: DBValue::from_vec(value) }); + // todo[dvdplm] can avoid copy with `from_buf()` here? + self.ops.push(DBOp::Insert { col, key: DBKey::from_slice(key), value: DBValue::from_vec(value) }); } /// Delete value by key. pub fn delete(&mut self, col: Option, key: &[u8]) { - let mut ekey = ElasticArray32::new(); - ekey.append_slice(key); - self.ops.push(DBOp::Delete { col: col, key: ekey }); + // todo[dvdplm] can avoid copy with `from_buf()` here? + self.ops.push(DBOp::Delete { col, key: DBKey::from_slice(key) }); } } diff --git a/parity-util-mem/Cargo.toml b/parity-util-mem/Cargo.toml index cf3dab6f5..f46650d14 100644 --- a/parity-util-mem/Cargo.toml +++ b/parity-util-mem/Cargo.toml @@ -20,7 +20,7 @@ wee_alloc = { version = "0.4.5", optional = true } mimallocator = { version = "0.1.3", features = ["secure"], optional = true } mimalloc-sys = { version = "0.1.6", optional = true } -elastic-array = { version = "0.10.2", optional = true } +smallvec = { version = "1.0.0", optional = true } ethereum-types = { version = "0.8.0", optional = true, path = "../ethereum-types" } parking_lot = { version = "0.9.0", optional = true } @@ -43,6 +43,6 @@ jemalloc-global = ["jemallocator"] # use mimalloc as global allocator mimalloc-global = ["mimallocator", "mimalloc-sys"] # implement additional types -ethereum-impls = ["ethereum-types", "elastic-array", "parking_lot"] +ethereum-impls = ["ethereum-types", "parking_lot", "smallvec"] # Full estimate: no call to allocator estimate-heapsize = [] diff --git a/parity-util-mem/src/impls.rs b/parity-util-mem/src/impls.rs index ca36ce193..b34ed9610 100644 --- a/parity-util-mem/src/impls.rs +++ b/parity-util-mem/src/impls.rs @@ -15,15 +15,13 @@ // along with Parity. If not, see . //! Implementation of `MallocSize` for common types : -//! - etheureum types uint and fixed hash. -//! - elastic_array arrays +//! - ethereum types uint and fixed hash. +//! - smallvec arrays of sizes 32, 36, 64, 128 and 256 //! - parking_lot mutex structures use super::{MallocSizeOf, MallocSizeOfOps}; -use elastic_array::{ - ElasticArray1024, ElasticArray128, ElasticArray16, ElasticArray2, ElasticArray2048, ElasticArray256, - ElasticArray32, ElasticArray36, ElasticArray4, ElasticArray512, ElasticArray64, ElasticArray8, -}; + +use smallvec::SmallVec; use ethereum_types::{Bloom, H128, H160, H256, H264, H32, H512, H520, H64, U128, U256, U512, U64}; use parking_lot::{Mutex, RwLock}; @@ -36,31 +34,23 @@ malloc_size_of_is_0!(std::time::Duration); malloc_size_of_is_0!(U64, U128, U256, U512, H32, H64, H128, H160, H256, H264, H512, H520, Bloom); -macro_rules! impl_elastic_array { - ($name: ident, $dummy: ident, $size: expr) => { - impl MallocSizeOf for $name - where - T: MallocSizeOf, - { +macro_rules! impl_smallvec { + ($size: expr) => { + impl MallocSizeOf for SmallVec<[T; $size]> where T: MallocSizeOf { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + // todo[dvdplm] this will always be `0` because it resolves to the `[T]` impl. Can do better than that? self[..].size_of(ops) } } }; } -impl_elastic_array!(ElasticArray2, ElasticArray2Dummy, 2); -impl_elastic_array!(ElasticArray4, ElasticArray4Dummy, 4); -impl_elastic_array!(ElasticArray8, ElasticArray8Dummy, 8); -impl_elastic_array!(ElasticArray16, ElasticArray16Dummy, 16); -impl_elastic_array!(ElasticArray32, ElasticArray32Dummy, 32); -impl_elastic_array!(ElasticArray36, ElasticArray36Dummy, 36); -impl_elastic_array!(ElasticArray64, ElasticArray64Dummy, 64); -impl_elastic_array!(ElasticArray128, ElasticArray128Dummy, 128); -impl_elastic_array!(ElasticArray256, ElasticArray256Dummy, 256); -impl_elastic_array!(ElasticArray512, ElasticArray512Dummy, 512); -impl_elastic_array!(ElasticArray1024, ElasticArray1024Dummy, 1024); -impl_elastic_array!(ElasticArray2048, ElasticArray2048Dummy, 2048); +// todo[dvdplm]: check if we really need all these impls. +impl_smallvec!(32); +impl_smallvec!(36); // trie-db use this +impl_smallvec!(64); +impl_smallvec!(128); +impl_smallvec!(256); impl MallocSizeOf for Mutex { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { @@ -73,3 +63,21 @@ impl MallocSizeOf for RwLock { self.read().size_of(ops) } } + +#[cfg(test)] +mod tests { + use smallvec::SmallVec; + use crate::{MallocSizeOf, allocators::new_malloc_size_ops}; + + #[test] + fn test_smallvec() { + let mut v: SmallVec<[u8; 2]> = SmallVec::new(); + let mut ops = new_malloc_size_ops(); + assert_eq!(v.size_of(&mut ops), 0); + v.push(1); + v.push(2); + assert_eq!(v.size_of(&mut ops), 0); + v.push(3); + assert_eq!(v.size_of(&mut ops), 0); // todo[dvdplm] why isn't this 1? use `spilled()` and to see if we allocated a `Vec`. + } +} From 1c361a858127310092f0a1cda54823f07f26fae5 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 13 Dec 2019 14:13:13 +0100 Subject: [PATCH 02/18] Resolve todos --- kvdb/src/lib.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/kvdb/src/lib.rs b/kvdb/src/lib.rs index d16180d8f..473137ebf 100644 --- a/kvdb/src/lib.rs +++ b/kvdb/src/lib.rs @@ -27,7 +27,7 @@ pub const PREFIX_LEN: usize = 12; /// Database value. pub type DBValue = SmallVec<[u8; 128]>; -// todo[dvdplm] is this type useful or harmful? +/// Database keys. pub type DBKey = SmallVec<[u8; 32]>; /// Write transaction. Batches a sequence of put/delete operations for efficiency. @@ -75,19 +75,16 @@ impl DBTransaction { /// Insert a key-value pair in the transaction. Any existing value will be overwritten upon write. pub fn put(&mut self, col: Option, key: &[u8], value: &[u8]) { - // todo[dvdplm] can avoid copy with `from_buf()` here? self.ops.push(DBOp::Insert { col, key: DBKey::from_slice(key), value: DBValue::from_slice(value) }) } /// Insert a key-value pair in the transaction. Any existing value will be overwritten upon write. pub fn put_vec(&mut self, col: Option, key: &[u8], value: Bytes) { - // todo[dvdplm] can avoid copy with `from_buf()` here? self.ops.push(DBOp::Insert { col, key: DBKey::from_slice(key), value: DBValue::from_vec(value) }); } /// Delete value by key. pub fn delete(&mut self, col: Option, key: &[u8]) { - // todo[dvdplm] can avoid copy with `from_buf()` here? self.ops.push(DBOp::Delete { col, key: DBKey::from_slice(key) }); } } From cb7713c4d6d95e7839827f7a2b9409f3f094bfb0 Mon Sep 17 00:00:00 2001 From: David Palm Date: Sun, 15 Dec 2019 15:28:36 +0100 Subject: [PATCH 03/18] Fix formatting --- kvdb-rocksdb/src/lib.rs | 2 +- parity-util-mem/src/impls.rs | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/kvdb-rocksdb/src/lib.rs b/kvdb-rocksdb/src/lib.rs index 16952c52c..dc7653f6d 100644 --- a/kvdb-rocksdb/src/lib.rs +++ b/kvdb-rocksdb/src/lib.rs @@ -26,7 +26,7 @@ use rocksdb::{ use crate::iter::KeyValuePair; use fs_swap::{swap, swap_nonatomic}; use interleaved_ordered::interleave_ordered; -use kvdb::{DBOp, DBTransaction, DBValue, DBKey, KeyValueDB}; +use kvdb::{DBKey, DBOp, DBTransaction, DBValue, KeyValueDB}; use log::{debug, warn}; #[cfg(target_os = "linux")] diff --git a/parity-util-mem/src/impls.rs b/parity-util-mem/src/impls.rs index b34ed9610..f76ea26fb 100644 --- a/parity-util-mem/src/impls.rs +++ b/parity-util-mem/src/impls.rs @@ -21,9 +21,9 @@ use super::{MallocSizeOf, MallocSizeOfOps}; -use smallvec::SmallVec; use ethereum_types::{Bloom, H128, H160, H256, H264, H32, H512, H520, H64, U128, U256, U512, U64}; use parking_lot::{Mutex, RwLock}; +use smallvec::SmallVec; #[cfg(not(feature = "std"))] use core as std; @@ -36,7 +36,10 @@ malloc_size_of_is_0!(U64, U128, U256, U512, H32, H64, H128, H160, H256, H264, H5 macro_rules! impl_smallvec { ($size: expr) => { - impl MallocSizeOf for SmallVec<[T; $size]> where T: MallocSizeOf { + impl MallocSizeOf for SmallVec<[T; $size]> + where + T: MallocSizeOf, + { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { // todo[dvdplm] this will always be `0` because it resolves to the `[T]` impl. Can do better than that? self[..].size_of(ops) @@ -66,8 +69,8 @@ impl MallocSizeOf for RwLock { #[cfg(test)] mod tests { + use crate::{allocators::new_malloc_size_ops, MallocSizeOf}; use smallvec::SmallVec; - use crate::{MallocSizeOf, allocators::new_malloc_size_ops}; #[test] fn test_smallvec() { From 83c139a33832a4d664e3d51c1f671aa995b41c4d Mon Sep 17 00:00:00 2001 From: David Palm Date: Sun, 15 Dec 2019 16:25:11 +0100 Subject: [PATCH 04/18] Attempt a sane impl of MallocSizeOf for SmallVec --- parity-util-mem/src/impls.rs | 29 +++++++++++++++++------------ parity-util-mem/src/malloc_size.rs | 2 ++ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/parity-util-mem/src/impls.rs b/parity-util-mem/src/impls.rs index f76ea26fb..6e9f19404 100644 --- a/parity-util-mem/src/impls.rs +++ b/parity-util-mem/src/impls.rs @@ -40,20 +40,20 @@ macro_rules! impl_smallvec { where T: MallocSizeOf, { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - // todo[dvdplm] this will always be `0` because it resolves to the `[T]` impl. Can do better than that? - self[..].size_of(ops) + fn size_of(&self, _: &mut MallocSizeOfOps) -> usize { + if self.spilled() { + self.capacity() * core::mem::size_of::() + } else { + 0 + } } } }; } // todo[dvdplm]: check if we really need all these impls. -impl_smallvec!(32); -impl_smallvec!(36); // trie-db use this -impl_smallvec!(64); -impl_smallvec!(128); -impl_smallvec!(256); +impl_smallvec!(32); // kvdb uses this +impl_smallvec!(36); // trie-db uses this impl MallocSizeOf for Mutex { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { @@ -69,18 +69,23 @@ impl MallocSizeOf for RwLock { #[cfg(test)] mod tests { - use crate::{allocators::new_malloc_size_ops, MallocSizeOf}; + use crate::{allocators::new_malloc_size_ops, MallocSizeOf, MallocSizeOfOps}; use smallvec::SmallVec; + impl_smallvec!(3); #[test] fn test_smallvec() { - let mut v: SmallVec<[u8; 2]> = SmallVec::new(); + let mut v: SmallVec<[u8; 3]> = SmallVec::new(); let mut ops = new_malloc_size_ops(); assert_eq!(v.size_of(&mut ops), 0); v.push(1); v.push(2); - assert_eq!(v.size_of(&mut ops), 0); v.push(3); - assert_eq!(v.size_of(&mut ops), 0); // todo[dvdplm] why isn't this 1? use `spilled()` and to see if we allocated a `Vec`. + assert_eq!(v.size_of(&mut ops), 0); + assert!(!v.spilled()); + v.push(4); + assert!(v.spilled(), "SmallVec spills when going beyond the capacity of the inner backing array"); + assert_eq!(v.len(), 4); + assert_eq!(v.size_of(&mut ops), 4); } } diff --git a/parity-util-mem/src/malloc_size.rs b/parity-util-mem/src/malloc_size.rs index a3018af1f..6a7717a55 100644 --- a/parity-util-mem/src/malloc_size.rs +++ b/parity-util-mem/src/malloc_size.rs @@ -344,6 +344,7 @@ where impl MallocSizeOf for [T] { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + println!("[T] IMPL"); let mut n = 0; for elem in self.iter() { n += elem.size_of(ops); @@ -355,6 +356,7 @@ impl MallocSizeOf for [T] { impl MallocSizeOf for Vec { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { let mut n = self.shallow_size_of(ops); + println!("VEC IMPL, shallow size: {:?}", n); for elem in self.iter() { n += elem.size_of(ops); } From 28f46f28c87ea64c8c3a2a4929836fd094fa8eca Mon Sep 17 00:00:00 2001 From: David Palm Date: Sun, 15 Dec 2019 16:28:27 +0100 Subject: [PATCH 05/18] remove debug code --- parity-util-mem/src/malloc_size.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/parity-util-mem/src/malloc_size.rs b/parity-util-mem/src/malloc_size.rs index 6a7717a55..a3018af1f 100644 --- a/parity-util-mem/src/malloc_size.rs +++ b/parity-util-mem/src/malloc_size.rs @@ -344,7 +344,6 @@ where impl MallocSizeOf for [T] { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - println!("[T] IMPL"); let mut n = 0; for elem in self.iter() { n += elem.size_of(ops); @@ -356,7 +355,6 @@ impl MallocSizeOf for [T] { impl MallocSizeOf for Vec { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { let mut n = self.shallow_size_of(ops); - println!("VEC IMPL, shallow size: {:?}", n); for elem in self.iter() { n += elem.size_of(ops); } From 6e2745d2183fb55417b7d5c6378bac97ba73009e Mon Sep 17 00:00:00 2001 From: David Palm Date: Sun, 15 Dec 2019 20:59:54 +0100 Subject: [PATCH 06/18] More tests for MallocSizeOf impl for SmallVec --- parity-util-mem/src/impls.rs | 46 +++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/parity-util-mem/src/impls.rs b/parity-util-mem/src/impls.rs index 6e9f19404..867e10cda 100644 --- a/parity-util-mem/src/impls.rs +++ b/parity-util-mem/src/impls.rs @@ -40,18 +40,17 @@ macro_rules! impl_smallvec { where T: MallocSizeOf, { - fn size_of(&self, _: &mut MallocSizeOfOps) -> usize { - if self.spilled() { - self.capacity() * core::mem::size_of::() - } else { - 0 + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = 0; + for elem in self.iter() { + n += elem.size_of(ops); } + n } } }; } -// todo[dvdplm]: check if we really need all these impls. impl_smallvec!(32); // kvdb uses this impl_smallvec!(36); // trie-db uses this @@ -74,7 +73,7 @@ mod tests { impl_smallvec!(3); #[test] - fn test_smallvec() { + fn test_smallvec_stack_allocated_type() { let mut v: SmallVec<[u8; 3]> = SmallVec::new(); let mut ops = new_malloc_size_ops(); assert_eq!(v.size_of(&mut ops), 0); @@ -85,7 +84,38 @@ mod tests { assert!(!v.spilled()); v.push(4); assert!(v.spilled(), "SmallVec spills when going beyond the capacity of the inner backing array"); - assert_eq!(v.len(), 4); + assert_eq!(v.size_of(&mut ops), 0); // <–– WHY NOT 4? + } + + #[test] + fn test_smallvec_boxed_stack_allocated_type() { + let mut v: SmallVec<[Box; 3]> = SmallVec::new(); + let mut ops = new_malloc_size_ops(); + assert_eq!(v.size_of(&mut ops), 0); + v.push(Box::new(1u8)); + v.push(Box::new(2u8)); + v.push(Box::new(3u8)); + assert_eq!(v.size_of(&mut ops), 3); // <– shouldn't this be 3 * pointer_size = 24? + assert!(!v.spilled()); + v.push(Box::new(4u8)); + assert!(v.spilled(), "SmallVec spills when going beyond the capacity of the inner backing array"); + let mut ops = new_malloc_size_ops(); assert_eq!(v.size_of(&mut ops), 4); } + + #[test] + fn test_smallvec_heap_allocated_type() { + let mut v: SmallVec<[String; 3]> = SmallVec::new(); + let mut ops = new_malloc_size_ops(); + assert_eq!(v.size_of(&mut ops), 0); + v.push("COW".into()); + v.push("PIG".into()); + v.push("DUCK".into()); + assert!(!v.spilled()); + assert_eq!(v.size_of(&mut ops), 10); + v.push("ÖWL".into()); + assert!(v.spilled()); + let mut ops = new_malloc_size_ops(); + assert_eq!(v.size_of(&mut ops), 14); + } } From 539e7e4d9d6d14dc9e195937fb15fdaa70ffa787 Mon Sep 17 00:00:00 2001 From: David Palm Date: Sun, 15 Dec 2019 21:58:53 +0100 Subject: [PATCH 07/18] Include shallow size of the SmallVec when it's spilled Annotate tests --- parity-util-mem/src/impls.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/parity-util-mem/src/impls.rs b/parity-util-mem/src/impls.rs index 867e10cda..c8df39f07 100644 --- a/parity-util-mem/src/impls.rs +++ b/parity-util-mem/src/impls.rs @@ -41,7 +41,11 @@ macro_rules! impl_smallvec { T: MallocSizeOf, { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - let mut n = 0; + let mut n = if self.spilled() { + self.capacity() * core::mem::size_of::() + } else { + 0 + }; for elem in self.iter() { n += elem.size_of(ops); } @@ -84,7 +88,7 @@ mod tests { assert!(!v.spilled()); v.push(4); assert!(v.spilled(), "SmallVec spills when going beyond the capacity of the inner backing array"); - assert_eq!(v.size_of(&mut ops), 0); // <–– WHY NOT 4? + assert_eq!(v.size_of(&mut ops), 4); // 4 u8s on the heap } #[test] @@ -95,12 +99,12 @@ mod tests { v.push(Box::new(1u8)); v.push(Box::new(2u8)); v.push(Box::new(3u8)); - assert_eq!(v.size_of(&mut ops), 3); // <– shouldn't this be 3 * pointer_size = 24? + assert_eq!(v.size_of(&mut ops), 3); // 3 u8s on the heap, boxes are on the stack assert!(!v.spilled()); v.push(Box::new(4u8)); assert!(v.spilled(), "SmallVec spills when going beyond the capacity of the inner backing array"); let mut ops = new_malloc_size_ops(); - assert_eq!(v.size_of(&mut ops), 4); + assert_eq!(v.size_of(&mut ops), 36); // 4*8 (boxes) + 4 u8 in the heap } #[test] @@ -116,6 +120,8 @@ mod tests { v.push("ÖWL".into()); assert!(v.spilled()); let mut ops = new_malloc_size_ops(); - assert_eq!(v.size_of(&mut ops), 14); + // Not super clear where 110 comes from tbh, should be 14 bytes of data + 4 pointers = 14 + 32 = 46 + // so the allocator is likely doing something interesting with Strings. + assert_eq!(v.size_of(&mut ops), 110); } } From 95251b4cc2c6672621f95f1c94cd0f572bd8c698 Mon Sep 17 00:00:00 2001 From: David Palm Date: Sun, 15 Dec 2019 22:12:04 +0100 Subject: [PATCH 08/18] Update CHANGELOGs --- kvdb-memorydb/CHANGELOG.md | 1 + kvdb-rocksdb/CHANGELOG.md | 1 + kvdb/CHANGELOG.md | 4 ++-- parity-util-mem/CHANGELOG.md | 1 + parity-util-mem/src/impls.rs | 2 +- 5 files changed, 6 insertions(+), 3 deletions(-) diff --git a/kvdb-memorydb/CHANGELOG.md b/kvdb-memorydb/CHANGELOG.md index 8e4f2a3b5..fa067272d 100644 --- a/kvdb-memorydb/CHANGELOG.md +++ b/kvdb-memorydb/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog]. ## [Unreleased] ### Fixed - `iter_from_prefix` behaviour synced with the `kvdb-rocksdb` + ### Changed - Default column support removed from the API - Column argument type changed from `Option` to `u32` diff --git a/kvdb-rocksdb/CHANGELOG.md b/kvdb-rocksdb/CHANGELOG.md index 632d6ff24..6f345f51d 100644 --- a/kvdb-rocksdb/CHANGELOG.md +++ b/kvdb-rocksdb/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog]. - `DatabaseConfig::default()` defaults to 1 column - `Database::with_columns` still accepts `u32`, but panics if `0` is provided - `Database::open` panics if configuration with 0 columns is provided +- [BREAKING] Remove `ElasticArray` and use the new `DBValue` (alias for `Vec`) and `DBKey` types from `kvdb`. (See [PR #282](https://github.com/paritytech/parity-common/pull/282/files)) ## [0.2.0] - 2019-11-28 - Switched away from using [parity-rocksdb](https://crates.io/crates/parity-rocksdb) in favour of upstream [rust-rocksdb](https://crates.io/crates/rocksdb) (see [PR #257](https://github.com/paritytech/parity-common/pull/257) for details) diff --git a/kvdb/CHANGELOG.md b/kvdb/CHANGELOG.md index 6aeb26f41..e0573b18e 100644 --- a/kvdb/CHANGELOG.md +++ b/kvdb/CHANGELOG.md @@ -6,10 +6,10 @@ The format is based on [Keep a Changelog]. ## [Unreleased] ### Changed -- Default column support removed from the API +- [BREAKING] Default column support removed from the API - Column argument type changed from `Option` to `u32` - Migration `None` -> `0`, `Some(0)` -> `1`, `Some(1)` -> `2`, etc. - +- [BREAKING] Remove `ElasticArray` and change `DBValue` to be a type alias for `Vec` and add a `DBKey` backed by a `SmallVec`. (See [PR #282](https://github.com/paritytech/parity-common/pull/282/files)) ## [0.1.1] - 2019-10-24 ### Dependencies - Updated dependencies (https://github.com/paritytech/parity-common/pull/239) diff --git a/parity-util-mem/CHANGELOG.md b/parity-util-mem/CHANGELOG.md index 8f69530b7..edb63e8fc 100644 --- a/parity-util-mem/CHANGELOG.md +++ b/parity-util-mem/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] +- [BREAKING] Remove `MallocSizeOf` impls for `ElasticArray` and implement it for `SmallVec` (32 and 36). (See [PR #282](https://github.com/paritytech/parity-common/pull/282/files)) ## [0.2.1] - 2019-10-24 ### Dependencies diff --git a/parity-util-mem/src/impls.rs b/parity-util-mem/src/impls.rs index c8df39f07..bb0799516 100644 --- a/parity-util-mem/src/impls.rs +++ b/parity-util-mem/src/impls.rs @@ -16,7 +16,7 @@ //! Implementation of `MallocSize` for common types : //! - ethereum types uint and fixed hash. -//! - smallvec arrays of sizes 32, 36, 64, 128 and 256 +//! - smallvec arrays of sizes 32, 36 //! - parking_lot mutex structures use super::{MallocSizeOf, MallocSizeOfOps}; From a619a8f08000d4973b62bfe3daadbb76df0128c9 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 16 Dec 2019 09:25:49 +0100 Subject: [PATCH 09/18] Attempt to fix windows build --- parity-util-mem/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parity-util-mem/Cargo.toml b/parity-util-mem/Cargo.toml index f46650d14..2b928dcf3 100644 --- a/parity-util-mem/Cargo.toml +++ b/parity-util-mem/Cargo.toml @@ -25,7 +25,7 @@ ethereum-types = { version = "0.8.0", optional = true, path = "../ethereum-types parking_lot = { version = "0.9.0", optional = true } [target.'cfg(target_os = "windows")'.dependencies] -winapi = "0.3.8" +winapi = { version = "0.3.8", features = ["heapapi"] } [target.'cfg(not(target_os = "windows"))'.dependencies.jemallocator] version = "0.3.2" From 4d086205d1aafea2e05a952748d204caba1555d9 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 16 Dec 2019 17:18:12 +0100 Subject: [PATCH 10/18] Attempt to fix allocator differences --- parity-util-mem/src/impls.rs | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/parity-util-mem/src/impls.rs b/parity-util-mem/src/impls.rs index bb0799516..a4276281f 100644 --- a/parity-util-mem/src/impls.rs +++ b/parity-util-mem/src/impls.rs @@ -99,14 +99,39 @@ mod tests { v.push(Box::new(1u8)); v.push(Box::new(2u8)); v.push(Box::new(3u8)); - assert_eq!(v.size_of(&mut ops), 3); // 3 u8s on the heap, boxes are on the stack + cfg_if::cfg_if! { + if #[cfg(any( + target_os = "windows", + all(target_os = "macos", not(feature = "jemalloc-global")), + feature = "estimate-heapsize", + feature = "weealloc-global", + feature = "dlmalloc-global", + ))] { + assert_eq!(v.size_of(&mut ops), 3); // 3 u8s on the heap, boxes are on the stack + } else { + assert_eq!(v.size_of(&mut ops), 24); + } + } assert!(!v.spilled()); v.push(Box::new(4u8)); assert!(v.spilled(), "SmallVec spills when going beyond the capacity of the inner backing array"); let mut ops = new_malloc_size_ops(); - assert_eq!(v.size_of(&mut ops), 36); // 4*8 (boxes) + 4 u8 in the heap + cfg_if::cfg_if! { + if #[cfg(any( + target_os = "windows", + all(target_os = "macos", not(feature = "jemalloc-global")), + feature = "estimate-heapsize", + feature = "weealloc-global", + feature = "dlmalloc-global", + ))] { + assert_eq!(v.size_of(&mut ops), 36); // 4*8 (boxes) + 4 u8 in the heap + } else { + assert_eq!(v.size_of(&mut ops), 64); + } + } } + #[ignore] #[test] fn test_smallvec_heap_allocated_type() { let mut v: SmallVec<[String; 3]> = SmallVec::new(); From a4361a0782c0fdb70ac1225ca9c69312cea8c72b Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 16 Dec 2019 18:05:06 +0100 Subject: [PATCH 11/18] One more attempt --- parity-util-mem/src/impls.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/parity-util-mem/src/impls.rs b/parity-util-mem/src/impls.rs index a4276281f..202e9cb0b 100644 --- a/parity-util-mem/src/impls.rs +++ b/parity-util-mem/src/impls.rs @@ -108,6 +108,8 @@ mod tests { feature = "dlmalloc-global", ))] { assert_eq!(v.size_of(&mut ops), 3); // 3 u8s on the heap, boxes are on the stack + } else if #[cfg(target_os = "linux")] { + assert_eq!(v.size_of(&mut ops), 72); } else { assert_eq!(v.size_of(&mut ops), 24); } @@ -125,6 +127,8 @@ mod tests { feature = "dlmalloc-global", ))] { assert_eq!(v.size_of(&mut ops), 36); // 4*8 (boxes) + 4 u8 in the heap + } else if #[cfg(target_os = "linux")] { + assert_eq!(v.size_of(&mut ops), 72); } else { assert_eq!(v.size_of(&mut ops), 64); } From 6b8370264fa432695702e1fde2c0bfb8f1ad734f Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 16 Dec 2019 19:12:52 +0100 Subject: [PATCH 12/18] One more try --- parity-util-mem/src/impls.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parity-util-mem/src/impls.rs b/parity-util-mem/src/impls.rs index 202e9cb0b..1541308f9 100644 --- a/parity-util-mem/src/impls.rs +++ b/parity-util-mem/src/impls.rs @@ -128,7 +128,7 @@ mod tests { ))] { assert_eq!(v.size_of(&mut ops), 36); // 4*8 (boxes) + 4 u8 in the heap } else if #[cfg(target_os = "linux")] { - assert_eq!(v.size_of(&mut ops), 72); + assert_eq!(v.size_of(&mut ops), 128); } else { assert_eq!(v.size_of(&mut ops), 64); } From a62b3c7e5ca25de60b19661de3098f60d2a653f8 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 16 Dec 2019 22:06:48 +0100 Subject: [PATCH 13/18] Maybe both 24 and 72 are correct --- parity-util-mem/src/impls.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/parity-util-mem/src/impls.rs b/parity-util-mem/src/impls.rs index 1541308f9..75cbc82e0 100644 --- a/parity-util-mem/src/impls.rs +++ b/parity-util-mem/src/impls.rs @@ -109,9 +109,12 @@ mod tests { ))] { assert_eq!(v.size_of(&mut ops), 3); // 3 u8s on the heap, boxes are on the stack } else if #[cfg(target_os = "linux")] { - assert_eq!(v.size_of(&mut ops), 72); - } else { - assert_eq!(v.size_of(&mut ops), 24); + assert!( + // Ubuntus default allocator returns 24 + v.size_of(&mut ops) == 24 || + // Whatever Linux Travis is using has a default allocator that returns 72. + v.size_of(&mut ops) == 72 + ); } } assert!(!v.spilled()); From 6c17b1b5b24876de7e75b1610381f6adae24446c Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 16 Dec 2019 22:30:53 +0100 Subject: [PATCH 14/18] getting there? --- parity-util-mem/src/impls.rs | 45 +++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/parity-util-mem/src/impls.rs b/parity-util-mem/src/impls.rs index 75cbc82e0..a00cb9d87 100644 --- a/parity-util-mem/src/impls.rs +++ b/parity-util-mem/src/impls.rs @@ -131,14 +131,14 @@ mod tests { ))] { assert_eq!(v.size_of(&mut ops), 36); // 4*8 (boxes) + 4 u8 in the heap } else if #[cfg(target_os = "linux")] { - assert_eq!(v.size_of(&mut ops), 128); - } else { - assert_eq!(v.size_of(&mut ops), 64); + assert!( + v.size_of(&mut ops) == 64 || + v.size_of(&mut ops) == 128 + ); } } } - #[ignore] #[test] fn test_smallvec_heap_allocated_type() { let mut v: SmallVec<[String; 3]> = SmallVec::new(); @@ -148,12 +148,41 @@ mod tests { v.push("PIG".into()); v.push("DUCK".into()); assert!(!v.spilled()); - assert_eq!(v.size_of(&mut ops), 10); + cfg_if::cfg_if! { + if #[cfg(any( + target_os = "windows", + all(target_os = "macos", not(feature = "jemalloc-global")), + feature = "estimate-heapsize", + feature = "weealloc-global", + feature = "dlmalloc-global", + ))] { + assert_eq!(v.size_of(&mut ops), 10); + } else { + assert!( + // Ubuntus default allocator returns 24 + v.size_of(&mut ops) == 24 || + // Whatever Linux Travis is using has a default allocator that returns 72. + v.size_of(&mut ops) == 72 + ); + } + } v.push("ÖWL".into()); assert!(v.spilled()); let mut ops = new_malloc_size_ops(); - // Not super clear where 110 comes from tbh, should be 14 bytes of data + 4 pointers = 14 + 32 = 46 - // so the allocator is likely doing something interesting with Strings. - assert_eq!(v.size_of(&mut ops), 110); + cfg_if::cfg_if! { + if #[cfg(any( + target_os = "windows", + all(target_os = "macos", not(feature = "jemalloc-global")), + feature = "estimate-heapsize", + feature = "weealloc-global", + feature = "dlmalloc-global", + ))] { + // Not super clear where 110 comes from tbh, should be 14 bytes of data + 4 pointers = 14 + 32 = 46 + // so the allocator is likely doing something interesting with Strings. + assert_eq!(v.size_of(&mut ops), 110); + } else { + assert_eq!(v.size_of(&mut ops), 192); + } + } } } From a215f59a913b136c9553d8755217a1d71e64b247 Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 17 Dec 2019 08:57:07 +0100 Subject: [PATCH 15/18] Assert the allocator does *something* when the SmallVec spills --- parity-util-mem/src/impls.rs | 71 ++---------------------------------- 1 file changed, 4 insertions(+), 67 deletions(-) diff --git a/parity-util-mem/src/impls.rs b/parity-util-mem/src/impls.rs index a00cb9d87..a74620ad0 100644 --- a/parity-util-mem/src/impls.rs +++ b/parity-util-mem/src/impls.rs @@ -99,44 +99,12 @@ mod tests { v.push(Box::new(1u8)); v.push(Box::new(2u8)); v.push(Box::new(3u8)); - cfg_if::cfg_if! { - if #[cfg(any( - target_os = "windows", - all(target_os = "macos", not(feature = "jemalloc-global")), - feature = "estimate-heapsize", - feature = "weealloc-global", - feature = "dlmalloc-global", - ))] { - assert_eq!(v.size_of(&mut ops), 3); // 3 u8s on the heap, boxes are on the stack - } else if #[cfg(target_os = "linux")] { - assert!( - // Ubuntus default allocator returns 24 - v.size_of(&mut ops) == 24 || - // Whatever Linux Travis is using has a default allocator that returns 72. - v.size_of(&mut ops) == 72 - ); - } - } + assert!(v.size_of(&mut ops) >= 3); assert!(!v.spilled()); v.push(Box::new(4u8)); assert!(v.spilled(), "SmallVec spills when going beyond the capacity of the inner backing array"); let mut ops = new_malloc_size_ops(); - cfg_if::cfg_if! { - if #[cfg(any( - target_os = "windows", - all(target_os = "macos", not(feature = "jemalloc-global")), - feature = "estimate-heapsize", - feature = "weealloc-global", - feature = "dlmalloc-global", - ))] { - assert_eq!(v.size_of(&mut ops), 36); // 4*8 (boxes) + 4 u8 in the heap - } else if #[cfg(target_os = "linux")] { - assert!( - v.size_of(&mut ops) == 64 || - v.size_of(&mut ops) == 128 - ); - } - } + assert!(v.size_of(&mut ops) >= 36); } #[test] @@ -148,41 +116,10 @@ mod tests { v.push("PIG".into()); v.push("DUCK".into()); assert!(!v.spilled()); - cfg_if::cfg_if! { - if #[cfg(any( - target_os = "windows", - all(target_os = "macos", not(feature = "jemalloc-global")), - feature = "estimate-heapsize", - feature = "weealloc-global", - feature = "dlmalloc-global", - ))] { - assert_eq!(v.size_of(&mut ops), 10); - } else { - assert!( - // Ubuntus default allocator returns 24 - v.size_of(&mut ops) == 24 || - // Whatever Linux Travis is using has a default allocator that returns 72. - v.size_of(&mut ops) == 72 - ); - } - } + assert!(v.size_of(&mut ops) >= 10); v.push("ÖWL".into()); assert!(v.spilled()); let mut ops = new_malloc_size_ops(); - cfg_if::cfg_if! { - if #[cfg(any( - target_os = "windows", - all(target_os = "macos", not(feature = "jemalloc-global")), - feature = "estimate-heapsize", - feature = "weealloc-global", - feature = "dlmalloc-global", - ))] { - // Not super clear where 110 comes from tbh, should be 14 bytes of data + 4 pointers = 14 + 32 = 46 - // so the allocator is likely doing something interesting with Strings. - assert_eq!(v.size_of(&mut ops), 110); - } else { - assert_eq!(v.size_of(&mut ops), 192); - } - } + assert!(v.size_of(&mut ops) >= 110); } } From 168adf01732900f70d5fbbc13de24f6860a17ab9 Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 17 Dec 2019 11:47:48 +0100 Subject: [PATCH 16/18] formatting --- parity-util-mem/src/impls.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/parity-util-mem/src/impls.rs b/parity-util-mem/src/impls.rs index a74620ad0..4002a9e16 100644 --- a/parity-util-mem/src/impls.rs +++ b/parity-util-mem/src/impls.rs @@ -41,11 +41,7 @@ macro_rules! impl_smallvec { T: MallocSizeOf, { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - let mut n = if self.spilled() { - self.capacity() * core::mem::size_of::() - } else { - 0 - }; + let mut n = if self.spilled() { self.capacity() * core::mem::size_of::() } else { 0 }; for elem in self.iter() { n += elem.size_of(ops); } From 420cfaf895092552ae420fade6f573a9e7d532fd Mon Sep 17 00:00:00 2001 From: David Date: Tue, 17 Dec 2019 13:22:26 +0100 Subject: [PATCH 17/18] Update kvdb-rocksdb/CHANGELOG.md Co-Authored-By: Andronik Ordian --- kvdb-rocksdb/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kvdb-rocksdb/CHANGELOG.md b/kvdb-rocksdb/CHANGELOG.md index 6f345f51d..e7b5e4553 100644 --- a/kvdb-rocksdb/CHANGELOG.md +++ b/kvdb-rocksdb/CHANGELOG.md @@ -16,7 +16,8 @@ The format is based on [Keep a Changelog]. - `DatabaseConfig::default()` defaults to 1 column - `Database::with_columns` still accepts `u32`, but panics if `0` is provided - `Database::open` panics if configuration with 0 columns is provided -- [BREAKING] Remove `ElasticArray` and use the new `DBValue` (alias for `Vec`) and `DBKey` types from `kvdb`. (See [PR #282](https://github.com/paritytech/parity-common/pull/282/files)) +### Breaking +- Remove `ElasticArray` and use the new `DBValue` (alias for `Vec`) and `DBKey` types from `kvdb`. (See [PR #282](https://github.com/paritytech/parity-common/pull/282/files)) ## [0.2.0] - 2019-11-28 - Switched away from using [parity-rocksdb](https://crates.io/crates/parity-rocksdb) in favour of upstream [rust-rocksdb](https://crates.io/crates/rocksdb) (see [PR #257](https://github.com/paritytech/parity-common/pull/257) for details) From 60347a8da1b85e036ae7ea3f7a5edefd02418bbe Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 17 Dec 2019 13:42:31 +0100 Subject: [PATCH 18/18] address review grumbles --- kvdb/CHANGELOG.md | 4 +++- parity-util-mem/src/impls.rs | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/kvdb/CHANGELOG.md b/kvdb/CHANGELOG.md index e0573b18e..4c6a80642 100644 --- a/kvdb/CHANGELOG.md +++ b/kvdb/CHANGELOG.md @@ -9,7 +9,9 @@ The format is based on [Keep a Changelog]. - [BREAKING] Default column support removed from the API - Column argument type changed from `Option` to `u32` - Migration `None` -> `0`, `Some(0)` -> `1`, `Some(1)` -> `2`, etc. -- [BREAKING] Remove `ElasticArray` and change `DBValue` to be a type alias for `Vec` and add a `DBKey` backed by a `SmallVec`. (See [PR #282](https://github.com/paritytech/parity-common/pull/282/files)) +### BREAKING +- Remove `ElasticArray` and change `DBValue` to be a type alias for `Vec` and add a `DBKey` backed by a `SmallVec`. (See [PR #282](https://github.com/paritytech/parity-common/pull/282/files)) + ## [0.1.1] - 2019-10-24 ### Dependencies - Updated dependencies (https://github.com/paritytech/parity-common/pull/239) diff --git a/parity-util-mem/src/impls.rs b/parity-util-mem/src/impls.rs index 4002a9e16..ad1962d7f 100644 --- a/parity-util-mem/src/impls.rs +++ b/parity-util-mem/src/impls.rs @@ -70,6 +70,7 @@ impl MallocSizeOf for RwLock { mod tests { use crate::{allocators::new_malloc_size_ops, MallocSizeOf, MallocSizeOfOps}; use smallvec::SmallVec; + use std::mem; impl_smallvec!(3); #[test] @@ -100,7 +101,8 @@ mod tests { v.push(Box::new(4u8)); assert!(v.spilled(), "SmallVec spills when going beyond the capacity of the inner backing array"); let mut ops = new_malloc_size_ops(); - assert!(v.size_of(&mut ops) >= 36); + let expected_min_allocs = mem::size_of::>() * 4 + 4; + assert!(v.size_of(&mut ops) >= expected_min_allocs); } #[test] @@ -112,10 +114,11 @@ mod tests { v.push("PIG".into()); v.push("DUCK".into()); assert!(!v.spilled()); - assert!(v.size_of(&mut ops) >= 10); + assert!(v.size_of(&mut ops) >= "COW".len() + "PIG".len() + "DUCK".len()); v.push("ÖWL".into()); assert!(v.spilled()); let mut ops = new_malloc_size_ops(); - assert!(v.size_of(&mut ops) >= 110); + let expected_min_allocs = mem::size_of::() * 4 + "ÖWL".len() + "COW".len() + "PIG".len() + "DUCK".len(); + assert!(v.size_of(&mut ops) >= expected_min_allocs); } }