Replace ElasticArray with SmallVec

kvdb-memorydb/CHANGELOG.md

@@ -7,6 +7,7 @@ The format is based on [Keep a Changelog].
 ## [Unreleased]
 ### Fixed
 - `iter_from_prefix` behaviour synced with the `kvdb-rocksdb`
+
 ### Changed
 - Default column support removed from the API
   - Column argument type changed from `Option<u32>` to `u32`

kvdb-memorydb/src/lib.rs

@@ -86,7 +86,7 @@ impl KeyValueDB for InMemory {
 		match self.columns.read().get(&col) {
 			Some(map) => Box::new(
 				// TODO: worth optimizing at all?
-				map.clone().into_iter().map(|(k, v)| (k.into_boxed_slice(), v.into_vec().into_boxed_slice())),
+				map.clone().into_iter().map(|(k, v)| (k.into_boxed_slice(), v.into_boxed_slice())),
 			),
 			None => Box::new(None.into_iter()),
 		}
@@ -102,7 +102,7 @@ impl KeyValueDB for InMemory {
 				map.clone()
 					.into_iter()
 					.filter(move |&(ref k, _)| k.starts_with(prefix))
-					.map(|(k, v)| (k.into_boxed_slice(), v.into_vec().into_boxed_slice())),
+					.map(|(k, v)| (k.into_boxed_slice(), v.into_boxed_slice())),
 			),
 			None => Box::new(None.into_iter()),
 		}

kvdb-rocksdb/CHANGELOG.md

@@ -16,6 +16,8 @@ The format is based on [Keep a Changelog].
   - `DatabaseConfig::default()` defaults to 1 column
   - `Database::with_columns` still accepts `u32`, but panics if `0` is provided 
   - `Database::open` panics if configuration with 0 columns is provided
+### Breaking 
+- Remove `ElasticArray` and use the new `DBValue` (alias for `Vec<u8>`) and `DBKey` types from `kvdb`. (See [PR #282](https://github.com/paritytech/parity-common/pull/282/files))  
 
 ## [0.2.0] - 2019-11-28
 - Switched away from using [parity-rocksdb](https://crates.io/crates/parity-rocksdb) in favour of upstream [rust-rocksdb](https://crates.io/crates/rocksdb) (see [PR #257](https://github.com/paritytech/parity-common/pull/257) for details)

kvdb-rocksdb/Cargo.toml

@@ -12,7 +12,7 @@ name = "bench_read_perf"
 harness = false
 
 [dependencies]
-elastic-array = "0.10.2"
+smallvec = "1.0.0"
 fs-swap = "0.2.4"
 interleaved-ordered = "0.1.1"
 kvdb = { path = "../kvdb", version = "0.1" }

kvdb-rocksdb/benches/bench_read_perf.rs

@@ -48,7 +48,7 @@ criterion_main!(benches);
 /// family and default options. Needs manual cleanup.
 fn open_db() -> Database {
 	let tempdir_str = "./benches/_rocksdb_bench_get";
-	let cfg = DatabaseConfig::with_columns(Some(1));
+	let cfg = DatabaseConfig::with_columns(1);
 	let db = Database::open(&cfg, tempdir_str).expect("rocksdb works");
 	db
 }
@@ -81,7 +81,7 @@ fn populate(db: &Database) -> io::Result<Vec<H256>> {
 			}
 		}
 		// In ethereum keys are mostly 32 bytes and payloads ~140bytes.
-		batch.put(Some(0), &key.as_bytes(), &n_random_bytes(140));
+		batch.put(0, &key.as_bytes(), &n_random_bytes(140));
 	}
 	db.write(batch)?;
 	// Clear the overlay
@@ -106,7 +106,7 @@ fn get(c: &mut Criterion) {
 				for _ in 0..iterations {
 					// This has no measurable impact on performance (~30ns)
 					let needle = needles.choose(&mut rand::thread_rng()).expect("needles is not empty");
-					black_box(db.get(Some(0), needle.as_bytes()).unwrap());
+					black_box(db.get(0, needle.as_bytes()).unwrap());
 				}
 				elapsed = start.elapsed();
 			});
@@ -135,7 +135,7 @@ fn get(c: &mut Criterion) {
 				for _ in 0..iterations {
 					// This has no measurable impact on performance (~30ns)
 					let needle = needles.choose(&mut rand::thread_rng()).expect("needles is not empty");
-					black_box(db.get_by_prefix(Some(0), &needle.as_bytes()[..8]).unwrap());
+					black_box(db.get_by_prefix(0, &needle.as_bytes()[..8]).unwrap());
 				}
 				elapsed = start.elapsed();
 			});
@@ -166,7 +166,7 @@ fn iter(c: &mut Criterion) {
 			let (alloc_stats, _) = count_alloc(|| {
 				let start = Instant::now();
 				for _ in 0..iterations {
-					black_box(db.iter(Some(0)).take(1000).collect::<Vec<_>>());
+					black_box(db.iter(0).take(1000).collect::<Vec<_>>());
 				}
 				elapsed = start.elapsed();
 			});
@@ -193,7 +193,7 @@ fn iter(c: &mut Criterion) {
 			let (alloc_stats, _) = count_alloc(|| {
 				let start = Instant::now();
 				for _ in 0..iterations {
-					black_box(db.iter(Some(0)).next().unwrap());
+					black_box(db.iter(0).next().unwrap());
 				}
 				elapsed = start.elapsed();
 			});

kvdb-rocksdb/src/lib.rs

@@ -24,10 +24,9 @@ use rocksdb::{
 };
 
 use crate::iter::KeyValuePair;
-use elastic_array::ElasticArray32;
 use fs_swap::{swap, swap_nonatomic};
 use interleaved_ordered::interleave_ordered;
-use kvdb::{DBOp, DBTransaction, DBValue, KeyValueDB};
+use kvdb::{DBKey, DBOp, DBTransaction, DBValue, KeyValueDB};
 use log::{debug, warn};
 
 #[cfg(target_os = "linux")]
@@ -245,9 +244,9 @@ pub struct Database {
 	read_opts: ReadOptions,
 	block_opts: BlockBasedOptions,
 	// Dirty values added with `write_buffered`. Cleaned on `flush`.
-	overlay: RwLock<Vec<HashMap<ElasticArray32<u8>, KeyState>>>,
+	overlay: RwLock<Vec<HashMap<DBKey, KeyState>>>,
 	// Values currently being flushed. Cleared when `flush` completes.
-	flushing: RwLock<Vec<HashMap<ElasticArray32<u8>, KeyState>>>,
+	flushing: RwLock<Vec<HashMap<DBKey, KeyState>>>,
 	// Prevents concurrent flushes.
 	// Value indicates if a flush is in progress.
 	flushing_lock: Mutex<bool>,
@@ -483,7 +482,7 @@ impl Database {
 							None => cfs
 								.db
 								.get_pinned_cf_opt(cfs.cf(col as usize), key, &self.read_opts)
-								.map(|r| r.map(|v| DBValue::from_slice(&v)))
+								.map(|r| r.map(|v| v.to_vec()))
 								.map_err(other_io_err),
 						}
 					}
@@ -511,7 +510,7 @@ impl Database {
 					.iter()
 					.filter_map(|(k, v)| match *v {
 						KeyState::Insert(ref value) => {
-							Some((k.clone().into_vec().into_boxed_slice(), value.clone().into_vec().into_boxed_slice()))
+							Some((k.clone().into_vec().into_boxed_slice(), value.clone().into_boxed_slice()))
 						}
 						KeyState::Delete => None,
 					})
@@ -892,7 +891,7 @@ mod tests {
 		batch.put(0, b"foo", b"baz");
 		db.write(batch).unwrap();
 
-		assert_eq!(db.get(0, b"foo").unwrap().unwrap().as_ref(), b"baz");
+		assert_eq!(db.get(0, b"foo").unwrap().unwrap(), b"baz");
 	}
 
 	#[test]

kvdb-web/tests/indexed_db.rs

@@ -39,7 +39,7 @@ async fn reopen_the_database_with_more_columns() {
 	batch.put(0, b"hello", b"world");
 	db.write_buffered(batch);
 
-	assert_eq!(db.get(0, b"hello").unwrap().unwrap().as_ref(), b"world");
+	assert_eq!(db.get(0, b"hello").unwrap().unwrap(), b"world");
 
 	// Check the database version
 	assert_eq!(db.version(), 1);
@@ -51,7 +51,7 @@ async fn reopen_the_database_with_more_columns() {
 	let db = open_db(3).await;
 
 	// The value should still be present
-	assert_eq!(db.get(0, b"hello").unwrap().unwrap().as_ref(), b"world");
+	assert_eq!(db.get(0, b"hello").unwrap().unwrap(), b"world");
 	assert!(db.get(0, b"trash").unwrap().is_none());
 
 	// The version should be bumped

kvdb/CHANGELOG.md

@@ -6,9 +6,11 @@ The format is based on [Keep a Changelog].
 
 ## [Unreleased]
 ### Changed
-- Default column support removed from the API
+- [BREAKING] Default column support removed from the API
   - Column argument type changed from `Option<u32>` to `u32`
   - Migration `None` -> `0`, `Some(0)` -> `1`, `Some(1)` -> `2`, etc.
+### BREAKING
+- Remove `ElasticArray` and change `DBValue` to be a type alias for `Vec<u8>` and add a `DBKey` backed by a `SmallVec`.  (See [PR #282](https://github.com/paritytech/parity-common/pull/282/files))
 
 ## [0.1.1] - 2019-10-24
 ### Dependencies

kvdb/Cargo.toml

@@ -8,5 +8,5 @@ license = "GPL-3.0"
 edition = "2018"
 
 [dependencies]
-elastic-array = "0.10.2"
+smallvec = "1.0.0"
 bytes = { package = "parity-bytes", version = "0.1", path = "../parity-bytes" }

kvdb/src/lib.rs

@@ -17,7 +17,7 @@
 //! Key-Value store abstraction with `RocksDB` backend.
 
 use bytes::Bytes;
-use elastic_array::{ElasticArray128, ElasticArray32};
+use smallvec::SmallVec;
 use std::io;
 use std::path::Path;
 use std::sync::Arc;
@@ -26,7 +26,9 @@ use std::sync::Arc;
 pub const PREFIX_LEN: usize = 12;
 
 /// Database value.
-pub type DBValue = ElasticArray128<u8>;
+pub type DBValue = Vec<u8>;
+/// Database keys.
+pub type DBKey = SmallVec<[u8; 32]>;
 
 /// Write transaction. Batches a sequence of put/delete operations for efficiency.
 #[derive(Default, Clone, PartialEq)]
@@ -38,8 +40,8 @@ pub struct DBTransaction {
 /// Database operation.
 #[derive(Clone, PartialEq)]
 pub enum DBOp {
-	Insert { col: u32, key: ElasticArray32<u8>, value: DBValue },
-	Delete { col: u32, key: ElasticArray32<u8> },
+	Insert { col: u32, key: DBKey, value: DBValue },
+	Delete { col: u32, key: DBKey },
 }
 
 impl DBOp {
@@ -73,23 +75,17 @@ impl DBTransaction {
 
 	/// Insert a key-value pair in the transaction. Any existing value will be overwritten upon write.
 	pub fn put(&mut self, col: u32, key: &[u8], value: &[u8]) {
-		let mut ekey = ElasticArray32::new();
-		ekey.append_slice(key);
-		self.ops.push(DBOp::Insert { col, key: ekey, value: DBValue::from_slice(value) });
+		self.ops.push(DBOp::Insert { col, key: DBKey::from_slice(key), value: value.to_vec() })
 	}
 
 	/// Insert a key-value pair in the transaction. Any existing value will be overwritten upon write.
 	pub fn put_vec(&mut self, col: u32, key: &[u8], value: Bytes) {
-		let mut ekey = ElasticArray32::new();
-		ekey.append_slice(key);
-		self.ops.push(DBOp::Insert { col, key: ekey, value: DBValue::from_vec(value) });
+		self.ops.push(DBOp::Insert { col, key: DBKey::from_slice(key), value });
 	}
 
 	/// Delete value by key.
 	pub fn delete(&mut self, col: u32, key: &[u8]) {
-		let mut ekey = ElasticArray32::new();
-		ekey.append_slice(key);
-		self.ops.push(DBOp::Delete { col, key: ekey });
+		self.ops.push(DBOp::Delete { col, key: DBKey::from_slice(key) });
 	}
 }
 

parity-util-mem/CHANGELOG.md

@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog].
 [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/
 
 ## [Unreleased]
+- [BREAKING] Remove `MallocSizeOf` impls for `ElasticArray` and implement it for `SmallVec` (32 and 36). (See [PR #282](https://github.com/paritytech/parity-common/pull/282/files))
 
 ## [0.2.1] - 2019-10-24
 ### Dependencies

parity-util-mem/Cargo.toml

@@ -20,12 +20,12 @@ wee_alloc = { version = "0.4.5", optional = true }
 mimallocator = { version = "0.1.3", features = ["secure"], optional = true }
 mimalloc-sys = { version = "0.1.6", optional = true }
 
-elastic-array = { version = "0.10.2", optional = true }
+smallvec = { version = "1.0.0", optional = true }
 ethereum-types = { version = "0.8.0", optional = true, path = "../ethereum-types" }
 parking_lot = { version = "0.9.0", optional = true }
 
 [target.'cfg(target_os = "windows")'.dependencies]
-winapi = "0.3.8"
+winapi = { version = "0.3.8", features = ["heapapi"] }
 
 [target.'cfg(not(target_os = "windows"))'.dependencies.jemallocator]
 version = "0.3.2"
@@ -43,6 +43,6 @@ jemalloc-global = ["jemallocator"]
 # use mimalloc as global allocator
 mimalloc-global = ["mimallocator", "mimalloc-sys"]
 # implement additional types
-ethereum-impls = ["ethereum-types", "elastic-array", "parking_lot"]
+ethereum-impls = ["ethereum-types", "parking_lot", "smallvec"]
 # Full estimate: no call to allocator
 estimate-heapsize = []