stratus/eth/storage/permanent/rocks/
cf_versions.rs

1//! Column Family (CF) versioning.
2//!
3//! This allows our KV-store to have different versions on the Value.
4//!
5//! Versions are tested against snapshots to avoid breaking changes.
6
7use std::fmt::Debug;
8use std::ops::Deref;
9use std::ops::DerefMut;
10
11use serde::Deserialize;
12use serde::Serialize;
13use strum::EnumCount;
14use strum::IntoStaticStr;
15use strum::VariantNames;
16
17use super::types::AccountRocksdb;
18use super::types::BlockNumberRocksdb;
19use super::types::BlockRocksdb;
20use super::types::SlotValueRocksdb;
21use crate::eth::primitives::Account;
22use crate::eth::primitives::Block;
23use crate::eth::primitives::BlockNumber;
24use crate::eth::primitives::SlotValue;
25use crate::eth::storage::permanent::rocks::SerializeDeserializeWithContext;
26use crate::eth::storage::permanent::rocks::types::BlockChangesRocksdb;
27
28macro_rules! impl_single_version_cf_value {
29    ($name:ident, $inner_type:ty, $non_rocks_equivalent: ty) => {
30        #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, EnumCount, VariantNames, IntoStaticStr, fake::Dummy, bincode::Encode, bincode::Decode)]
31        pub enum $name {
32            V1($inner_type),
33        }
34
35        impl $name {
36            #[allow(dead_code)]
37            pub fn into_inner(self) -> $inner_type {
38                match self {
39                    Self::V1(v1) => v1,
40                }
41            }
42        }
43
44        // `From` conversion should only exist for values with a single version
45        static_assertions::const_assert_eq!($name::COUNT, 1);
46
47        // implement `From` for the v1 type.
48        impl From<$inner_type> for $name {
49            fn from(v1: $inner_type) -> Self {
50                Self::V1(v1)
51            }
52        }
53
54        impl Deref for $name {
55            type Target = $inner_type;
56            fn deref(&self) -> &Self::Target {
57                match self {
58                    Self::V1(v1) => v1,
59                }
60            }
61        }
62
63        impl DerefMut for $name {
64            fn deref_mut(&mut self) -> &mut Self::Target {
65                match self {
66                    Self::V1(v1) => v1,
67                }
68            }
69        }
70
71        // Do `$non_rocks_equivalent -> $inner_type -> $name` in one conversion.
72        impl From<$non_rocks_equivalent> for $name {
73            fn from(value: $non_rocks_equivalent) -> Self {
74                let value = <$inner_type>::from(value);
75                Self::V1(value)
76            }
77        }
78    };
79}
80
81impl_single_version_cf_value!(CfAccountsValue, AccountRocksdb, Account);
82impl_single_version_cf_value!(CfAccountsHistoryValue, AccountRocksdb, Account);
83impl_single_version_cf_value!(CfAccountSlotsValue, SlotValueRocksdb, SlotValue);
84impl_single_version_cf_value!(CfAccountSlotsHistoryValue, SlotValueRocksdb, SlotValue);
85impl_single_version_cf_value!(CfTransactionsValue, BlockNumberRocksdb, BlockNumber);
86impl_single_version_cf_value!(CfBlocksByNumberValue, BlockRocksdb, Block);
87impl_single_version_cf_value!(CfBlocksByHashValue, BlockNumberRocksdb, BlockNumber);
88impl_single_version_cf_value!(CfBlockChangesValue, BlockChangesRocksdb, ());
89
90impl SerializeDeserializeWithContext for CfAccountSlotsHistoryValue {}
91impl SerializeDeserializeWithContext for CfAccountSlotsValue {}
92impl SerializeDeserializeWithContext for CfAccountsHistoryValue {}
93impl SerializeDeserializeWithContext for CfAccountsValue {}
94impl SerializeDeserializeWithContext for CfBlocksByHashValue {}
95impl SerializeDeserializeWithContext for CfTransactionsValue {}
96impl SerializeDeserializeWithContext for CfBlockChangesValue {}
97impl SerializeDeserializeWithContext for CfBlocksByNumberValue {}
98
99#[cfg_attr(not(test), allow(dead_code))]
100trait ToCfName {
101    const CF_NAME: &'static str;
102}
103
104macro_rules! impl_to_cf_name {
105    ($type:ident, $cf_name:expr) => {
106        impl ToCfName for $type {
107            const CF_NAME: &'static str = $cf_name;
108        }
109    };
110}
111
112impl_to_cf_name!(CfAccountsValue, "accounts");
113impl_to_cf_name!(CfAccountsHistoryValue, "accounts_history");
114impl_to_cf_name!(CfAccountSlotsValue, "account_slots");
115impl_to_cf_name!(CfAccountSlotsHistoryValue, "account_slots_history");
116impl_to_cf_name!(CfTransactionsValue, "transactions");
117impl_to_cf_name!(CfBlocksByNumberValue, "blocks_by_number");
118impl_to_cf_name!(CfBlocksByHashValue, "blocks_by_hash");
119/// Test that deserialization works for each variant of the enum.
120///
121/// This is intended to give an error when the following happens:
122///
123/// 1. A new variant is added to the enum.
124/// 2. A variant is renamed.
125/// 3. A variant is removed.
126/// 4. A variant is modified.
127/// 5. A variant is reordered.
128///
129/// Here is a breakdown of why, and how to proceed:
130///
131/// 1. New variants need to be tested, go to the test below and cover it, but watch out for:
132///   - You'll need an ENV VAR to create the new snapshot file.
133///   - When commiting the change, make sure you're just adding your new snapshot, and not editing others by accident.
134/// 2. For renamed variants, because we use bincode, you just need to update the snapshot file.
135///   - Rename it locally.
136/// 3. Previous variants can't be removed as they break our database, because they won't be able to read the older data.
137///   - Don't do it¹.
138/// 4. If you modify a variant, the database won't be able to read it anymore.
139///   - Don't do it¹.
140/// 5. Reordering variants will break deserialization because bincode uses their order to determine the enum tag.
141///   - Don't do it¹.
142///
143/// ¹: if you really want to do it, make sure you can reload your entire database from scratch.
144#[cfg(test)]
145mod tests {
146    use std::env;
147    use std::fmt::Debug;
148    use std::fs;
149    use std::marker::PhantomData;
150    use std::path::Path;
151
152    use anyhow::Context;
153    use anyhow::Result;
154    use anyhow::bail;
155    use anyhow::ensure;
156
157    use super::*;
158    use crate::ext::not;
159    use crate::ext::type_basename;
160    use crate::utils::test_utils::glob_to_string_paths;
161
162    /// A drop bomb that guarantees that all variants of an enum have been tested.
163    struct EnumCoverageDropBombChecker<CfValue>
164    where
165        CfValue: VariantNames + ToCfName,
166    {
167        confirmations: Vec<TestRunConfirmation<CfValue>>,
168    }
169
170    impl<CfValue> EnumCoverageDropBombChecker<CfValue>
171    where
172        CfValue: VariantNames + ToCfName,
173    {
174        fn new() -> Self {
175            Self { confirmations: Vec::new() }
176        }
177
178        fn add(&mut self, rhs: TestRunConfirmation<CfValue>) {
179            self.confirmations.push(rhs);
180        }
181    }
182
183    impl<CfValue> Drop for EnumCoverageDropBombChecker<CfValue>
184    where
185        CfValue: VariantNames + ToCfName,
186    {
187        fn drop(&mut self) {
188            // check for missing confirmations
189            for variant_name in CfValue::VARIANTS {
190                let found = self.confirmations.iter().find(|confirmation| confirmation.variant_name == *variant_name);
191
192                if found.is_none() {
193                    panic!(
194                        "TestRunDropBombChecker<{enum_typename}> panic on drop: cf {}: missing test for variant '{}' of enum {enum_typename}",
195                        CfValue::CF_NAME,
196                        variant_name,
197                        enum_typename = type_basename::<CfValue>(),
198                    );
199                }
200            }
201        }
202    }
203
204    /// A confirmation that a test was run for a specific variant of an enum, used by the drop bomb.
205    struct TestRunConfirmation<CfValue> {
206        variant_name: &'static str,
207        _marker: PhantomData<CfValue>,
208    }
209
210    impl<CfValue> TestRunConfirmation<CfValue> {
211        fn new(variant_name: &'static str) -> Self {
212            Self {
213                variant_name,
214                _marker: PhantomData,
215            }
216        }
217    }
218
219    fn get_all_bincode_snapshots_from_folder(folder: impl AsRef<str>) -> Result<Vec<String>> {
220        let pattern = format!("{}/*.bincode", folder.as_ref());
221        glob_to_string_paths(pattern).context("failed to get all bincode snapshots from folder")
222    }
223
224    fn load_or_generate_json_fixture<CfValue>(cf_name: &str, _variant_name: &str) -> Result<CfValue>
225    where
226        CfValue: for<'de> Deserialize<'de> + Serialize + fake::Dummy<fake::Faker> + ToCfName + bincode::Encode + bincode::Decode<()>,
227    {
228        let json_path = format!("tests/fixtures/cf_versions/{cf_name}/{cf_name}.json");
229        let json_parent_path = format!("tests/fixtures/cf_versions/{cf_name}");
230
231        // Try to load existing fixture first
232        if Path::new(&json_path).exists() {
233            let json_content = fs::read_to_string(&json_path).with_context(|| format!("failed to read JSON fixture at {json_path}"))?;
234            return serde_json::from_str(&json_content).with_context(|| format!("failed to deserialize CfValue from JSON fixture at {json_path}"));
235        }
236
237        // Generate fixture if it doesn't exist and DANGEROUS_UPDATE_SNAPSHOTS is set
238        if env::var("DANGEROUS_UPDATE_SNAPSHOTS").is_ok() {
239            let generated_fixture = crate::utils::test_utils::fake_first::<CfValue>();
240            let json_content =
241                serde_json::to_string_pretty(&generated_fixture).with_context(|| format!("failed to serialize generated fixture for {cf_name}"))?;
242
243            fs::create_dir_all(&json_parent_path).with_context(|| format!("failed to create directory {json_parent_path}"))?;
244            fs::write(&json_path, &json_content).with_context(|| format!("failed to write JSON fixture to {json_path}"))?;
245
246            return Ok(generated_fixture);
247        }
248
249        bail!("JSON fixture at '{json_path}' doesn't exist and DANGEROUS_UPDATE_SNAPSHOTS is not set");
250    }
251
252    /// Store snapshots of the current serialization format for each version.
253    #[test]
254    fn test_snapshot_bincode_deserialization_for_single_version_enums() {
255        fn test_deserialization<CfValue>() -> Result<TestRunConfirmation<CfValue>>
256        where
257            CfValue: for<'de> Deserialize<'de>
258                + Serialize
259                + Clone
260                + Debug
261                + PartialEq
262                + Into<&'static str>
263                + ToCfName
264                + fake::Dummy<fake::Faker>
265                + bincode::Encode
266                + bincode::Decode<()>,
267        {
268            let cf_name = CfValue::CF_NAME;
269            // For single version enums, we expect V1 variant
270            let variant_name = "V1";
271            let expected: CfValue = load_or_generate_json_fixture(cf_name, variant_name)?;
272
273            let snapshot_parent_path = format!("tests/fixtures/cf_versions/{cf_name}");
274            let snapshot_path = format!("{snapshot_parent_path}/{variant_name}.bincode");
275
276            // create snapshot if it doesn't exist
277            if not(Path::new(&snapshot_path).exists()) {
278                // -> CAREFUL WHEN UPDATING SNAPSHOTS <-
279                // the snapshots are supposed to prevent you from breaking the DB accidentally
280                // the DB must be able to deserialize older versions, and those versions can't change
281                // don't reorder variants, remove older variants or modify the data inside existing ones
282                // adding a new snapshot for a new variant is safe as long as you don't mess up in the points above
283                // -> CAREFUL WHEN UPDATING SNAPSHOTS <-
284                if env::var("DANGEROUS_UPDATE_SNAPSHOTS").is_ok() {
285                    use crate::rocks_bincode_config;
286                    let serialized = bincode::encode_to_vec(&expected, rocks_bincode_config())?;
287                    fs::create_dir_all(&snapshot_parent_path)?;
288                    fs::write(snapshot_path, serialized)?;
289                } else {
290                    bail!("snapshot file at '{snapshot_path:?}' doesn't exist and DANGEROUS_UPDATE_SNAPSHOTS is not set");
291                }
292            }
293
294            let snapshots = get_all_bincode_snapshots_from_folder(&snapshot_parent_path)?;
295
296            let [snapshot_path] = snapshots.as_slice() else {
297                bail!("expected 1 snapshot, found {}: {snapshots:?}", snapshots.len());
298            };
299
300            ensure!(
301                snapshot_path == snapshot_path,
302                "snapshot path {snapshot_path:?} doesn't match the expected for v1: {snapshot_path:?}"
303            );
304
305            use crate::rocks_bincode_config;
306            let (deserialized, _) = bincode::decode_from_slice(&fs::read(snapshot_path)?, rocks_bincode_config())?;
307            ensure!(
308                expected == deserialized,
309                "deserialized value doesn't match expected\n deserialized = {deserialized:?}\n expected = {expected:?}",
310            );
311
312            Ok(TestRunConfirmation::new(variant_name))
313        }
314
315        let mut accounts_checker = EnumCoverageDropBombChecker::<CfAccountsValue>::new();
316        let mut accounts_history_checker = EnumCoverageDropBombChecker::<CfAccountsHistoryValue>::new();
317        let mut account_slots_checker = EnumCoverageDropBombChecker::<CfAccountSlotsValue>::new();
318        let mut account_slots_history_checker = EnumCoverageDropBombChecker::<CfAccountSlotsHistoryValue>::new();
319        let mut transactions_checker = EnumCoverageDropBombChecker::<CfTransactionsValue>::new();
320        let mut blocks_by_number_checker = EnumCoverageDropBombChecker::<CfBlocksByNumberValue>::new();
321        let mut blocks_by_hash_checker = EnumCoverageDropBombChecker::<CfBlocksByHashValue>::new();
322
323        accounts_checker.add(test_deserialization::<CfAccountsValue>().unwrap());
324        accounts_history_checker.add(test_deserialization::<CfAccountsHistoryValue>().unwrap());
325        account_slots_checker.add(test_deserialization::<CfAccountSlotsValue>().unwrap());
326        account_slots_history_checker.add(test_deserialization::<CfAccountSlotsHistoryValue>().unwrap());
327        transactions_checker.add(test_deserialization::<CfTransactionsValue>().unwrap());
328        blocks_by_number_checker.add(test_deserialization::<CfBlocksByNumberValue>().unwrap());
329        blocks_by_hash_checker.add(test_deserialization::<CfBlocksByHashValue>().unwrap());
330    }
331}
stratus/eth/storage/permanent/rocks/cf_versions.rs

stratus/eth/storage/permanent/rocks/
cf_versions.rs
