Introduce automatic ABI maintenance mechanism (1/2; prepare) (#10335)

* Introduce automatic ABI maintenance mechanism

* Compile fix...

* Docs fix...

* Programs compilation fix...

* Simplify source credit

Co-authored-by: Michael Vines <mvines@gmail.com>

* Cargo.lock...

Co-authored-by: Michael Vines <mvines@gmail.com>

sdk/src/abi_digester.rs

@@ -0,0 +1,635 @@
+use crate::abi_example::{normalize_type_name, AbiEnumVisitor};
+use crate::hash::{Hash, Hasher};
+
+use log::*;
+
+use serde::ser::Error as SerdeError;
+use serde::ser::*;
+use serde::{Serialize, Serializer};
+
+use std::any::type_name;
+use std::io::Write;
+
+use thiserror::Error;
+
+#[derive(Debug)]
+pub struct AbiDigester {
+    data_types: std::rc::Rc<std::cell::RefCell<Vec<String>>>,
+    depth: usize,
+    for_enum: bool,
+    opaque_scope: Option<String>,
+}
+
+pub type DigestResult = Result<AbiDigester, DigestError>;
+type NoResult = Result<(), DigestError>;
+type Sstr = &'static str;
+
+#[derive(Debug, Error)]
+pub enum DigestError {
+    #[error("Option::None is serialized; no ABI digest for Option::Some")]
+    NoneIsSerialized,
+    #[error("nested error")]
+    Node(Sstr, Box<DigestError>),
+    #[error("leaf error")]
+    Leaf(Sstr, Sstr, Box<DigestError>),
+}
+
+impl SerdeError for DigestError {
+    fn custom<T: std::fmt::Display>(_msg: T) -> DigestError {
+        unreachable!("This error should never be used");
+    }
+}
+
+impl DigestError {
+    pub(crate) fn wrap_by_type<T: ?Sized>(e: DigestError) -> DigestError {
+        DigestError::Node(type_name::<T>(), Box::new(e))
+    }
+
+    pub(crate) fn wrap_by_str(e: DigestError, s: Sstr) -> DigestError {
+        DigestError::Node(s, Box::new(e))
+    }
+}
+
+const INDENT_WIDTH: usize = 4;
+
+impl AbiDigester {
+    pub fn create() -> Self {
+        AbiDigester {
+            data_types: std::rc::Rc::new(std::cell::RefCell::new(vec![])),
+            for_enum: false,
+            depth: 0,
+            opaque_scope: None,
+        }
+    }
+
+    // must create separate instances because we can't pass the single instnace to
+    // `.serialize()` multiple times
+    pub fn create_new(&self) -> Self {
+        Self {
+            data_types: self.data_types.clone(),
+            depth: self.depth,
+            for_enum: false,
+            opaque_scope: self.opaque_scope.clone(),
+        }
+    }
+
+    pub fn create_new_opaque(&self, top_scope: &str) -> Self {
+        Self {
+            data_types: self.data_types.clone(),
+            depth: self.depth,
+            for_enum: false,
+            opaque_scope: Some(top_scope.to_owned()),
+        }
+    }
+
+    pub fn create_child(&self) -> Self {
+        Self {
+            data_types: self.data_types.clone(),
+            depth: self.depth + 1,
+            for_enum: false,
+            opaque_scope: self.opaque_scope.clone(),
+        }
+    }
+
+    pub fn create_enum_child(&self) -> Self {
+        Self {
+            data_types: self.data_types.clone(),
+            depth: self.depth + 1,
+            for_enum: true,
+            opaque_scope: self.opaque_scope.clone(),
+        }
+    }
+
+    pub fn digest_data<T: ?Sized + Serialize>(&mut self, value: &T) -> DigestResult {
+        let type_name = normalize_type_name(type_name::<T>());
+        if type_name.ends_with("__SerializeWith")
+            || (self.opaque_scope.is_some()
+                && type_name.starts_with(self.opaque_scope.as_ref().unwrap()))
+        {
+            // we can't use the AbiEnumVisitor trait for these cases.
+            value.serialize(self.create_new())
+        } else {
+            // Don't call value.visit_for_abi(...) to prefer autoref specialization
+            // resolution for IgnoreAsHelper
+            <&T>::visit_for_abi(&value, &mut self.create_new())
+        }
+    }
+
+    pub fn update(&mut self, strs: &[&str]) {
+        let mut buf = strs
+            .iter()
+            .map(|s| {
+                // this is a bit crude, but just normalize all strings as if they're
+                // `type_name`s!
+                normalize_type_name(s)
+            })
+            .collect::<Vec<_>>()
+            .join(" ");
+        buf = format!("{:0width$}{}\n", "", buf, width = self.depth * INDENT_WIDTH);
+        info!("updating with: {}", buf.trim_end());
+        (*self.data_types.borrow_mut()).push(buf);
+    }
+
+    pub fn update_with_type<T: ?Sized>(&mut self, label: &str) {
+        self.update(&[label, type_name::<T>()]);
+    }
+
+    pub fn update_with_string(&mut self, label: String) {
+        self.update(&[&label]);
+    }
+
+    fn digest_primitive<T: Serialize>(mut self) -> Result<AbiDigester, DigestError> {
+        self.update_with_type::<T>("primitive");
+        Ok(self)
+    }
+
+    fn digest_element<T: ?Sized + Serialize>(&mut self, v: &T) -> NoResult {
+        self.update_with_type::<T>("element");
+        self.create_child().digest_data(v).map(|_| ())
+    }
+
+    fn digest_named_field<T: ?Sized + Serialize>(&mut self, key: Sstr, v: &T) -> NoResult {
+        self.update_with_string(format!("field {}: {}", key, type_name::<T>()));
+        self.create_child()
+            .digest_data(v)
+            .map(|_| ())
+            .map_err(|e| DigestError::wrap_by_str(e, key))
+    }
+
+    fn digest_unnamed_field<T: ?Sized + Serialize>(&mut self, v: &T) -> NoResult {
+        self.update_with_type::<T>("field");
+        self.create_child().digest_data(v).map(|_| ())
+    }
+
+    fn check_for_enum(&mut self, label: &'static str, variant: &'static str) -> NoResult {
+        if !self.for_enum {
+            panic!("derive AbiEnumVisitor or implement it for the enum, which contains a variant ({}) named {}", label, variant);
+        }
+        Ok(())
+    }
+
+    pub fn finalize(self) -> Hash {
+        let mut hasher = Hasher::default();
+
+        for buf in (*self.data_types.borrow()).iter() {
+            hasher.hash(buf.as_bytes());
+        }
+
+        let hash = hasher.result();
+
+        if let Ok(dir) = std::env::var("SOLANA_ABI_DUMP_DIR") {
+            let thread_name = std::thread::current()
+                .name()
+                .unwrap_or("unknown-test-thread")
+                .replace(':', "_");
+            if thread_name == "main" {
+                error!("Bad thread name detected for dumping; Maybe, --test-threads=1? Sorry, SOLANA_ABI_DUMP_DIR doesn't work under 1; increase it");
+            }
+
+            let path = format!("{}/{}_{}", dir, thread_name, hash,);
+            let mut file = std::fs::File::create(path).unwrap();
+            for buf in (*self.data_types.borrow()).iter() {
+                file.write_all(buf.as_bytes()).unwrap();
+            }
+            file.sync_data().unwrap();
+        }
+
+        hash
+    }
+}
+
+impl Serializer for AbiDigester {
+    type Ok = Self;
+    type Error = DigestError;
+    type SerializeSeq = Self;
+    type SerializeTuple = Self;
+    type SerializeTupleStruct = Self;
+    type SerializeTupleVariant = Self;
+    type SerializeMap = Self;
+    type SerializeStruct = Self;
+    type SerializeStructVariant = Self;
+
+    fn serialize_bool(self, _data: bool) -> DigestResult {
+        self.digest_primitive::<bool>()
+    }
+
+    fn serialize_i8(self, _data: i8) -> DigestResult {
+        self.digest_primitive::<i8>()
+    }
+
+    fn serialize_i16(self, _data: i16) -> DigestResult {
+        self.digest_primitive::<i16>()
+    }
+
+    fn serialize_i32(self, _data: i32) -> DigestResult {
+        self.digest_primitive::<i32>()
+    }
+
+    fn serialize_i64(self, _data: i64) -> DigestResult {
+        self.digest_primitive::<i64>()
+    }
+
+    fn serialize_i128(self, _data: i128) -> DigestResult {
+        self.digest_primitive::<i128>()
+    }
+
+    fn serialize_u8(self, _data: u8) -> DigestResult {
+        self.digest_primitive::<u8>()
+    }
+
+    fn serialize_u16(self, _data: u16) -> DigestResult {
+        self.digest_primitive::<u16>()
+    }
+
+    fn serialize_u32(self, _data: u32) -> DigestResult {
+        self.digest_primitive::<u32>()
+    }
+
+    fn serialize_u64(self, _data: u64) -> DigestResult {
+        self.digest_primitive::<u64>()
+    }
+
+    fn serialize_u128(self, _data: u128) -> DigestResult {
+        self.digest_primitive::<u128>()
+    }
+
+    fn serialize_f32(self, _data: f32) -> DigestResult {
+        self.digest_primitive::<f32>()
+    }
+
+    fn serialize_f64(self, _data: f64) -> DigestResult {
+        self.digest_primitive::<f64>()
+    }
+
+    fn serialize_char(self, _data: char) -> DigestResult {
+        self.digest_primitive::<char>()
+    }
+
+    fn serialize_str(self, _data: &str) -> DigestResult {
+        self.digest_primitive::<&str>()
+    }
+
+    fn serialize_unit(self) -> DigestResult {
+        self.digest_primitive::<()>()
+    }
+
+    fn serialize_bytes(mut self, v: &[u8]) -> DigestResult {
+        self.update_with_string(format!("bytes [u8] (len = {})", v.len()));
+        Ok(self)
+    }
+
+    fn serialize_none(self) -> DigestResult {
+        Err(DigestError::NoneIsSerialized)
+    }
+
+    fn serialize_some<T>(mut self, v: &T) -> DigestResult
+    where
+        T: ?Sized + Serialize,
+    {
+        // emulate the ABI digest for the Option enum; see TestMyOption
+        self.update(&["enum Option (variants = 2)"]);
+        let mut variant_digester = self.create_child();
+
+        variant_digester.update_with_string("variant(0) None (unit)".to_owned());
+        variant_digester
+            .update_with_string(format!("variant(1) Some({}) (newtype)", type_name::<T>()));
+        variant_digester.create_child().digest_data(v)
+    }
+
+    fn serialize_unit_struct(mut self, name: Sstr) -> DigestResult {
+        self.update(&["struct", name, "(unit)"]);
+        Ok(self)
+    }
+
+    fn serialize_unit_variant(mut self, _name: Sstr, index: u32, variant: Sstr) -> DigestResult {
+        self.check_for_enum("unit_variant", variant)?;
+        self.update_with_string(format!("variant({}) {} (unit)", index, variant));
+        Ok(self)
+    }
+
+    fn serialize_newtype_struct<T>(mut self, name: Sstr, v: &T) -> DigestResult
+    where
+        T: ?Sized + Serialize,
+    {
+        self.update_with_string(format!("struct {}({}) (newtype)", name, type_name::<T>()));
+        self.create_child()
+            .digest_data(v)
+            .map_err(|e| DigestError::wrap_by_str(e, "newtype_struct"))
+    }
+
+    fn serialize_newtype_variant<T>(
+        mut self,
+        _name: Sstr,
+        i: u32,
+        variant: Sstr,
+        v: &T,
+    ) -> DigestResult
+    where
+        T: ?Sized + Serialize,
+    {
+        self.check_for_enum("newtype_variant", variant)?;
+        self.update_with_string(format!(
+            "variant({}) {}({}) (newtype)",
+            i,
+            variant,
+            type_name::<T>()
+        ));
+        self.create_child()
+            .digest_data(v)
+            .map_err(|e| DigestError::wrap_by_str(e, "newtype_variant"))
+    }
+
+    fn serialize_seq(mut self, len: Option<usize>) -> DigestResult {
+        self.update_with_string(format!("seq (elements = {})", len.unwrap()));
+        Ok(self.create_child())
+    }
+
+    fn serialize_tuple(mut self, len: usize) -> DigestResult {
+        self.update_with_string(format!("tuple (elements = {})", len));
+        Ok(self.create_child())
+    }
+
+    fn serialize_tuple_struct(mut self, name: Sstr, len: usize) -> DigestResult {
+        self.update_with_string(format!("struct {} (fields = {}) (tuple)", name, len));
+        Ok(self.create_child())
+    }
+
+    fn serialize_tuple_variant(
+        mut self,
+        _name: Sstr,
+        i: u32,
+        variant: Sstr,
+        len: usize,
+    ) -> DigestResult {
+        self.check_for_enum("tuple_variant", variant)?;
+        self.update_with_string(format!("variant({}) {} (fields = {})", i, variant, len));
+        Ok(self.create_child())
+    }
+
+    fn serialize_map(mut self, len: Option<usize>) -> DigestResult {
+        self.update_with_string(format!("map (entries = {})", len.unwrap()));
+        Ok(self.create_child())
+    }
+
+    fn serialize_struct(mut self, name: Sstr, len: usize) -> DigestResult {
+        self.update_with_string(format!("struct {} (fields = {})", name, len));
+        Ok(self.create_child())
+    }
+
+    fn serialize_struct_variant(
+        mut self,
+        _name: Sstr,
+        i: u32,
+        variant: Sstr,
+        len: usize,
+    ) -> DigestResult {
+        self.check_for_enum("struct_variant", variant)?;
+        self.update_with_string(format!(
+            "variant({}) struct {} (fields = {})",
+            i, variant, len
+        ));
+        Ok(self.create_child())
+    }
+}
+
+impl SerializeSeq for AbiDigester {
+    type Ok = Self;
+    type Error = DigestError;
+
+    fn serialize_element<T: ?Sized + Serialize>(&mut self, data: &T) -> NoResult {
+        self.digest_element(data)
+    }
+
+    fn end(self) -> DigestResult {
+        Ok(self)
+    }
+}
+
+impl SerializeTuple for AbiDigester {
+    type Ok = Self;
+    type Error = DigestError;
+
+    fn serialize_element<T: ?Sized + Serialize>(&mut self, data: &T) -> NoResult {
+        self.digest_element(data)
+    }
+
+    fn end(self) -> DigestResult {
+        Ok(self)
+    }
+}
+impl SerializeTupleStruct for AbiDigester {
+    type Ok = Self;
+    type Error = DigestError;
+
+    fn serialize_field<T: ?Sized + Serialize>(&mut self, data: &T) -> NoResult {
+        self.digest_unnamed_field(data)
+    }
+
+    fn end(self) -> DigestResult {
+        Ok(self)
+    }
+}
+
+impl SerializeTupleVariant for AbiDigester {
+    type Ok = Self;
+    type Error = DigestError;
+
+    fn serialize_field<T: ?Sized + Serialize>(&mut self, data: &T) -> NoResult {
+        self.digest_unnamed_field(data)
+    }
+
+    fn end(self) -> DigestResult {
+        Ok(self)
+    }
+}
+
+impl SerializeMap for AbiDigester {
+    type Ok = Self;
+    type Error = DigestError;
+
+    fn serialize_key<T: ?Sized + Serialize>(&mut self, key: &T) -> NoResult {
+        self.update_with_type::<T>("key");
+        self.create_child().digest_data(key).map(|_| ())
+    }
+
+    fn serialize_value<T: ?Sized + Serialize>(&mut self, value: &T) -> NoResult {
+        self.update_with_type::<T>("value");
+        self.create_child().digest_data(value).map(|_| ())
+    }
+
+    fn end(self) -> DigestResult {
+        Ok(self)
+    }
+}
+
+impl SerializeStruct for AbiDigester {
+    type Ok = Self;
+    type Error = DigestError;
+
+    fn serialize_field<T: ?Sized + Serialize>(&mut self, key: Sstr, data: &T) -> NoResult {
+        self.digest_named_field(key, data)
+    }
+
+    fn end(self) -> DigestResult {
+        Ok(self)
+    }
+}
+
+impl SerializeStructVariant for AbiDigester {
+    type Ok = Self;
+    type Error = DigestError;
+
+    fn serialize_field<T: ?Sized + Serialize>(&mut self, key: Sstr, data: &T) -> NoResult {
+        self.digest_named_field(key, data)
+    }
+
+    fn end(self) -> DigestResult {
+        Ok(self)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+    use std::sync::atomic::AtomicIsize;
+
+    #[frozen_abi(digest = "CQiGCzsGquChkwffHjZKFqa3tCYtS3GWYRRYX7iDR38Q")]
+    type TestTypeAlias = i32;
+
+    #[frozen_abi(digest = "Apwkp9Ah9zKirzwuSzVoU9QRc43EghpkD1nGVakJLfUY")]
+    #[derive(Serialize, AbiExample)]
+    struct TestStruct {
+        test_field: i8,
+        test_field2: i8,
+    }
+
+    #[frozen_abi(digest = "4LbuvQLX78XPbm4hqqZcHFHpseDJcw4qZL9EUZXSi2Ss")]
+    #[derive(Serialize, AbiExample)]
+    struct TestTupleStruct(i8, i8);
+
+    #[frozen_abi(digest = "FNHa6mNYJZa59Fwbipep5dXRXcFreaDHn9jEUZEH1YLv")]
+    #[derive(Serialize, AbiExample)]
+    struct TestNewtypeStruct(i8);
+
+    #[frozen_abi(digest = "5qio5qYurHDv6fq5kcwP2ue2RBEazSZF8CPk2kUuwC2j")]
+    #[derive(Serialize, AbiExample)]
+    struct TestStructReversed {
+        test_field2: i8,
+        test_field: i8,
+    }
+
+    #[frozen_abi(digest = "DLLrTWprsMjdJGR447A4mui9HpqxbKdsFXBfaWPcwhny")]
+    #[derive(Serialize, AbiExample)]
+    struct TestStructAnotherType {
+        test_field: i16,
+        test_field2: i8,
+    }
+
+    #[frozen_abi(digest = "Hv597t4PieHYvgiXnwRSpKBRTWqteUS4nHZHY6ZxX69v")]
+    #[derive(Serialize, AbiExample)]
+    struct TestNest {
+        nested_field: [TestStruct; 5],
+    }
+
+    #[frozen_abi(digest = "GttWH8FAY3teUjTaSds9mL3YbiDQ7qWw7WAvDXKd4ZzX")]
+    type TestUnitStruct = std::marker::PhantomData<i8>;
+
+    #[frozen_abi(digest = "2zvXde11f8sNnFbc9E6ZZeFxV7D2BTVLKEZmNTsCDBpS")]
+    #[derive(Serialize, AbiExample, AbiEnumVisitor)]
+    enum TestEnum {
+        VARIANT1,
+        VARIANT2,
+    }
+
+    #[frozen_abi(digest = "6keb3v7GXLahhL6zoinzCWwSvB3KhmvZMB3tN2mamAm3")]
+    #[derive(Serialize, AbiExample, AbiEnumVisitor)]
+    enum TestTupleVariant {
+        VARIANT1(u8, u16),
+        VARIANT2(u8, u16),
+    }
+
+    #[frozen_abi(digest = "CKxzv7VjyUrNR9fGJpTpKyMBWJM4gepKshCS8oV14T1Q")]
+    #[derive(Serialize, AbiExample)]
+    struct TestVecEnum {
+        enums: Vec<TestTupleVariant>,
+    }
+
+    #[derive(Serialize, AbiExample)]
+    struct TestGenericStruct<T: Ord> {
+        test_field: T,
+    }
+
+    #[frozen_abi(digest = "2Dr5k3Z513mV4KrGeUfcMwjsVHLmVyLiZarmfnXawEbf")]
+    type TestConcreteStruct = TestGenericStruct<i64>;
+
+    #[derive(Serialize, AbiExample, AbiEnumVisitor)]
+    enum TestGenericEnum<T: serde::Serialize + Sized + Ord> {
+        TestVariant(T),
+    }
+
+    #[frozen_abi(digest = "2B2HqxHaziSfW3kdxJqV9vEMpCpRaEipXL6Bskv1GV7J")]
+    type TestConcreteEnum = TestGenericEnum<u128>;
+
+    #[frozen_abi(digest = "GyExD8nkYb9e6tijFL5S1gFtdN9GfY6L2sUDjTLhVGn4")]
+    type TestMap = HashMap<char, i128>;
+
+    #[frozen_abi(digest = "AFLTVyVBkjc1SAPnzyuwTvmie994LMhJGN7PrP7hCVwL")]
+    type TestVec = Vec<f32>;
+
+    #[frozen_abi(digest = "F5RniBQtNMBiDnyLEf72aQKHskV1TuBrD4jrEH5odPAW")]
+    type TestArray = [f64; 10];
+
+    #[frozen_abi(digest = "8cgZGpckC4dFovh3QuZpgvcvK2125ig7P4HsK9KCw39N")]
+    type TestUnit = ();
+
+    #[frozen_abi(digest = "FgnBPy2T5iNNbykMteq1M4FRpNeSkzRoi9oXeCjEW6uq")]
+    type TestResult = Result<u8, u16>;
+
+    #[frozen_abi(digest = "F5s6YyJkfz7LM56q5j9RzTLa7QX4Utx1ecNkHX5UU9Fp")]
+    type TestAtomic = AtomicIsize;
+
+    #[frozen_abi(digest = "7rH7gnEhJ8YouzqPT6VPyUDELvL51DGednSPcoLXG2rg")]
+    type TestOptionWithIsize = Option<isize>;
+
+    #[derive(Serialize, AbiExample, AbiEnumVisitor)]
+    enum TestMyOption<T: serde::Serialize + Sized + Ord> {
+        None,
+        Some(T),
+    }
+    #[frozen_abi(digest = "BzXkoRacijFTCPW4PyyvhkqMVgcuhmvPXjZfMsHJCeet")]
+    type TestMyOptionWithIsize = TestMyOption<isize>;
+
+    #[frozen_abi(digest = "9PMdHRb49BpkywrmPoJyZWMsEmf5E1xgmsFGkGmea5RW")]
+    type TestBitVec = bv::BitVec<u64>;
+
+    mod skip_should_be_same {
+        #[frozen_abi(digest = "4LbuvQLX78XPbm4hqqZcHFHpseDJcw4qZL9EUZXSi2Ss")]
+        #[derive(Serialize, AbiExample)]
+        struct TestTupleStruct(i8, i8, #[serde(skip)] i8);
+
+        #[frozen_abi(digest = "Hk7BYjZ71upWQJAx2PqoNcapggobPmFbMJd34xVdvRso")]
+        #[derive(Serialize, AbiExample)]
+        struct TestStruct {
+            test_field: i8,
+            #[serde(skip)]
+            _skipped_test_field: i8,
+        }
+
+        #[frozen_abi(digest = "2zvXde11f8sNnFbc9E6ZZeFxV7D2BTVLKEZmNTsCDBpS")]
+        #[derive(Serialize, AbiExample, AbiEnumVisitor)]
+        enum TestEnum {
+            VARIANT1,
+            VARIANT2,
+            #[serde(skip)]
+            #[allow(dead_code)]
+            VARIANT3,
+        }
+
+        #[frozen_abi(digest = "6keb3v7GXLahhL6zoinzCWwSvB3KhmvZMB3tN2mamAm3")]
+        #[derive(Serialize, AbiExample, AbiEnumVisitor)]
+        enum TestTupleVariant {
+            VARIANT1(u8, u16),
+            VARIANT2(u8, u16, #[serde(skip)] u32),
+        }
+    }
+}

sdk/src/abi_example.rs

@@ -0,0 +1,507 @@
+use crate::abi_digester::{AbiDigester, DigestError, DigestResult};
+
+use log::*;
+
+use serde::Serialize;
+use std::any::type_name;
+
+pub trait AbiExample: Sized {
+    fn example() -> Self;
+}
+
+// Following code snippets are copied and adapted from the official rustc implementation to
+// implement AbiExample trait for most of basic types.
+// These are licensed under Apache-2.0 + MIT (compatible because we're Apache-2.0)
+
+// Source: https://github.com/rust-lang/rust/blob/ba18875557aabffe386a2534a1aa6118efb6ab88/src/libcore/tuple.rs#L7
+macro_rules! tuple_example_impls {
+    ($(
+        $Tuple:ident {
+            $(($idx:tt) -> $T:ident)+
+        }
+    )+) => {
+        $(
+            impl<$($T:AbiExample),+> AbiExample for ($($T,)+) {
+                fn example() -> Self {
+                        ($({ let x: $T = AbiExample::example(); x},)+)
+                }
+            }
+        )+
+    }
+}
+
+// Source: https://github.com/rust-lang/rust/blob/ba18875557aabffe386a2534a1aa6118efb6ab88/src/libcore/tuple.rs#L110
+tuple_example_impls! {
+    Tuple1 {
+        (0) -> A
+    }
+    Tuple2 {
+        (0) -> A
+        (1) -> B
+    }
+    Tuple3 {
+        (0) -> A
+        (1) -> B
+        (2) -> C
+    }
+    Tuple4 {
+        (0) -> A
+        (1) -> B
+        (2) -> C
+        (3) -> D
+    }
+    Tuple5 {
+        (0) -> A
+        (1) -> B
+        (2) -> C
+        (3) -> D
+        (4) -> E
+    }
+    Tuple6 {
+        (0) -> A
+        (1) -> B
+        (2) -> C
+        (3) -> D
+        (4) -> E
+        (5) -> F
+    }
+    Tuple7 {
+        (0) -> A
+        (1) -> B
+        (2) -> C
+        (3) -> D
+        (4) -> E
+        (5) -> F
+        (6) -> G
+    }
+    Tuple8 {
+        (0) -> A
+        (1) -> B
+        (2) -> C
+        (3) -> D
+        (4) -> E
+        (5) -> F
+        (6) -> G
+        (7) -> H
+    }
+    Tuple9 {
+        (0) -> A
+        (1) -> B
+        (2) -> C
+        (3) -> D
+        (4) -> E
+        (5) -> F
+        (6) -> G
+        (7) -> H
+        (8) -> I
+    }
+    Tuple10 {
+        (0) -> A
+        (1) -> B
+        (2) -> C
+        (3) -> D
+        (4) -> E
+        (5) -> F
+        (6) -> G
+        (7) -> H
+        (8) -> I
+        (9) -> J
+    }
+    Tuple11 {
+        (0) -> A
+        (1) -> B
+        (2) -> C
+        (3) -> D
+        (4) -> E
+        (5) -> F
+        (6) -> G
+        (7) -> H
+        (8) -> I
+        (9) -> J
+        (10) -> K
+    }
+    Tuple12 {
+        (0) -> A
+        (1) -> B
+        (2) -> C
+        (3) -> D
+        (4) -> E
+        (5) -> F
+        (6) -> G
+        (7) -> H
+        (8) -> I
+        (9) -> J
+        (10) -> K
+        (11) -> L
+    }
+}
+
+// Source: https://github.com/rust-lang/rust/blob/ba18875557aabffe386a2534a1aa6118efb6ab88/src/libcore/array/mod.rs#L417
+macro_rules! array_example_impls {
+    {$n:expr, $t:ident $($ts:ident)*} => {
+        impl<T> AbiExample for [T; $n] where T: AbiExample {
+            fn example() -> Self {
+                [$t::example(), $($ts::example()),*]
+            }
+        }
+        array_example_impls!{($n - 1), $($ts)*}
+    };
+    {$n:expr,} => {
+        impl<T> AbiExample for [T; $n] {
+        fn example() -> Self { [] }
+        }
+    };
+}
+
+array_example_impls! {32, T T T T T T T T T T T T T T T T T T T T T T T T T T T T T T T T}
+
+// Source: https://github.com/rust-lang/rust/blob/ba18875557aabffe386a2534a1aa6118efb6ab88/src/libcore/default.rs#L137
+macro_rules! example_impls {
+    ($t:ty, $v:expr) => {
+        impl AbiExample for $t {
+            fn example() -> Self {
+                $v
+            }
+        }
+    };
+}
+
+example_impls! { (), () }
+example_impls! { bool, false }
+example_impls! { char, '\x00' }
+
+example_impls! { usize, 0 }
+example_impls! { u8, 0 }
+example_impls! { u16, 0 }
+example_impls! { u32, 0 }
+example_impls! { u64, 0 }
+example_impls! { u128, 0 }
+
+example_impls! { isize, 0 }
+example_impls! { i8, 0 }
+example_impls! { i16, 0 }
+example_impls! { i32, 0 }
+example_impls! { i64, 0 }
+example_impls! { i128, 0 }
+
+example_impls! { f32, 0.0f32 }
+example_impls! { f64, 0.0f64 }
+example_impls! { String, String::new() }
+example_impls! { std::time::Duration, std::time::Duration::from_secs(0) }
+
+use std::sync::atomic::*;
+
+// Source: https://github.com/rust-lang/rust/blob/ba18875557aabffe386a2534a1aa6118efb6ab88/src/libcore/sync/atomic.rs#L1199
+macro_rules! atomic_example_impls {
+    ($atomic_type: ident) => {
+        impl AbiExample for $atomic_type {
+            fn example() -> Self {
+                Self::new(AbiExample::example())
+            }
+        }
+    };
+}
+atomic_example_impls! { AtomicU8 }
+atomic_example_impls! { AtomicU16 }
+atomic_example_impls! { AtomicU32 }
+atomic_example_impls! { AtomicU64 }
+atomic_example_impls! { AtomicUsize }
+atomic_example_impls! { AtomicI8 }
+atomic_example_impls! { AtomicI16 }
+atomic_example_impls! { AtomicI32 }
+atomic_example_impls! { AtomicI64 }
+atomic_example_impls! { AtomicIsize }
+atomic_example_impls! { AtomicBool }
+
+#[cfg(not(feature = "program"))]
+use generic_array::{ArrayLength, GenericArray};
+#[cfg(not(feature = "program"))]
+impl<T: Default, U: ArrayLength<T>> AbiExample for GenericArray<T, U> {
+    fn example() -> Self {
+        Self::default()
+    }
+}
+
+use bv::{BitVec, BlockType};
+impl<T: BlockType> AbiExample for BitVec<T> {
+    fn example() -> Self {
+        Self::default()
+    }
+}
+
+impl<T: BlockType> IgnoreAsHelper for BitVec<T> {}
+impl<T: BlockType> EvenAsOpaque for BitVec<T> {}
+
+pub(crate) fn normalize_type_name(type_name: &str) -> String {
+    type_name.chars().filter(|c| *c != '&').collect()
+}
+
+type Placeholder = ();
+
+impl<T: Sized> AbiExample for T {
+    default fn example() -> Self {
+        <Placeholder>::type_erased_example()
+    }
+}
+
+// this works like a type erasure and a hatch to escape type error to runtime error
+trait TypeErasedExample<T> {
+    fn type_erased_example() -> T;
+}
+
+impl<T: Sized> TypeErasedExample<T> for Placeholder {
+    default fn type_erased_example() -> T {
+        panic!(
+            "derive or implement AbiExample/AbiEnumVisitor for {}",
+            type_name::<T>()
+        );
+    }
+}
+
+impl<T: Default + Serialize> TypeErasedExample<T> for Placeholder {
+    default fn type_erased_example() -> T {
+        let original_type_name = type_name::<T>();
+        let normalized_type_name = normalize_type_name(original_type_name);
+
+        if normalized_type_name.starts_with("solana") {
+            panic!(
+                "derive or implement AbiExample/AbiEnumVisitor for {}",
+                original_type_name
+            );
+        } else {
+            panic!(
+                "new unrecognized type for ABI digest!: {}",
+                original_type_name
+            )
+        }
+    }
+}
+
+impl<T: AbiExample> AbiExample for Option<T> {
+    fn example() -> Self {
+        info!("AbiExample for (Option<T>): {}", type_name::<Self>());
+        Some(T::example())
+    }
+}
+
+impl<O: AbiExample, E: AbiExample> AbiExample for Result<O, E> {
+    fn example() -> Self {
+        info!("AbiExample for (Result<O, E>): {}", type_name::<Self>());
+        Ok(O::example())
+    }
+}
+
+impl<T: AbiExample> AbiExample for Box<T> {
+    fn example() -> Self {
+        info!("AbiExample for (Box<T>): {}", type_name::<Self>());
+        Box::new(T::example())
+    }
+}
+
+impl<T> AbiExample for Box<dyn Fn(&mut T) -> () + Sync + Send> {
+    fn example() -> Self {
+        info!("AbiExample for (Box<T>): {}", type_name::<Self>());
+        Box::new(move |_t: &mut T| {})
+    }
+}
+
+impl<T: AbiExample> AbiExample for Box<[T]> {
+    fn example() -> Self {
+        info!("AbiExample for (Box<[T]>): {}", type_name::<Self>());
+        Box::new([T::example()])
+    }
+}
+
+impl<T: AbiExample> AbiExample for std::marker::PhantomData<T> {
+    fn example() -> Self {
+        info!("AbiExample for (PhantomData<T>): {}", type_name::<Self>());
+        <std::marker::PhantomData<T>>::default()
+    }
+}
+
+impl<T: AbiExample> AbiExample for std::sync::Arc<T> {
+    fn example() -> Self {
+        info!("AbiExample for (Arc<T>): {}", type_name::<Self>());
+        std::sync::Arc::new(T::example())
+    }
+}
+
+impl<T: AbiExample> AbiExample for std::rc::Rc<T> {
+    fn example() -> Self {
+        info!("AbiExample for (Rc<T>): {}", type_name::<Self>());
+        std::rc::Rc::new(T::example())
+    }
+}
+
+impl<T: AbiExample> AbiExample for std::sync::Mutex<T> {
+    fn example() -> Self {
+        info!("AbiExample for (Mutex<T>): {}", type_name::<Self>());
+        std::sync::Mutex::new(T::example())
+    }
+}
+
+impl<T: AbiExample> AbiExample for std::sync::RwLock<T> {
+    fn example() -> Self {
+        info!("AbiExample for (RwLock<T>): {}", type_name::<Self>());
+        std::sync::RwLock::new(T::example())
+    }
+}
+
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
+
+impl<
+        T: std::cmp::Eq + std::hash::Hash + AbiExample,
+        S: AbiExample,
+        H: std::hash::BuildHasher + Default,
+    > AbiExample for HashMap<T, S, H>
+{
+    fn example() -> Self {
+        info!("AbiExample for (HashMap<T, S, H>): {}", type_name::<Self>());
+        let mut map = HashMap::default();
+        map.insert(T::example(), S::example());
+        map
+    }
+}
+
+impl<T: std::cmp::Ord + AbiExample, S: AbiExample> AbiExample for BTreeMap<T, S> {
+    fn example() -> Self {
+        info!("AbiExample for (BTreeMap<T, S>): {}", type_name::<Self>());
+        let mut map = BTreeMap::default();
+        map.insert(T::example(), S::example());
+        map
+    }
+}
+
+impl<T: AbiExample> AbiExample for Vec<T> {
+    fn example() -> Self {
+        info!("AbiExample for (Vec<T>): {}", type_name::<Self>());
+        vec![T::example()]
+    }
+}
+
+impl<T: std::cmp::Eq + std::hash::Hash + AbiExample, H: std::hash::BuildHasher + Default> AbiExample
+    for HashSet<T, H>
+{
+    fn example() -> Self {
+        info!("AbiExample for (HashSet<T, H>): {}", type_name::<Self>());
+        let mut set: HashSet<T, H> = HashSet::default();
+        set.insert(T::example());
+        set
+    }
+}
+
+impl<T: std::cmp::Ord + AbiExample> AbiExample for BTreeSet<T> {
+    fn example() -> Self {
+        info!("AbiExample for (BTreeSet<T>): {}", type_name::<Self>());
+        let mut set: BTreeSet<T> = BTreeSet::default();
+        set.insert(T::example());
+        set
+    }
+}
+
+#[cfg(not(feature = "program"))]
+impl AbiExample for memmap::MmapMut {
+    fn example() -> Self {
+        memmap::MmapMut::map_anon(1).expect("failed to map the data file")
+    }
+}
+
+#[cfg(not(feature = "program"))]
+impl AbiExample for std::path::PathBuf {
+    fn example() -> Self {
+        std::path::PathBuf::from(String::example())
+    }
+}
+
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+impl AbiExample for SocketAddr {
+    fn example() -> Self {
+        SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0)
+    }
+}
+
+// This is a control flow indirection needed for digesting all variants of an enum
+pub trait AbiEnumVisitor: Serialize {
+    fn visit_for_abi(&self, digester: &mut AbiDigester) -> DigestResult;
+}
+
+pub trait IgnoreAsHelper {}
+pub trait EvenAsOpaque {}
+
+impl<T: Serialize + ?Sized> AbiEnumVisitor for T {
+    default fn visit_for_abi(&self, _digester: &mut AbiDigester) -> DigestResult {
+        unreachable!(
+            "AbiEnumVisitor must be implemented for {}",
+            type_name::<T>()
+        );
+    }
+}
+
+impl<T: Serialize + ?Sized + AbiExample> AbiEnumVisitor for T {
+    default fn visit_for_abi(&self, digester: &mut AbiDigester) -> DigestResult {
+        info!("AbiEnumVisitor for (default): {}", type_name::<T>());
+        T::example()
+            .serialize(digester.create_new())
+            .map_err(DigestError::wrap_by_type::<T>)
+    }
+}
+
+// even (experimental) rust specialization isn't enough for us, resort to
+// the autoref hack: https://github.com/dtolnay/case-studies/blob/master/autoref-specialization/README.md
+// relevant test: TestVecEnum
+impl<T: Serialize + ?Sized + AbiEnumVisitor> AbiEnumVisitor for &T {
+    default fn visit_for_abi(&self, digester: &mut AbiDigester) -> DigestResult {
+        info!("AbiEnumVisitor for (&default): {}", type_name::<T>());
+        // Don't call self.visit_for_abi(...) to avoid the infinite recursion!
+        T::visit_for_abi(&self, digester)
+    }
+}
+
+// force to call self.serialize instead of T::visit_for_abi() for serialization
+// helper structs like ad-hoc iterator `struct`s
+impl<T: Serialize + IgnoreAsHelper> AbiEnumVisitor for &T {
+    default fn visit_for_abi(&self, digester: &mut AbiDigester) -> DigestResult {
+        info!("AbiEnumVisitor for (IgnoreAsHelper): {}", type_name::<T>());
+        self.serialize(digester.create_new())
+            .map_err(DigestError::wrap_by_type::<T>)
+    }
+}
+
+// force to call self.serialize instead of T::visit_for_abi() to work around the
+// inability of implementing AbiExample for private structs from other crates
+impl<T: Serialize + IgnoreAsHelper + EvenAsOpaque> AbiEnumVisitor for &T {
+    default fn visit_for_abi(&self, digester: &mut AbiDigester) -> DigestResult {
+        info!("AbiEnumVisitor for (IgnoreAsOpaque): {}", type_name::<T>());
+        let top_scope = type_name::<T>().split("::").next().unwrap();
+        self.serialize(digester.create_new_opaque(top_scope))
+            .map_err(DigestError::wrap_by_type::<T>)
+    }
+}
+
+// Because Option and Result enums are so common enums, provide generic trait implementations
+// The digesting pattern must match with what is derived from #[derive(AbiEnumVisitor)]
+impl<T: AbiEnumVisitor> AbiEnumVisitor for Option<T> {
+    fn visit_for_abi(&self, digester: &mut AbiDigester) -> DigestResult {
+        info!("AbiEnumVisitor for (Option<T>): {}", type_name::<Self>());
+
+        let variant: Self = Option::Some(T::example());
+        // serde calls serialize_some(); not serialize_variant();
+        // so create_new is correct, not create_enum_child or create_enum_new
+        variant.serialize(digester.create_new())
+    }
+}
+
+impl<O: AbiEnumVisitor, E: AbiEnumVisitor> AbiEnumVisitor for Result<O, E> {
+    fn visit_for_abi(&self, digester: &mut AbiDigester) -> DigestResult {
+        info!("AbiEnumVisitor for (Result<O, E>): {}", type_name::<Self>());
+
+        digester.update(&["enum Result (variants = 2)"]);
+        let variant: Self = Result::Ok(O::example());
+        variant.serialize(digester.create_enum_child())?;
+
+        let variant: Self = Result::Err(E::example());
+        variant.serialize(digester.create_enum_child())?;
+
+        Ok(digester.create_child())
+    }
+}

sdk/src/lib.rs

@@ -1,6 +1,13 @@
+#![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(specialization))]
+
 // Allows macro expansion of `use ::solana_sdk::*` to work within this crate
 extern crate self as solana_sdk;
 
+#[cfg(RUSTC_WITH_SPECIALIZATION)]
+pub mod abi_digester;
+#[cfg(RUSTC_WITH_SPECIALIZATION)]
+pub mod abi_example;
+
 pub mod account;
 pub mod account_utils;
 pub mod bpf_loader;
@@ -91,3 +98,8 @@ pub mod transport;
 extern crate serde_derive;
 pub extern crate bs58;
 extern crate log as logger;
+
+#[cfg(RUSTC_WITH_SPECIALIZATION)]
+#[cfg(test)]
+#[macro_use]
+extern crate solana_sdk_macro_frozen_abi;