Test off curve pubkey (bp #11299) (#11317)

* Allow inspection of signature verification failures

(cherry picked from commit 251f974b50)

* Test that off-curve pubkeys fail signature verify

(cherry picked from commit c421d7f1b8)

Co-authored-by: Trent Nelson <trent@solana.com>

Cargo.lock

@@ -4688,6 +4688,7 @@ dependencies = [
  "bv",
  "byteorder",
  "chrono",
+ "curve25519-dalek 2.1.0",
  "ed25519-dalek",
  "generic-array 0.14.2",
  "hex 0.4.2",

sdk/Cargo.toml

@@ -59,6 +59,7 @@ solana-sdk-macro = { path = "macro", version = "1.2.16" }
 rustversion = "1.0.3"
 
 [dev-dependencies]
+curve25519-dalek = "2.1.0"
 tiny-bip39 = "0.7.0"
 
 [package.metadata.docs.rs]

sdk/src/signature.rs

@@ -57,16 +57,18 @@ impl Signature {
         Self(GenericArray::clone_from_slice(&signature_slice))
     }
 
+    pub(self) fn verify_verbose(
+        &self,
+        pubkey_bytes: &[u8],
+        message_bytes: &[u8],
+    ) -> Result<(), ed25519_dalek::SignatureError> {
+        let publickey = ed25519_dalek::PublicKey::from_bytes(pubkey_bytes)?;
+        let signature = ed25519_dalek::Signature::from_bytes(self.0.as_slice())?;
+        publickey.verify_strict(message_bytes, &signature)
+    }
+
     pub fn verify(&self, pubkey_bytes: &[u8], message_bytes: &[u8]) -> bool {
-        let pubkey = ed25519_dalek::PublicKey::from_bytes(pubkey_bytes);
-        let signature = ed25519_dalek::Signature::from_bytes(self.0.as_slice());
-        if pubkey.is_err() || signature.is_err() {
-            return false;
-        }
-        pubkey
-            .unwrap()
-            .verify_strict(message_bytes, &signature.unwrap())
-            .is_ok()
+        self.verify_verbose(pubkey_bytes, message_bytes).is_ok()
     }
 }
 
@@ -573,4 +575,25 @@ mod tests {
             pubkeys(&[&alice, &bob])
         );
     }
+
+    #[test]
+    fn test_off_curve_pubkey_verify_fails() {
+        // Golden point off the ed25519 curve
+        let off_curve_bytes = bs58::decode("9z5nJyQar1FUxVJxpBXzon6kHehbomeYiDaLi9WAMhCq")
+            .into_vec()
+            .unwrap();
+
+        // Confirm golden's off-curvedness
+        let mut off_curve_bits = [0u8; 32];
+        off_curve_bits.copy_from_slice(&off_curve_bytes);
+        let off_curve_point = curve25519_dalek::edwards::CompressedEdwardsY(off_curve_bits);
+        assert_eq!(off_curve_point.decompress(), None);
+
+        let pubkey = Pubkey::new(&off_curve_bytes);
+        let signature = Signature::default();
+        // Unfortunately, ed25519-dalek doesn't surface the internal error types that we'd ideally
+        // `source()` out of the `SignatureError` returned by `verify_strict()`.  So the best we
+        // can do is `is_err()` here.
+        assert!(signature.verify_verbose(pubkey.as_ref(), &[0u8]).is_err());
+    }
 }