During env open check for pub error which indicated unencrypted payload.

whisper/envelope.go

@@ -11,6 +11,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethutil"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/obscuren/ecies"
 )
 
 const (
@@ -76,7 +77,10 @@ func (self *Envelope) Open(prv *ecdsa.PrivateKey) (msg *Message, err error) {
 	message.Payload = data[dataStart:]
 	if prv != nil {
 		message.Payload, err = crypto.Decrypt(prv, message.Payload)
-		if err != nil {
+		switch err {
+		case ecies.ErrInvalidPublicKey: // Payload isn't encrypted
+			return &message, err
+		default:
 			return nil, fmt.Errorf("unable to open envelope. Decrypt failed: %v", err)
 		}
 	}