watchtower: Add Slack/Discord sanity failure notification (#7467)

automerge

watchtower/src/main.rs

@@ -1,5 +1,8 @@
 //! A command-line executable for monitoring the health of a cluster
 
+mod notifier;
+
+use crate::notifier::Notifier;
 use clap::{crate_description, crate_name, value_t_or_exit, App, Arg};
 use log::*;
 use solana_clap_utils::input_validators::is_url;
@@ -38,6 +41,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
 
     let rpc_client = RpcClient::new(json_rpc_url.to_string());
 
+    let notifier = Notifier::new();
     let mut last_transaction_count = 0;
     loop {
         let ok = rpc_client
@@ -111,6 +115,9 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                 });
 
         datapoint_info!("watchtower-sanity", ("ok", ok, bool));
+        if !ok {
+            notifier.send("solana-watchtower sanity failure");
+        }
         sleep(interval);
     }
 }

watchtower/src/notifier.rs

@@ -0,0 +1,46 @@
+use log::*;
+use reqwest::Client;
+use serde_json::json;
+use std::env;
+
+pub struct Notifier {
+    client: Client,
+    discord_webhook: Option<String>,
+    slack_webhook: Option<String>,
+}
+
+impl Notifier {
+    pub fn new() -> Self {
+        let discord_webhook = env::var("DISCORD_WEBHOOK")
+            .map_err(|_| {
+                warn!("Discord notifications disabled");
+            })
+            .ok();
+        let slack_webhook = env::var("SLACK_WEBHOOK")
+            .map_err(|_| {
+                warn!("Slack notifications disabled");
+            })
+            .ok();
+        Notifier {
+            client: Client::new(),
+            discord_webhook,
+            slack_webhook,
+        }
+    }
+
+    pub fn send(&self, msg: &str) {
+        if let Some(webhook) = &self.discord_webhook {
+            let data = json!({ "content": msg });
+            if let Err(err) = self.client.post(webhook).json(&data).send() {
+                warn!("Failed to send Discord message: {:?}", err);
+            }
+        }
+
+        if let Some(webhook) = &self.slack_webhook {
+            let data = json!({ "text": msg });
+            if let Err(err) = self.client.post(webhook).json(&data).send() {
+                warn!("Failed to send Slack message: {:?}", err);
+            }
+        }
+    }
+}