Compare iftop logs using log-analyzer (#6684)

* Compare iftop logs using log-analyzer

* fixes

* fix clippy errors

Cargo.lock

@@ -3724,6 +3724,7 @@ dependencies = [
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana-logger 0.21.0",
 ]

log-analyzer/Cargo.toml

@@ -11,10 +11,11 @@ publish = false
 
 [dependencies]
 byte-unit = "3.0.3"
-clap = { version = "2.33.0" }
+clap = "2.33.0"
 log = "0.4.8"
 semver = "0.9.0"
 serde = "1.0.102"
+serde_derive = "1.0.102"
 serde_json = "1.0.41"
 solana-logger = { path = "../logger", version = "0.21.0" }
 

log-analyzer/src/main.rs

@@ -1,17 +1,17 @@
 extern crate byte_unit;
 
 use byte_unit::Byte;
-use clap::{crate_description, crate_name, crate_version, value_t_or_exit, App, Arg, SubCommand};
+use clap::{
-use serde::export::fmt::Error;
+    crate_description, crate_name, crate_version, value_t_or_exit, App, Arg, ArgMatches, SubCommand,
-use serde::export::Formatter;
+};
+
 use serde::{Deserialize, Serialize};
-use serde_json::{self};
 use std::collections::HashMap;
-use std::fmt::Debug;
 use std::fs;
+use std::ops::Sub;
 use std::path::PathBuf;
 
-#[derive(Serialize, Deserialize)]
+#[derive(Deserialize, Serialize, Debug)]
 struct LogLine {
     a: String,
     b: String,
@@ -19,57 +19,70 @@ struct LogLine {
     b_to_a: String,
 }
 
-impl Debug for LogLine {
+impl Default for LogLine {
-    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> {
+    fn default() -> Self {
-        let a_to_b = Byte::from_str(&self.a_to_b).expect("Failed to read a_to_b bytes");
+        Self {
-        let b_to_a = Byte::from_str(&self.b_to_a).expect("Failed to read b_to_a bytes");
+            a: String::default(),
-        write!(
+            b: String::default(),
-            f,
+            a_to_b: "0B".to_string(),
-            "{{ \"{}\", \"{}\", {}, {} }}",
+            b_to_a: "0B".to_string(),
-            self.a,
+        }
-            self.b,
+    }
-            a_to_b.get_bytes(),
+}
-            b_to_a.get_bytes()
+
+impl LogLine {
+    fn output(a: &str, b: &str, v1: u128, v2: u128) -> String {
+        format!(
+            "Lost {} bytes ({} - {}, {}%)  from {} to {}",
+            v1 - v2,
+            v1,
+            v2,
+            ((v1 - v2) * 100 / v1),
+            a,
+            b
         )
     }
 }
 
-fn main() {
+impl Sub for &LogLine {
-    solana_logger::setup();
+    type Output = String;
 
-    let matches = App::new(crate_name!())
+    fn sub(self, rhs: Self) -> Self::Output {
-        .about(crate_description!())
+        let a_to_b = Byte::from_str(&self.a_to_b)
-        .version(crate_version!())
+            .expect("Failed to read a_to_b bytes")
-        .arg(
+            .get_bytes();
-            Arg::with_name("iftop")
+        let b_to_a = Byte::from_str(&self.b_to_a)
-                .short("i")
+            .expect("Failed to read b_to_a bytes")
-                .long("iftop")
+            .get_bytes();
-                .value_name("iftop log file")
+        let rhs_a_to_b = Byte::from_str(&rhs.a_to_b)
-                .takes_value(true)
+            .expect("Failed to read a_to_b bytes")
-                .help("Location of the log file generated by iftop"),
+            .get_bytes();
-        )
+        let rhs_b_to_a = Byte::from_str(&rhs.b_to_a)
-        .subcommand(
+            .expect("Failed to read b_to_a bytes")
-            SubCommand::with_name("map-IP")
+            .get_bytes();
-                .about("Public IP Address")
+        let mut out1 = if a_to_b > rhs_b_to_a {
-                .arg(
+            LogLine::output(&self.a, &self.b, a_to_b, rhs_b_to_a)
-                    Arg::with_name("priv")
+        } else if a_to_b < rhs_b_to_a {
-                        .long("priv")
+            LogLine::output(&self.b, &self.a, rhs_b_to_a, a_to_b)
-                        .value_name("IP Address")
+        } else {
-                        .takes_value(true)
+            String::default()
-                        .required(true)
+        };
-                        .help("The private IP address that should be mapped"),
+        let out2 = if rhs_a_to_b > b_to_a {
-                )
+            LogLine::output(&self.a, &self.b, rhs_a_to_b, b_to_a)
-                .arg(
+        } else if rhs_a_to_b < b_to_a {
-                    Arg::with_name("pub")
+            LogLine::output(&self.b, &self.a, b_to_a, rhs_a_to_b)
-                        .long("pub")
+        } else {
-                        .value_name("IP Address")
+            String::default()
-                        .takes_value(true)
+        };
-                        .required(true)
+        if !out1.is_empty() && !out2.is_empty() {
-                        .help("The public IP address"),
+            out1.push('\n');
-                ),
+        }
-        )
+        out1.push_str(&out2);
-        .get_matches();
+        out1
+    }
+}
 
+fn process_iftop_logs(matches: &ArgMatches) {
     let map_address;
     let private_address;
     let public_address;
@@ -94,7 +107,7 @@ fn main() {
         }
     };
 
-    let log_path = PathBuf::from(value_t_or_exit!(matches, "iftop", String));
+    let log_path = PathBuf::from(value_t_or_exit!(matches, "file", String));
     let mut log = fs::read_to_string(&log_path).expect("Unable to read log file");
     log.insert(0, '[');
     let terminate_at = log.rfind('}').expect("Didn't find a terminating '}'") + 1;
@@ -110,10 +123,7 @@ fn main() {
             unique_latest_logs.entry(key).or_insert(l);
         }
     });
-
+    let output: Vec<LogLine> = unique_latest_logs
-    println!(
-        "{:#?}",
-        unique_latest_logs
         .into_iter()
         .map(|(_, l)| {
             if map_address {
@@ -127,6 +137,118 @@ fn main() {
                 l
             }
         })
-            .collect::<Vec<_>>()
+        .collect();
-    );
+
+    println!("{}", serde_json::to_string(&output).unwrap());
+}
+
+fn analyze_logs(matches: &ArgMatches) {
+    let dir_path = PathBuf::from(value_t_or_exit!(matches, "folder", String));
+    if !dir_path.is_dir() {
+        panic!("Need a folder that contains all log files");
+    }
+    let list_all_diffs = matches.is_present("all");
+    let files = fs::read_dir(dir_path).expect("Failed to read log folder");
+    let logs: Vec<_> = files
+        .flat_map(|f| {
+            if let Ok(f) = f {
+                let log_str = fs::read_to_string(&f.path()).expect("Unable to read log file");
+                let log: Vec<LogLine> =
+                    serde_json::from_str(log_str.as_str()).expect("Failed to deserialize log");
+                log
+            } else {
+                vec![]
+            }
+        })
+        .collect();
+    let mut logs_hash = HashMap::new();
+    logs.iter().for_each(|l| {
+        let key = (l.a.clone(), l.b.clone());
+        logs_hash.entry(key).or_insert(l);
+    });
+
+    logs.iter().for_each(|l| {
+        let diff = logs_hash
+            .remove(&(l.a.clone(), l.b.clone()))
+            .map(|v1| {
+                logs_hash.remove(&(l.b.clone(), l.a.clone())).map_or(
+                    if list_all_diffs {
+                        v1 - &LogLine::default()
+                    } else {
+                        String::default()
+                    },
+                    |v2| v1 - v2,
+                )
+            })
+            .unwrap_or_default();
+        if !diff.is_empty() {
+            println!("{}", diff);
+        }
+    });
+}
+
+fn main() {
+    solana_logger::setup();
+
+    let matches = App::new(crate_name!())
+        .about(crate_description!())
+        .version(crate_version!())
+        .subcommand(
+            SubCommand::with_name("iftop")
+                .about("Process iftop log file")
+                .arg(
+                    Arg::with_name("file")
+                        .short("f")
+                        .long("file")
+                        .value_name("iftop log file")
+                        .takes_value(true)
+                        .help("Location of the log file generated by iftop"),
+                )
+                .subcommand(
+                    SubCommand::with_name("map-IP")
+                        .about("Public IP Address")
+                        .arg(
+                            Arg::with_name("priv")
+                                .long("priv")
+                                .value_name("IP Address")
+                                .takes_value(true)
+                                .required(true)
+                                .help("The private IP address that should be mapped"),
+                        )
+                        .arg(
+                            Arg::with_name("pub")
+                                .long("pub")
+                                .value_name("IP Address")
+                                .takes_value(true)
+                                .required(true)
+                                .help("The public IP address"),
+                        ),
+                ),
+        )
+        .subcommand(
+            SubCommand::with_name("analyze")
+                .about("Compare processed network log files")
+                .arg(
+                    Arg::with_name("folder")
+                        .short("f")
+                        .long("folder")
+                        .value_name("DIR")
+                        .takes_value(true)
+                        .help("Location of processed log files"),
+                )
+                .arg(
+                    Arg::with_name("all")
+                        .short("a")
+                        .long("all")
+                        .takes_value(false)
+                        .help("List all differences"),
+                ),
+        )
+        .get_matches();
+
+    match matches.subcommand() {
+        ("iftop", Some(args_matches)) => process_iftop_logs(args_matches),
+        ("analyze", Some(args_matches)) => analyze_logs(args_matches),
+        _ => {}
+    };
 }

scripts/iftop-postprocess.sh

@@ -24,9 +24,9 @@ awk '{ if ($3 ~ "=>") { print $2, $7 } else if ($2 ~ "<=") { print $1, $6 }} ' <
   | awk '{ print "{ \"a\": \""$1"\", " "\"b\": \""$3"\", \"a_to_b\": \""$2"\", \"b_to_a\": \""$4"\"}," }' > "$2"
 
 if [ "$#" -ne 3 ]; then
-  solana-log-analyzer -i "$2"
+  solana-log-analyzer iftop -f "$2"
 else
-  solana-log-analyzer -i "$2" map-IP --priv "$(hostname -i)" --pub "$3"
+  solana-log-analyzer iftop -f "$2" map-IP --priv "$(hostname -i)" --pub "$3"
 fi
 
 exit 1