From 85b830d43cba053c63e4326a99772d76ebe22ec9 Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Sat, 13 Feb 2021 20:43:54 -0800 Subject: [PATCH] Fmt, Bulk Verify and nicer SVG output --- Cargo.toml | 2 +- rustfmt.toml | 6 ++- src/datasets.rs | 75 ++++++++++++++++++++------------- src/main.rs | 36 +++++++++------- src/oracle.rs | 74 +++++++++++++++++++++++---------- src/parties.rs | 6 +-- src/probability/binomial.rs | 2 +- src/server.rs | 82 +++++++++++++++++-------------------- 8 files changed, 168 insertions(+), 115 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 757c1c2..0269c2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -fuzzytags = {path="../../fuzzymetatag", features=["entangled"]} +fuzzytags = {path="../../fuzzymetatag", features=["entangled","bulk_verify"]} rand = "0.8.3" rand_distr = "0.4.0" hashbrown = "0.9.1" diff --git a/rustfmt.toml b/rustfmt.toml index b355306..ad894cd 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -19,7 +19,8 @@ fn_single_line = false where_single_line = false imports_indent = "Block" imports_layout = "Mixed" -merge_imports = false +imports_granularity = "Preserve" +group_imports = "Preserve" reorder_imports = true reorder_modules = true reorder_impl_items = false @@ -34,6 +35,7 @@ overflow_delimited_expr = false struct_field_align_threshold = 0 enum_discrim_align_threshold = 0 match_arm_blocks = true +match_arm_leading_pipes = "Never" force_multiline_blocks = false fn_args_layout = "Tall" brace_style = "SameLineWhere" @@ -52,7 +54,7 @@ use_field_init_shorthand = false force_explicit_abi = true condense_wildcard_suffixes = false color = "Auto" -required_version = "1.4.21" +required_version = "1.4.34" unstable_features = false disable_all_formatting = false skip_children = false diff --git a/src/datasets.rs b/src/datasets.rs index 3918f15..9c1ec3c 100644 --- a/src/datasets.rs +++ b/src/datasets.rs @@ -1,15 +1,19 @@ -use serde::Deserialize; +use crate::oracle::Oracle; use crate::server::SimulatedServer; -use fuzzytags::{RootSecret, TaggingKey}; +use fuzzytags::{RootSecret, Tag, TaggingKey}; use rand::Rng; +use serde::Deserialize; use tracing::event; use tracing::span; use tracing::Level; -use crate::oracle::Oracle; pub trait TemporalDataset { - fn register_with_server(&self, server: &mut SimulatedServer, rng: &mut R, min_p: usize, max_p: usize, oracle: &mut Oracle) where R: Rng; - fn playthough_traffic(&self, server: &mut SimulatedServer, oracle: &mut Oracle); + fn register_with_server(&self, server: &mut SimulatedServer, rng: &mut R, min_p: usize, max_p: usize, oracle: &mut Oracle) + where + R: Rng; + fn playthough_traffic(&self, server: &mut SimulatedServer, oracle: &mut Oracle, sample: usize, skip: usize, entangled_prob: f64, rng: &mut R) + where + R: Rng; } #[derive(Clone, Debug, Deserialize)] @@ -19,11 +23,10 @@ struct TemporalSocialNetworkRecord { timestamp: u64, } - pub struct CsvDataset { root_secrets: Vec>, tagging_keys: Vec>, - records: Vec + records: Vec, } impl CsvDataset { @@ -31,7 +34,7 @@ impl CsvDataset { let mut rdr = csv::Reader::from_path(filename).unwrap(); let mut num_recipients = 0; - let mut records : Vec = vec![]; + let mut records: Vec = vec![]; for result in rdr.deserialize() { // Notice that we need to provide a type hint for automatic // deserialization. @@ -45,7 +48,7 @@ impl CsvDataset { num_recipients = record.src_node; } records.push(record.clone()); - }, + } Err(err) => { panic!("invalid data record found in {}", filename) } @@ -55,18 +58,14 @@ impl CsvDataset { // generate a root secret for each member of the network let mut root_secrets = vec![]; let mut tagging_keys = vec![]; - for i in 0..num_recipients+1 { + for i in 0..num_recipients + 1 { let secret = RootSecret::<24>::generate(); let tagging_key = secret.tagging_key(); root_secrets.push(secret); tagging_keys.push(tagging_key) } - CsvDataset { - root_secrets, - tagging_keys, - records - } + CsvDataset { root_secrets, tagging_keys, records } } pub fn num_parties(&self) -> usize { @@ -79,8 +78,10 @@ impl CsvDataset { } impl TemporalDataset for CsvDataset { - fn register_with_server(&self, server: &mut SimulatedServer, rng: &mut R, min_p: usize, max_p: usize, oracle: &mut Oracle) where - R: Rng { + fn register_with_server(&self, server: &mut SimulatedServer, rng: &mut R, min_p: usize, max_p: usize, oracle: &mut Oracle) + where + R: Rng, + { for secret in self.root_secrets.iter() { let n = rng.gen_range(min_p..max_p); let span = span!(Level::INFO, "register", party = secret.tagging_key().id().as_str()); @@ -93,23 +94,41 @@ impl TemporalDataset for CsvDataset { } } - fn playthough_traffic(&self, server: &mut SimulatedServer,oracle: &mut Oracle) { + fn playthough_traffic(&self, server: &mut SimulatedServer, oracle: &mut Oracle, sample: usize, skip: usize, entangled_prob: f64, rng: &mut R) + where + R: Rng, + { /// TODO timestamps? - for (i,record) in self.records.iter().enumerate() { - if i %1000 == 0 { - let progress = i as f64 /(self.records.len() as f64); + for (i, record) in self.records.iter().skip(skip).take(sample).enumerate() { + if i % 100 == 0 { + let progress = (i - skip) as f64 / (sample as f64); let days = (((record.timestamp as f64 / 60.0) / 60.0) / 24.0); - event!(Level::INFO, "progress..{:.2} ({} days)", progress*100.0, days); + event!(Level::INFO, "progress..{:.2} ({} days)", progress * 100.0, days); } // We pretend that the server will always have access to the sender, even though // in practical deployments we could mitigate this somewhat using Tor / mixnet. let tagging_key_src = &self.tagging_keys[record.src_node]; let tagging_key_dst = &self.tagging_keys[record.dst_node]; - event!(Level::TRACE, "regular send {party}", party = tagging_key_dst.id().as_str()); - let tag = tagging_key_dst.generate_tag(); - event!(Level::TRACE, "message sent server {tag}", tag = tag.to_string()); - server.add_message(tag, tagging_key_src); - oracle.add_event(tagging_key_src.id(), tagging_key_dst.id(), None, 0.0); + + let entangle = rng.gen_bool(entangled_prob); + if entangle { + let entangled_party = rng.gen_range(0..self.num_parties()); + let tagging_key_entangled = &self.tagging_keys[entangled_party]; + + event!(Level::TRACE, "entangled send {party}", party = tagging_key_dst.id().as_str()); + + let tag = TaggingKey::<24>::generate_entangled_tag(vec![tagging_key_dst.clone(), tagging_key_entangled.clone()], 8); + event!(Level::TRACE, "message sent server {tag}", tag = tag.to_string()); + server.add_message(tag, tagging_key_src); + + oracle.add_event(tagging_key_src.id(), tagging_key_dst.id(), Some(tagging_key_entangled.id()), 0.0); + } else { + event!(Level::TRACE, "regular send {party}", party = tagging_key_dst.id().as_str()); + let tag = tagging_key_dst.generate_tag(); + event!(Level::TRACE, "message sent server {tag}", tag = tag.to_string()); + server.add_message(tag, tagging_key_src); + oracle.add_event(tagging_key_src.id(), tagging_key_dst.id(), None, 0.0); + } } } } @@ -123,4 +142,4 @@ mod tests { let dataset = CsvDataset::load_dataset("datasets/email-Eu-core-temporal.txt"); assert_eq!(332334, dataset.num_records()); } -} \ No newline at end of file +} diff --git a/src/main.rs b/src/main.rs index 98262ae..efa5510 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,44 +3,41 @@ use crate::server::SimulatedServer; use rand_distr::Pareto; use std::io::Write; use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; +mod datasets; mod oracle; mod parties; -mod server; -mod datasets; mod probability; +mod server; use clap::Clap; use tracing::event; +use crate::datasets::{CsvDataset, TemporalDataset}; use crate::oracle::Oracle; use tracing::Level; use tracing_subscriber; use tracing_subscriber::FmtSubscriber; -use crate::datasets::{CsvDataset, TemporalDataset}; #[derive(Clap)] #[clap(version = "1.0", author = "Sarah Jamie Lewis ")] struct Opts { - /// the number of parties to simulate #[clap(short, long, default_value = "10")] num_parties: usize, - /// dataset #[clap(short, long, default_value = "")] dataset: String, - /// samples per round #[clap(short, long, default_value = "10")] samples_per_round: usize, /// minimum false positive rate - #[clap(short, long, default_value = "1")] + #[clap(long, default_value = "1")] min_p: usize, /// maximum false positive rate - #[clap(short, long, default_value = "8")] + #[clap(long, default_value = "8")] max_p: usize, #[clap(short, long)] @@ -48,6 +45,12 @@ struct Opts { #[clap(short, long, default_value = "0")] prob_entangled: f64, + + #[clap(long, default_value = "0")] + sample: usize, + + #[clap(long, default_value = "0")] + skip: usize, } fn main() { @@ -64,11 +67,12 @@ fn main() { let mut rng = rand::thread_rng(); let mut server = SimulatedServer::new(); - if opts.dataset == "" { + if opts.dataset == "none" { let simulated_parties = SimulatedParties::new_simulation(opts.num_parties); { event!(Level::INFO, "Generating {} Parties and registering them with the server", opts.num_parties); simulated_parties.register_with_server(&mut server, &mut rng, opts.min_p, opts.max_p, &mut oracle); + server.finalize(); } let pareto = Pareto::new(1.0, 1.0).unwrap(); @@ -80,10 +84,14 @@ fn main() { } else { let dataset = CsvDataset::load_dataset(opts.dataset.as_str()); event!(Level::INFO, "Registering parties from {} which containts {}", opts.dataset, dataset.num_parties()); - dataset.register_with_server(&mut server, &mut rng, opts.min_p, opts.max_p, &mut oracle); - - event!(Level::INFO, "Playing back {} events from {}", dataset.num_records(), opts.dataset); - dataset.playthough_traffic(&mut server, &mut oracle); + dataset.register_with_server(&mut server, &mut rng, opts.min_p, opts.max_p, &mut oracle); + server.finalize(); + event!(Level::INFO, "Playing back {} events from {}", dataset.num_records(), opts.dataset); + let mut sample = dataset.num_records(); + if opts.sample != 0 { + sample = opts.sample; + } + dataset.playthough_traffic(&mut server, &mut oracle, opts.sample, opts.skip, opts.prob_entangled, &mut rng); } { @@ -125,6 +133,4 @@ fn main() { //server_oracle.compile_to_dot("server_event_inverse.dot", true, true,max); } }); - - } diff --git a/src/oracle.rs b/src/oracle.rs index 5946d1f..f1d976b 100644 --- a/src/oracle.rs +++ b/src/oracle.rs @@ -1,24 +1,25 @@ // todo would be neat to just make this a subscriber of the tracing log, but there doesn't // seem to be a nice way for that API to do what we want.. so until then... -use hashbrown::HashMap; +use hashbrown::{HashMap, HashSet}; use rand::{thread_rng, Rng}; +use rand_distr::num_traits::FloatConst; use std::fs::File; use std::io::Write; -use rand_distr::num_traits::FloatConst; #[derive(Clone)] pub struct Event { sender: String, intended_receiver: String, entangled_receiver: Option, - confidence: f64 + confidence: f64, } #[derive(Clone)] pub struct Oracle { parties: Vec, actual_events: Vec, + suspect: HashMap<(String, String), f64>, } impl Oracle { @@ -26,6 +27,7 @@ impl Oracle { Oracle { parties: vec![], actual_events: vec![], + suspect: HashMap::new(), } } @@ -42,6 +44,10 @@ impl Oracle { }); } + pub fn add_suspect(&mut self, sender: &String, intended_receiver: &String, weight: f64) { + self.suspect.insert((sender.clone(), intended_receiver.clone()), weight); + } + pub fn compile_to_dot(&self, filename: &str, strict: bool, inverse: bool, max: f64) -> f64 { let mut output = File::create(filename).unwrap(); write!(output, "strict digraph {{\n"); @@ -55,16 +61,18 @@ impl Oracle { "## ); - for (i, party) in self.parties.iter().enumerate() { - - let x = (((f64::PI()*2.0)/self.parties.len() as f64) * (i as f64)).cos()*10.0; - let y = (((f64::PI()*2.0)/self.parties.len() as f64) * (i as f64)).sin()*10.0; + let x = (((f64::PI() * 2.0) / self.parties.len() as f64) * (i as f64)).cos() * 10.0; + let y = (((f64::PI() * 2.0) / self.parties.len() as f64) * (i as f64)).sin() * 10.0; let r = hex::decode(party).unwrap()[0]; let g = hex::decode(party).unwrap()[1]; let b = hex::decode(party).unwrap()[2]; - writeln!(output, "\"{}\" [shape=point, penwidth=0, fixedsize=true, width=0.005,height=0.005,peripheries=0,style=\"filled,setlinewidth(0)\", color=\"#{:02x}{:02x}{:02x}\"]", party, r, g, b); + writeln!( + output, + "\"{}\" [tooltip=\"{} ({})\", shape=point, penwidth=0, fixedsize=true, width=0.005,height=0.005,peripheries=0,style=\"filled,setlinewidth(0)\", color=\"#{:02x}{:02x}{:02x}\"]", + party, party, i, r, g, b + ); //writeln!(output, "\"{}\" [shape=point, color=\"#{:x}{:x}{:x}\",pos=\"{},{}!\"]", party, r, g, b,x,y); } @@ -74,12 +82,12 @@ impl Oracle { for event in self.actual_events.iter() { let key = (event.sender.clone(), event.intended_receiver.clone()); if real_connection_map.contains_key(&key) { - *real_connection_map.get_mut(&key).unwrap() += (1.0 * (1.0-event.confidence)); + *real_connection_map.get_mut(&key).unwrap() += (1.0 * (1.0 - event.confidence)); if real_connection_map[&key] > max_conn { max_conn = real_connection_map[&key]; } } else { - real_connection_map.insert(key, 1.0 * (1.0-event.confidence)); + real_connection_map.insert(key, 1.0 * (1.0 - event.confidence)); } match &event.entangled_receiver { Some(entangled_receiver) => { @@ -101,22 +109,46 @@ impl Oracle { let normalized = (*size as f64 / max_conn as f64); let mut transparency = (normalized * 64.0) as u8 + 172; let mut penwidth = (normalized * 0.01) as f64; - - writeln!( - output, - "\"{}\" -> \"{}\" [arrowhead=none, penwidth={}, color=\"#ffffff{:02x}\", weight={}]", - sender, receiver, f64::max(0.005, penwidth), transparency, penwidth - ); + match self.suspect.get(&(sender.clone(), receiver.clone())) { + Some(weight) => { + let normalized = ((*weight as f64 * 1000.0).max(999.0) - 999.0) / 1.0; + let mut penwidth = (normalized * 0.01) as f64; + let mut transparency = (normalized * 64.0) as u8 + 172; + writeln!( + output, + "\"{}\" -> \"{}\" [arrowhead=none, penwidth={}, color=\"#{:02x}0000{:02x}\", weight={}]", + sender, + receiver, + f64::max(0.005, penwidth), + transparency, + transparency, + penwidth + ); + } + _ => { + if self.suspect.is_empty() { + writeln!( + output, + "\"{}\" -> \"{}\" [arrowhead=none, penwidth={}, color=\"#ffffff{:02x}\", weight={}]", + sender, + receiver, + f64::max(0.005, penwidth), + transparency, + penwidth + ); + } + } + } } for ((sender, receiver), size) in entangled_connection_map.iter() { let normalized = (*size as f64 / max_conn as f64); let transparency = (normalized * 172.0) as u8 + 64; let penwidth = ((normalized * 32.0) as f64).log2(); - writeln!( - output, - "\"{}\" -> \"{}\" [arrowhead=none, style=dashed, penwidth={}, color=\"#ffffff{:02x}\"]", - sender, receiver, penwidth, transparency - ); + // writeln!( + // output, + // "\"{}\" -> \"{}\" [arrowhead=none, style=dashed, penwidth={}, color=\"#ffffff{:02x}\"]", + // sender, receiver, penwidth, transparency + //); } write!(output, "}}"); max_conn diff --git a/src/parties.rs b/src/parties.rs index 579e2e0..50000ef 100644 --- a/src/parties.rs +++ b/src/parties.rs @@ -1,12 +1,12 @@ use crate::oracle::Oracle; use crate::server::SimulatedServer; +use fuzzytags::{RootSecret, TaggingKey}; use rand::distributions::Distribution; use rand::Rng; use rand_distr::num_traits::ToPrimitive; use tracing::event; use tracing::span; use tracing::Level; -use fuzzytags::{RootSecret, TaggingKey}; pub struct SimulatedParties { parties: Vec>, @@ -71,7 +71,7 @@ impl SimulatedParties { server.add_message(tag, &sender_public_key); } - oracle.add_event(sender_public_key.id().clone(), receiver_public_key.id(), Some(receiver_public_key_2.id()),1.0); + oracle.add_event(sender_public_key.id().clone(), receiver_public_key.id(), Some(receiver_public_key_2.id()), 1.0); } else { event!(Level::INFO, "regular send {party}", party = receiver_public_key.id().as_str()); for _i in 0..v { @@ -79,7 +79,7 @@ impl SimulatedParties { event!(Level::INFO, "message sent server {tag}", tag = tag.to_string()); server.add_message(tag, &sender_public_key); } - oracle.add_event(sender_public_key.id().clone(), receiver_public_key.id(), None,1.0); + oracle.add_event(sender_public_key.id().clone(), receiver_public_key.id(), None, 1.0); } } } diff --git a/src/probability/binomial.rs b/src/probability/binomial.rs index c6708a5..6b1df38 100644 --- a/src/probability/binomial.rs +++ b/src/probability/binomial.rs @@ -1,5 +1,5 @@ /// calculate the binomial coefficient. -pub fn nchoosek(n: u64, k: u64) -> f64{ +pub fn nchoosek(n: u64, k: u64) -> f64 { if k > n { return 0.0; } diff --git a/src/server.rs b/src/server.rs index 37cd283..0458d70 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,19 +1,20 @@ use crate::oracle::Oracle; -use fuzzytags::{DetectionKey, TaggingKey, Tag}; +use crate::probability::binomial::{at_least_with_replacement, nchoosek}; +use fuzzytags::{DetectionKey, Tag, TaggingKey}; use hashbrown::{HashMap, HashSet}; -use tracing::event; -use tracing::span; -use tracing::Level; +use itertools::Itertools; use rayon::iter::IntoParallelRefIterator; use rayon::iter::ParallelIterator; use std::sync::mpsc::channel; -use std::time::Duration; use std::sync::Arc; -use itertools::Itertools; -use crate::probability::binomial::{nchoosek, at_least_with_replacement}; +use std::time::Duration; +use tracing::event; +use tracing::span; +use tracing::Level; pub struct SimulatedServer { - keybase: Vec<(DetectionKey<24>, TaggingKey<24>)>, + keybase: Vec<(DetectionKey<24>, TaggingKey<24>, String)>, + detection_key_cache: Vec>, messages: Vec<(Tag<24>, TaggingKey<24>)>, sender_tags: HashMap, sender_count: HashMap, @@ -22,8 +23,7 @@ pub struct SimulatedServer { oracle: Oracle, } -struct Event(String,String,String,f64); - +struct Event(String, String, String, f64); pub struct RoundStatistics { pub num_registered_parties: usize, @@ -46,6 +46,7 @@ impl SimulatedServer { SimulatedServer { keybase: vec![], messages: vec![], + detection_key_cache: vec![], sender_tags: HashMap::new(), sender_count: HashMap::new(), tags_to_keys_cache: HashMap::new(), @@ -55,15 +56,21 @@ impl SimulatedServer { } pub fn register_key(&mut self, detection_key: &DetectionKey<24>, tagging_key: &TaggingKey<24>) { - self.keybase.push((detection_key.clone(), tagging_key.clone())); + self.keybase.push((detection_key.clone(), tagging_key.clone(), tagging_key.id())); self.keys_to_tags_cache.insert(tagging_key.id(), HashSet::new()); self.oracle.register_party(tagging_key.id()); } + pub fn finalize(&mut self) { + self.detection_key_cache = self.keybase.iter().map(|(d, t, _)| d.clone()).collect(); + } + pub fn add_message(&mut self, tag: Tag<24>, sender_tagging_key: &TaggingKey<24>) { + let tag_id = tag.to_string(); + let sender_id = sender_tagging_key.id(); self.messages.push((tag.clone(), sender_tagging_key.clone())); - self.tags_to_keys_cache.insert(tag.to_string(), HashSet::new()); - self.sender_tags.insert(tag.to_string(), sender_tagging_key.id()); + self.tags_to_keys_cache.insert(tag_id.clone(), HashSet::new()); + self.sender_tags.insert(tag_id.clone(), sender_tagging_key.id()); let count = match self.sender_count.get(sender_tagging_key.id().as_str()) { Some(count) => *count + 1.0, @@ -71,37 +78,24 @@ impl SimulatedServer { }; self.sender_count.insert(sender_tagging_key.id(), count); - let (tx, rx) = channel(); - self.keybase.par_iter().for_each_with(tx.clone(), |tx,(detection_key, receiver_tagging_key)| { - if detection_key.test_tag(&tag) { - let tag_str = tag.to_string(); - tx.send(Event(tag_str.clone(), sender_tagging_key.id(), receiver_tagging_key.id(), detection_key.false_positive_probability())); - } - }); - std::mem::drop(tx); - loop { - let event = rx.recv(); - match event { - Ok(event) => { - event!(Level::TRACE, "Matched detection key for {key} to tag {tag} ", key = event.2, tag = event.0); - self.tags_to_keys_cache.get_mut(event.0.as_str()).unwrap().insert(event.2.clone()); - self.keys_to_tags_cache.get_mut(event.2.as_str()).unwrap().insert(event.0.clone()); - self.oracle.add_event(event.1.to_string(), event.2.to_string(), None, event.3); - } - _ => {break;} - } + let results = DetectionKey::test_tag_bulk(&self.detection_key_cache, &tag); + for index in results { + let detection_key = &self.keybase[index].0; + let receiver_tagging_id = &self.keybase[index].2; + event!(Level::TRACE, "Matched detection key for {key} to tag {tag} ", key = receiver_tagging_id.clone(), tag = tag_id.clone()); + self.tags_to_keys_cache.get_mut(tag_id.as_str()).unwrap().insert(receiver_tagging_id.clone()); + self.keys_to_tags_cache.get_mut(receiver_tagging_id.as_str()).unwrap().insert(tag_id.clone()); + self.oracle.add_event(sender_id.clone(), receiver_tagging_id.clone(), None, detection_key.false_positive_probability()); } } - - - pub fn statistics(&self) -> (Oracle, RoundStatistics, HashMap) { + pub fn statistics(&mut self) -> (Oracle, RoundStatistics, HashMap) { let mut party_stats = HashMap::new(); let round_stats = RoundStatistics { num_messages: self.messages.len(), num_registered_parties: self.keybase.len(), }; - for (recipient_index,(party, pub_key)) in self.keybase.iter().enumerate() { + for (recipient_index, (party, pub_key, id)) in self.keybase.iter().enumerate() { let matched = self.keys_to_tags_cache[pub_key.id().as_str()].clone(); let observed_messages = matched.len(); let ideal_rate = party.false_positive_probability(); @@ -115,7 +109,6 @@ impl SimulatedServer { let mut num_times_matched_with = HashMap::new(); for tag in matched.iter() { - let sender = self.sender_tags[tag].clone(); let num = match num_times_matched_with.get(sender.as_str()) { Some(num) => *num + 1.0, @@ -129,15 +122,16 @@ impl SimulatedServer { } for (sender, count) in num_times_matched_with.iter() { - let expected_matched_count = (ideal_rate * self.sender_count[sender.as_str()]); - let actual_matched_count = (*count); - let prob = at_least_with_replacement( actual_matched_count as u64, self.sender_count[sender.as_str()] as u64,ideal_rate).to_f64(); + let expected_matched_count = (ideal_rate * self.sender_count[sender.as_str()]); + let actual_matched_count = (*count); + let prob = at_least_with_replacement(actual_matched_count as u64, self.sender_count[sender.as_str()] as u64, ideal_rate).to_f64(); let diff = f64::abs(actual_matched_count - expected_matched_count); // these numbers are arbitrary, but the point is probability only works if your sample is big enough... - if diff > 4.0 && actual_matched_count > expected_matched_count && prob < 0.01 { - let sender_index = self.keybase.iter().find_position(|(d,t)|t.id() == *sender).unwrap().0; - event!(Level::INFO, "Found Anomalous Relationship Between {sender}({sender_index}) and {recipient}({recipient_index}) {falsepositiverate}, {detected} >> {expected} | Probability of Event: {prob:.9}", sender=sender, sender_index=sender_index, recipient=pub_key.id(), recipient_index=recipient_index, falsepositiverate=ideal_rate, detected=actual_matched_count, expected=expected_matched_count, prob=prob); - } + if diff > 4.0 && actual_matched_count > expected_matched_count && prob < 0.0001 { + let sender_index = self.keybase.iter().find_position(|(d, t, _)| t.id() == *sender).unwrap().0; + event!(Level::INFO, "Found Anomalous Relationship Between {sender}({sender_index}) and {recipient}({recipient_index}) {falsepositiverate}, {detected} >> {expected} | Probability of Event: {prob:.9}", sender=sender, sender_index=sender_index, recipient=pub_key.id(), recipient_index=recipient_index, falsepositiverate=ideal_rate, detected=actual_matched_count, expected=expected_matched_count, prob=prob); + self.oracle.add_suspect(&sender, &id, 1.0 - prob); + } } let p_stats = PartyStatistics {