2021-02-03 13:05:59 +00:00
use crate ::oracle ::Oracle ;
2021-02-10 07:23:50 +00:00
use fuzzytags ::{ DetectionKey , TaggingKey , Tag } ;
2021-02-13 07:36:04 +00:00
use hashbrown ::{ HashMap , HashSet } ;
2021-02-02 02:33:43 +00:00
use tracing ::event ;
use tracing ::span ;
use tracing ::Level ;
2021-02-13 07:36:04 +00:00
use rayon ::iter ::IntoParallelRefIterator ;
use rayon ::iter ::ParallelIterator ;
use std ::sync ::mpsc ::channel ;
use std ::time ::Duration ;
use std ::sync ::Arc ;
use itertools ::Itertools ;
use crate ::probability ::binomial ::{ nchoosek , at_least_with_replacement } ;
2021-02-01 07:50:59 +00:00
pub struct SimulatedServer {
2021-02-10 07:23:50 +00:00
keybase : Vec < ( DetectionKey < 24 > , TaggingKey < 24 > ) > ,
messages : Vec < ( Tag < 24 > , TaggingKey < 24 > ) > ,
2021-02-13 07:36:04 +00:00
sender_tags : HashMap < String , String > ,
sender_count : HashMap < String , f64 > ,
tags_to_keys_cache : HashMap < String , HashSet < String > > ,
keys_to_tags_cache : HashMap < String , HashSet < String > > ,
2021-02-03 13:05:59 +00:00
oracle : Oracle ,
2021-02-01 07:50:59 +00:00
}
2021-02-13 07:36:04 +00:00
struct Event ( String , String , String , f64 ) ;
2021-02-01 23:23:01 +00:00
pub struct RoundStatistics {
pub num_registered_parties : usize ,
pub num_messages : usize ,
}
#[ derive(Debug) ]
2021-02-01 07:50:59 +00:00
pub struct PartyStatistics {
pub ideal_rate : f64 ,
pub expected_messages : f64 ,
pub observed_messages : usize ,
pub observed_rate : f64 ,
pub observed_skew_messages : f64 ,
pub observed_skew : f64 ,
pub trivial_breaks : usize ,
}
impl SimulatedServer {
pub fn new ( ) -> SimulatedServer {
SimulatedServer {
keybase : vec ! [ ] ,
messages : vec ! [ ] ,
2021-02-13 07:36:04 +00:00
sender_tags : HashMap ::new ( ) ,
sender_count : HashMap ::new ( ) ,
2021-02-01 07:50:59 +00:00
tags_to_keys_cache : HashMap ::new ( ) ,
keys_to_tags_cache : HashMap ::new ( ) ,
2021-02-03 13:05:59 +00:00
oracle : Oracle ::new ( ) ,
2021-02-01 07:50:59 +00:00
}
}
2021-02-10 07:23:50 +00:00
pub fn register_key ( & mut self , detection_key : & DetectionKey < 24 > , tagging_key : & TaggingKey < 24 > ) {
self . keybase . push ( ( detection_key . clone ( ) , tagging_key . clone ( ) ) ) ;
2021-02-13 07:36:04 +00:00
self . keys_to_tags_cache . insert ( tagging_key . id ( ) , HashSet ::new ( ) ) ;
2021-02-10 07:23:50 +00:00
self . oracle . register_party ( tagging_key . id ( ) ) ;
2021-02-01 07:50:59 +00:00
}
2021-02-10 07:23:50 +00:00
pub fn add_message ( & mut self , tag : Tag < 24 > , sender_tagging_key : & TaggingKey < 24 > ) {
self . messages . push ( ( tag . clone ( ) , sender_tagging_key . clone ( ) ) ) ;
2021-02-13 07:36:04 +00:00
self . tags_to_keys_cache . insert ( tag . to_string ( ) , HashSet ::new ( ) ) ;
self . sender_tags . insert ( tag . to_string ( ) , sender_tagging_key . id ( ) ) ;
2021-02-01 07:50:59 +00:00
2021-02-13 07:36:04 +00:00
let count = match self . sender_count . get ( sender_tagging_key . id ( ) . as_str ( ) ) {
Some ( count ) = > * count + 1.0 ,
_ = > 1.0 ,
} ;
self . sender_count . insert ( sender_tagging_key . id ( ) , count ) ;
let ( tx , rx ) = channel ( ) ;
self . keybase . par_iter ( ) . for_each_with ( tx . clone ( ) , | tx , ( detection_key , receiver_tagging_key ) | {
if detection_key . test_tag ( & tag ) {
let tag_str = tag . to_string ( ) ;
tx . send ( Event ( tag_str . clone ( ) , sender_tagging_key . id ( ) , receiver_tagging_key . id ( ) , detection_key . false_positive_probability ( ) ) ) ;
}
} ) ;
std ::mem ::drop ( tx ) ;
loop {
let event = rx . recv ( ) ;
match event {
Ok ( event ) = > {
event! ( Level ::TRACE , " Matched detection key for {key} to tag {tag} " , key = event . 2 , tag = event . 0 ) ;
self . tags_to_keys_cache . get_mut ( event . 0. as_str ( ) ) . unwrap ( ) . insert ( event . 2. clone ( ) ) ;
self . keys_to_tags_cache . get_mut ( event . 2. as_str ( ) ) . unwrap ( ) . insert ( event . 0. clone ( ) ) ;
self . oracle . add_event ( event . 1. to_string ( ) , event . 2. to_string ( ) , None , event . 3 ) ;
2021-02-01 07:50:59 +00:00
}
2021-02-13 07:36:04 +00:00
_ = > { break ; }
2021-02-01 07:50:59 +00:00
}
}
}
2021-02-13 07:36:04 +00:00
2021-02-03 13:05:59 +00:00
pub fn statistics ( & self ) -> ( Oracle , RoundStatistics , HashMap < String , PartyStatistics > ) {
2021-02-01 23:23:01 +00:00
let mut party_stats = HashMap ::new ( ) ;
let round_stats = RoundStatistics {
num_messages : self . messages . len ( ) ,
num_registered_parties : self . keybase . len ( ) ,
} ;
2021-02-13 07:36:04 +00:00
for ( recipient_index , ( party , pub_key ) ) in self . keybase . iter ( ) . enumerate ( ) {
2021-02-01 23:23:01 +00:00
let matched = self . keys_to_tags_cache [ pub_key . id ( ) . as_str ( ) ] . clone ( ) ;
2021-02-01 07:50:59 +00:00
let observed_messages = matched . len ( ) ;
let ideal_rate = party . false_positive_probability ( ) ;
2021-02-01 23:23:01 +00:00
let expected_messages = ideal_rate * ( round_stats . num_messages as f64 ) ;
2021-02-01 07:50:59 +00:00
let observed_rate = ( observed_messages as f64 ) / ( self . messages . len ( ) as f64 ) ;
2021-02-01 23:23:01 +00:00
let observed_skew_messages = ( observed_messages as f64 ) - expected_messages ;
2021-02-01 20:20:30 +00:00
let observed_skew = ( observed_messages as f64 ) / expected_messages ;
2021-02-01 07:50:59 +00:00
let mut trivial_breaks = 0 ;
2021-02-13 07:36:04 +00:00
let mut num_times_matched_with = HashMap ::new ( ) ;
2021-02-01 07:50:59 +00:00
for tag in matched . iter ( ) {
2021-02-13 07:36:04 +00:00
let sender = self . sender_tags [ tag ] . clone ( ) ;
let num = match num_times_matched_with . get ( sender . as_str ( ) ) {
Some ( num ) = > * num + 1.0 ,
_ = > 1.0 ,
} ;
num_times_matched_with . insert ( sender , num ) ;
2021-02-01 07:50:59 +00:00
if self . tags_to_keys_cache [ tag . to_string ( ) . as_str ( ) ] . len ( ) = = 1 {
trivial_breaks + = 1 ;
}
}
2021-02-13 07:36:04 +00:00
for ( sender , count ) in num_times_matched_with . iter ( ) {
let expected_matched_count = ( ideal_rate * self . sender_count [ sender . as_str ( ) ] ) ;
let actual_matched_count = ( * count ) ;
let prob = at_least_with_replacement ( actual_matched_count as u64 , self . sender_count [ sender . as_str ( ) ] as u64 , ideal_rate ) . to_f64 ( ) ;
let diff = f64 ::abs ( actual_matched_count - expected_matched_count ) ;
// these numbers are arbitrary, but the point is probability only works if your sample is big enough...
if diff > 4.0 & & actual_matched_count > expected_matched_count & & prob < 0.01 {
let sender_index = self . keybase . iter ( ) . find_position ( | ( d , t ) | t . id ( ) = = * sender ) . unwrap ( ) . 0 ;
event! ( Level ::INFO , " Found Anomalous Relationship Between {sender}({sender_index}) and {recipient}({recipient_index}) {falsepositiverate}, {detected} >> {expected} | Probability of Event: {prob:.9} " , sender = sender , sender_index = sender_index , recipient = pub_key . id ( ) , recipient_index = recipient_index , falsepositiverate = ideal_rate , detected = actual_matched_count , expected = expected_matched_count , prob = prob ) ;
}
}
2021-02-01 23:23:01 +00:00
let p_stats = PartyStatistics {
ideal_rate ,
expected_messages ,
observed_messages ,
observed_rate ,
observed_skew_messages ,
observed_skew ,
trivial_breaks ,
} ;
party_stats . insert ( pub_key . id ( ) , p_stats ) ;
2021-02-01 07:50:59 +00:00
}
2021-02-01 23:23:01 +00:00
2021-02-03 13:05:59 +00:00
( self . oracle . clone ( ) , round_stats , party_stats )
2021-02-01 07:50:59 +00:00
}
}