Add Fallback Directory Candidate Selection Script

"Tor has included a feature to fetch the initial consensus from nodes
 other than the authorities for a while now. We just haven't shipped a
 list of alternate locations for clients to go to yet.

 Reasons why we might want to ship tor with a list of additional places
 where clients can find the consensus is that it makes authority
 reachability and BW less important.

 We want them to have been around and using their current key, address,
 and port for a while now (120 days), and have been running, a guard,
 and a v2 directory mirror for most of that time."

Features:
* whitelist and blacklist for an opt-in/opt-out trial.
* excludes BadExits, tor versions that aren't recommended, and low
  consensus weight directory mirrors.
* reduces the weighting of Exits to avoid overloading them.
* places limits on the weight of any one fallback.
* includes an IPv6 address and orport for each FallbackDir, as
  implemented in #17327. (Tor won't bootstrap using IPv6 fallbacks
  until #17840 is merged.)
* generated output includes timestamps & Onionoo URL for traceability.
* unit test ensures that we successfully load all included default
  fallback directories.

Closes ticket #15775. Patch by "teor".
OnionOO script by "weasel", "teor", "gsathya", and "karsten".

.gitignore

@@ -28,6 +28,11 @@ cscope.*
 # OSX junk
 *.dSYM
 .DS_Store
+# updateFallbackDirs.py temp files
+details-*.json
+uptime-*.json
+*.full_url
+*.last_modified
 
 # /
 /Makefile

changes/feature15775-fallback

@@ -0,0 +1,19 @@
+  o Major features (directory mirrors):
+    - Include an opt-in trial list of Default Fallback Directories in
+      add_default_fallback_dir_servers().
+      "Tor has included a feature to fetch the initial consensus from nodes
+       other than the authorities for a while now. We just haven't shipped a
+       list of alternate locations for clients to go to yet.
+       Reasons why we might want to ship tor with a list of additional places
+       where clients can find the consensus is that it makes authority
+       reachability and BW less important.
+       We want them to have been around and using their current key, address,
+       and port for a while now (120 days), and have been running, a guard,
+       and a v2 directory mirror for most of that time."
+      We exclude BadExits and tor versions that aren't recommended.
+      We include an IPv6 address for each FallbackDir (#8374).
+      (Tor might not use IPv6 fallbacks until #6027 is merged.)
+      The unit test ensures that we successfully load all included
+      default fallback directories.
+      Closes ticket #15775. Patch by "teor".
+      OnionOO script by "weasel", "teor", "gsathya", and "karsten".

scripts/maint/fallback.blacklist

@@ -0,0 +1,19 @@
+# updateFallbackDirs.py directory mirror blacklist
+#
+# Format:
+# [ IPv4[:DirPort] ] [ orport=<ORPort> ] [ id=<ID> ] ...
+#   [ ipv6=<IPv6>[:<IPv6 ORPort>] ]
+#
+# If a sufficiently specific group of attributes matches, the directory mirror
+# will be excluded: (each group is listed on its own line)
+# <IPv4>, <DirPort>
+# <IPv4>, <ORPort>
+# <ID>
+# <IPv6>, <DirPort>
+# <IPv6>, <IPv6 ORPort>
+# If DirPort and ORPort are not present, the entire IP address is blacklisted.
+# (The blacklist overrides the whitelist.)
+
+# If a relay operator doesn't want their relay to be a FallbackDir,
+# enter the following information here:
+# <IPv4>:<DirPort> orport=<ORPort> id=<ID> ipv6=<IPv6>:<IPv6 ORPort>

scripts/maint/fallback.whitelist

@@ -0,0 +1,13 @@
+# updateFallbackDirs.py directory mirror whitelist
+#
+# Format:
+# IPv4:DirPort orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ]
+#
+# All attributes must match for the directory mirror to be included.
+# If the fallback has an ipv6 key, the whitelist line must also have
+# it, and vice versa, otherwise they don't match.
+# (The blacklist overrides the whitelist.)
+
+# If a relay operator wants their relay to be a FallbackDir,
+# enter the following information here:
+# <IPv4>:<DirPort> orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ]

src/or/config.c

@@ -916,6 +916,7 @@ add_default_fallback_dir_servers,(void))
 {
   int i;
   const char *fallback[] = {
+#include "fallback_dirs.inc"
     NULL
   };
   for (i=0; fallback[i]; i++) {

src/or/fallback_dirs.inc

@@ -0,0 +1 @@
+/* This list will be empty until opt-ins are finalised. */

src/or/include.am

@@ -158,6 +158,7 @@ ORHEADERS = \
 	src/or/dnsserv.h				\
 	src/or/eventdns_tor.h				\
 	src/or/ext_orport.h				\
+	src/or/fallback_dirs.inc			\
 	src/or/fp_pair.h				\
 	src/or/geoip.h					\
 	src/or/entrynodes.h				\

src/or/routerlist.c

@@ -1295,8 +1295,8 @@ router_get_fallback_dir_servers(void)
 /** Try to find a running dirserver that supports operations of <b>type</b>.
  *
  * If there are no running dirservers in our routerlist and the
- * <b>PDS_RETRY_IF_NO_SERVERS</b> flag is set, set all the authoritative ones
- * as running again, and pick one.
+ * <b>PDS_RETRY_IF_NO_SERVERS</b> flag is set, set all the fallback ones
+ * (including authorities) as running again, and pick one.
  *
  * If the <b>PDS_IGNORE_FASCISTFIREWALL</b> flag is set, then include
  * dirservers that we can't reach.
@@ -1304,8 +1304,9 @@ router_get_fallback_dir_servers(void)
  * If the <b>PDS_ALLOW_SELF</b> flag is not set, then don't include ourself
  * (if we're a dirserver).
  *
- * Don't pick an authority if any non-authority is viable; try to avoid using
- * servers that have returned 503 recently.
+ * Don't pick a fallback directory mirror if any non-fallback is viable;
+ * (the fallback directory mirrors include the authorities)
+ * try to avoid using servers that have returned 503 recently.
  */
 const routerstatus_t *
 router_pick_directory_server(dirinfo_type_t type, int flags)
@@ -1332,7 +1333,7 @@ router_pick_directory_server(dirinfo_type_t type, int flags)
   log_info(LD_DIR,
            "No reachable router entries for dirservers. "
            "Trying them all again.");
-  /* mark all authdirservers as up again */
+  /* mark all fallback directory mirrors as up again */
   mark_all_dirservers_up(fallback_dir_servers);
   /* try again */
   choice = router_pick_directory_server_impl(type, flags, NULL);

src/test/test_config.c

@@ -1521,9 +1521,9 @@ test_config_adding_dir_servers(void *arg)
     );
 
   /* We need to know if add_default_fallback_dir_servers is called,
+   * whatever the size of the list in fallback_dirs.inc,
    * so we use a version of add_default_fallback_dir_servers that adds
-   * one known default fallback directory.
-   * There doesn't appear to be any need to test it unmocked. */
+   * one known default fallback directory. */
   MOCK(add_default_fallback_dir_servers,
        add_default_fallback_dir_servers_known_default);
 
@@ -1581,20 +1581,6 @@ test_config_adding_dir_servers(void *arg)
    *  Cases expected to yield this outcome:
    *    0 (DirAuthorities, AlternateBridgeAuthority, AlternateDirAuthority
    *       and FallbackDir are all NULL)
-   *
-   * Before #15642 but after #13163 - Stop using default authorities when both
-   * Alternate Dir and Bridge Authority are set
-   * (#13163 was committed in 0.2.6 as c1dd43d823c7)
-   *
-   * The behaviour is different in the following cases
-   * where FallbackDir is NULL:
-   *  2, 6, 8
-   *
-   * In these cases, the Default Fallback Directories are applied, even when
-   * DirAuthorities or AlternateDirAuthority are set.
-   *
-   * However, as the list of default fallback directories is currently empty,
-   * this change doesn't modify any user-visible behaviour.
    */
 
   /*
@@ -3209,11 +3195,39 @@ test_config_adding_dir_servers(void *arg)
   UNMOCK(add_default_fallback_dir_servers);
 }
 
+static void
+test_config_default_fallback_dirs(void *arg)
+{
+  const char *fallback[] = {
+#include "../or/fallback_dirs.inc"
+    NULL
+  };
+
+  int n_included_fallback_dirs = 0;
+  int n_added_fallback_dirs = 0;
+
+  (void)arg;
+  clear_dir_servers();
+
+  while (fallback[n_included_fallback_dirs])
+    n_included_fallback_dirs++;
+
+  add_default_fallback_dir_servers();
+
+  n_added_fallback_dirs = smartlist_len(router_get_fallback_dir_servers());
+
+  tt_assert(n_included_fallback_dirs == n_added_fallback_dirs);
+
+  done:
+  clear_dir_servers();
+}
+
 #define CONFIG_TEST(name, flags)                          \
   { #name, test_config_ ## name, flags, NULL, NULL }
 
 struct testcase_t config_tests[] = {
-  CONFIG_TEST(adding_dir_servers, TT_FORK),
+  CONFIG_TEST(adding_dir_servers, 0),
+  CONFIG_TEST(default_fallback_dirs, 0),
   CONFIG_TEST(resolve_my_address, TT_FORK),
   CONFIG_TEST(addressmap, 0),
   CONFIG_TEST(parse_bridge_line, 0),