Merge branch 'maint-0.3.3' into release-0.3.3

so they aren't mysteriously missing from the tarballs/etc

.appveyor.yml

@@ -1,62 +0,0 @@
-version: 1.0.{build}
-
-clone_depth: 50
-
-environment:
-  compiler: mingw
-
-  matrix:
-  - target: i686-w64-mingw32
-    compiler_path: mingw32
-    openssl_path: /c/OpenSSL-Win32
-  - target: x86_64-w64-mingw32
-    compiler_path: mingw64
-    openssl_path: /c/OpenSSL-Win64
-
-install:
-- ps: >-
-    Function Execute-Command ($commandPath)
-    {
-        & $commandPath $args 2>&1
-        if ( $LastExitCode -ne 0 ) {
-            $host.SetShouldExit( $LastExitCode )
-        }
-    }
-    Function Execute-Bash ()
-    {
-        Execute-Command 'c:\msys64\usr\bin\bash' '-e' '-c' $args
-    }
-    Execute-Command "C:\msys64\usr\bin\pacman" -Sy --noconfirm openssl-devel openssl libevent-devel libevent mingw-w64-i686-libevent mingw-w64-x86_64-libevent mingw-w64-i686-openssl mingw-w64-x86_64-openssl mingw-w64-i686-zstd mingw-w64-x86_64-zstd
-
-build_script:
-- ps: >-
-    if ($env:compiler -eq "mingw") {
-            $oldpath = ${env:Path} -split ';'
-            $buildpath = @("C:\msys64\${env:compiler_path}\bin", "C:\msys64\usr\bin") + $oldpath
-            $env:Path = @($buildpath) -join ';'
-            $env:build = @("${env:APPVEYOR_BUILD_FOLDER}", $env:target) -join '\'
-            Set-Location "${env:APPVEYOR_BUILD_FOLDER}"
-            Execute-Bash 'autoreconf -i'
-            mkdir "${env:build}"
-            Set-Location "${env:build}"
-            Execute-Bash "../configure --prefix=/${env:compiler_path} --build=${env:target} --host=${env:target} --disable-asciidoc --enable-fatal-warnings --with-openssl-dir=${env:openssl_path}"
-            Execute-Bash "V=1 make -j2"
-            Execute-Bash "V=1 make -j2 install"
-     }
-
-test_script:
-- ps: >-
-    if ($env:compiler -eq "mingw") {
-            $oldpath = ${env:Path} -split ';'
-            $buildpath = @("C:\msys64\${env:compiler_path}\bin") + $oldpath
-            $env:Path = $buildpath -join ';'
-            Set-Location "${env:build}"
-            Execute-Bash "VERBOSE=1 make -j2 check"
-    }
-
-on_success:
-- cmd: C:\Python27\python.exe %APPVEYOR_BUILD_FOLDER%\scripts\test\appveyor-irc-notify.py irc.oftc.net:6697 tor-ci success
-
-on_failure:
-- cmd: C:\Python27\python.exe %APPVEYOR_BUILD_FOLDER%\scripts\test\appveyor-irc-notify.py irc.oftc.net:6697 tor-ci failure
-

.travis.yml

@@ -106,8 +106,6 @@ matrix:
       env: DISTCHECK="yes" RUST_OPTIONS=""
     - compiler: gcc
       env: DISTCHECK="yes" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
-    - compiler: gcc
-      env: MODULES_OPTIONS="--disable-module-dirauth"
     ## The "sudo: required" forces non-containerized builds, working
     ## around a Travis CI environment issue: clang LeakAnalyzer fails
     ## because it requires ptrace and the containerized environment no
@@ -120,9 +118,6 @@ matrix:
     - compiler: clang
       sudo: required
       env: RUST_OPTIONS=""
-    - compiler: clang
-      sudo: required
-      env: MODULES_OPTIONS="--disable-module-dirauth"
 
 before_install:
   ## If we're on OSX, homebrew usually needs to updated first
@@ -153,7 +148,7 @@ install:
 
 script:
   - ./autogen.sh
-  - ./configure $RUST_OPTIONS $COVERAGE_OPTIONS $MODULES_OPTIONS --disable-asciidoc --enable-fatal-warnings --disable-silent-rules --enable-fragile-hardening
+  - ./configure $RUST_OPTIONS $COVERAGE_OPTIONS --disable-asciidoc --enable-fatal-warnings --disable-silent-rules --enable-fragile-hardening
   ## We run `make check` because that's what https://jenkins.torproject.org does.
   - if [[ "$DISTCHECK" == "" ]]; then make check; fi
   - if [[ "$DISTCHECK" != "" ]]; then make distcheck DISTCHECK_CONFIGURE_FLAGS="$RUST_OPTIONS $COVERAGE_OPTIONS --disable-asciidoc --enable-fatal-warnings --disable-silent-rules --enable-fragile-hardening"; fi

CONTRIBUTING

@@ -1,39 +0,0 @@
-Contributing to Tor
--------------------
-
-### Getting started
-
-Welcome!
-
-We have a bunch of documentation about how to develop Tor in the
-doc/HACKING/ directory.  We recommend that you start with
-doc/HACKING/README.1st.md , and then go from there.  It will tell
-you how to find your way around the source code, how to get
-involved with the Tor community, how to write patches, and much
-more!
-
-You don't have to be a C developer to help with Tor: have a look
-at https://www.torproject.org/getinvolved/volunteer !
-
-The Tor Project is committed to fostering a inclusive community
-where people feel safe to engage, share their points of view, and
-participate. For the latest version of our Code of Conduct, please
-see
-
-https://gitweb.torproject.org/community/policies.git/plain/code_of_conduct.txt
-
-
-
-### License issues
-
-Tor is distributed under the license terms in the LICENSE -- in
-brief, the "3-clause BSD license".  If you send us code to
-distribute with Tor, it needs to be code that we can distribute
-under those terms.  Please don't send us patches unless you agree
-to allow this.
-
-Some compatible licenses include:
-
-  - 3-clause BSD
-  - 2-clause BSD
-  - CC0 Public Domain Dedication

ChangeLog

@@ -97,456 +97,6 @@ Changes in version 0.3.3.6 - 2018-05-22
       ticket 25857. Patch from "CTassisF".
 
 
-Changes in version 0.3.4.1-alpha - 2018-05-17
-  Tor 0.3.4.1-alpha is the first release in the 0.3.4.x series. It
-  includes refactoring to begin reducing Tor's binary size and idle CPU
-  usage on mobile, along with prep work for new bandwidth scanners,
-  improvements to the experimental "vanguards" feature, and numerous
-  other small features and bugfixes.
-
-  o New system requirements:
-    - Tor no longer tries to support old operating systems without
-      mmap() or some local equivalent. Apparently, compilation on such
-      systems has been broken for some time, without anybody noticing or
-      complaining. Closes ticket 25398.
-
-  o Major feature (directory authority, modularization):
-    - The directory authority subsystem has been modularized. The code
-      is now located in src/or/dirauth/, and is compiled in by default.
-      To disable the module, the configure option
-      --disable-module-dirauth has been added. This module may be
-      disabled by default in some future release. Closes ticket 25610.
-
-  o Major features (main loop, CPU usage):
-    - When Tor is disabled (via DisableNetwork or via hibernation), it
-      no longer needs to run any per-second events. This change should
-      make it easier for mobile applications to disable Tor while the
-      device is sleeping, or Tor is not running. Closes ticket 26063.
-    - Tor no longer enables all of its periodic events by default.
-      Previously, Tor would enable all possible main loop events,
-      regardless of whether it needed them. Furthermore, many of these
-      events are now disabled with Tor is hibernating or DisableNetwork
-      is set. This is a big step towards reducing client CPU usage by
-      reducing the amount of wake-ups the daemon does. Closes ticket
-      25376 and 25762.
-    - The bandwidth-limitation logic has been refactored so that
-      bandwidth calculations are performed on-demand, rather than every
-      TokenBucketRefillInterval milliseconds. This change should improve
-      the granularity of our bandwidth calculations, and limit the
-      number of times that the Tor process needs to wake up when it is
-      idle. Closes ticket 25373.
-    - Move responsibility for many operations from a once-per-second
-      callback to a callback that is only scheduled as needed. Moving
-      this functionality has allowed us to disable the callback when
-      Tor's network is disabled. Once enough items are removed from our
-      once-per-second callback, we can eliminate it entirely to conserve
-      CPU when idle. The functionality removed includes: closing
-      connections, circuits, and channels (ticket 25932); consensus
-      voting (25937); flushing log callbacks (25951); honoring delayed
-      SIGNEWNYM requests (25949); rescanning the consensus cache
-      (25931); saving the state file to disk (25948); warning relay
-      operators about unreachable ports (25952); and keeping track of
-      Tor's uptime (26009).
-
-  o Major bugfixes (directory authorities, security):
-    - When directory authorities read a zero-byte bandwidth file, they
-      would previously log a warning with the contents of an
-      uninitialised buffer. They now log a warning about the empty file
-      instead. Fixes bug 26007; bugfix on 0.2.2.1-alpha.
-
-  o Major bugfixes (crash):
-    - Avoid a rare assertion failure in the circuit build timeout code
-      if we fail to allow any circuits to actually complete. Fixes bug
-      25733; bugfix on 0.2.2.2-alpha.
-
-  o Major bugfixes (directory authority):
-    - Avoid a crash when testing router reachability on a router that
-      could have an ed25519 ID, but which does not. Fixes bug 25415;
-      bugfix on 0.3.3.2-alpha.
-
-  o Major bugfixes (onion service):
-    - Correctly detect when onion services get disabled after HUP. Fixes
-      bug 25761; bugfix on 0.3.2.1.
-
-  o Major bugfixes (protover, voting):
-    - Revise Rust implementation of protover to use a more memory-
-      efficient voting algorithm and corresponding data structures, thus
-      avoiding a potential (but small impact) DoS attack where specially
-      crafted protocol strings would expand to several potential
-      megabytes in memory. In the process, several portions of code were
-      revised to be methods on new, custom types, rather than functions
-      taking interchangeable types, thus increasing type safety of the
-      module. Custom error types and handling were added as well, in
-      order to facilitate better error dismissal/handling in outside
-      crates and avoid mistakenly passing an internal error string to C
-      over the FFI boundary. Many tests were added, and some previous
-      differences between the C and Rust implementations have been
-      remedied. Fixes bug 24031; bugfix on 0.3.3.1-alpha.
-
-  o Major bugfixes (relay, denial of service):
-    - Impose a limit on circuit cell queue size. The limit can be
-      controlled by a consensus parameter. Fixes bug 25226; bugfix
-      on 0.2.4.14-alpha.
-
-  o Minor features (accounting):
-    - When Tor becomes dormant, it now uses a scheduled event to wake up
-      at the right time. Previously, we would use the per-second timer
-      to check whether to wake up, but we no longer have any per-second
-      timers enabled when the network is disabled. Closes ticket 26064.
-
-  o Minor features (code quality):
-    - Add optional spell-checking for the Tor codebase, using the
-      "misspell" program. To use this feature, run "make check-typos".
-      Closes ticket 25024.
-
-  o Minor features (compatibility):
-    - Tor now detects versions of OpenSSL 1.1.0 and later compiled with
-      the no-deprecated option, and builds correctly with them. Closes
-      tickets 19429, 19981, and 25353.
-    - Avoid some compilation warnings with recent versions of LibreSSL.
-      Closes ticket 26006.
-
-  o Minor features (compression, zstd):
-    - When running with zstd, Tor now considers using advanced functions
-      that the zstd maintainers have labeled as potentially unstable. To
-      prevent breakage, Tor will only use this functionality when the
-      runtime version of the zstd library matches the version with which
-      Tor was compiled. Closes ticket 25162.
-
-  o Minor features (configuration):
-    - The "DownloadSchedule" options have been renamed to end with
-      "DownloadInitialDelay". The old names are still allowed, but will
-      produce a warning. Comma-separated lists are still permitted for
-      these options, but all values after the first are ignored (as they
-      have been since 0.2.9). Closes ticket 23354.
-
-  o Minor features (continuous integration):
-    - Our .travis.yml configuration now includes support for testing the
-      results of "make distcheck". (It's not uncommon for "make check"
-      to pass but "make distcheck" to fail.) Closes ticket 25814.
-    - Our Travis CI configuration now integrates with the Coveralls
-      coverage analysis tool. Closes ticket 25818.
-
-  o Minor features (control port):
-    - Introduce GETINFO "current-time/{local,utc}" to return the local
-      and UTC times respectively in ISO format. This helps a controller
-      like Tor Browser detect a time-related error. Closes ticket 25511.
-      Patch by Neel Chauhan.
-    - Introduce new fields to the CIRC_BW event. There are two new
-      fields in each of the read and written directions. The DELIVERED
-      fields report the total valid data on the circuit, as measured by
-      the payload sizes of verified and error-checked relay command
-      cells. The OVERHEAD fields report the total unused bytes in each
-      of these cells. Closes ticket 25903.
-
-  o Minor features (directory authority):
-    - Directory authorities now open their key-pinning files as O_SYNC,
-      to limit their chances of accidentally writing partial lines.
-      Closes ticket 23909.
-
-  o Minor features (directory authority, forward compatibility):
-    - Make the lines of the measured bandwidth file able to contain
-      their entries in any order. Previously, the node_id entry needed
-      to come first. Closes ticket 26004.
-
-  o Minor features (entry guards):
-    - Introduce a new torrc option NumPrimaryGuards for controlling the
-      number of primary guards. Closes ticket 25843.
-
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the May 1 2018 Maxmind GeoLite2 Country
-      database. Closes ticket 26104.
-
-  o Minor features (performance):
-    - Avoid a needless call to malloc() when processing an incoming
-      relay cell. Closes ticket 24914.
-    - Make our timing-wheel code run a tiny bit faster on 32-bit
-      platforms, by preferring 32-bit math to 64-bit. Closes
-      ticket 24688.
-    - Avoid a needless malloc()/free() pair every time we handle an ntor
-      handshake. Closes ticket 25150.
-
-  o Minor features (testing):
-    - Add a unit test for voting_schedule_get_start_of_next_interval().
-      Closes ticket 26014, and helps make unit test coverage
-      more deterministic.
-    - A new unittests module specifically for testing the functions in
-      the (new-ish) bridges.c module has been created with new
-      unittests, raising the code coverage percentages. Closes 25425.
-    - We now have improved testing for addressmap_get_virtual_address()
-      function. This should improve our test coverage, and make our test
-      coverage more deterministic. Closes ticket 25993.
-
-  o Minor features (timekeeping, circuit scheduling):
-    - When keeping track of how busy each circuit have been recently on
-      a given connection, use coarse-grained monotonic timers rather
-      than gettimeofday(). This change should marginally increase
-      accuracy and performance. Implements part of ticket 25927.
-
-  o Minor bugfixes (bandwidth management):
-    - Consider ourselves "low on write bandwidth" if we have exhausted
-      our write bandwidth some time in the last second. This was the
-      documented behavior before, but the actual behavior was to change
-      this value every TokenBucketRefillInterval. Fixes bug 25828;
-      bugfix on 0.2.3.5-alpha.
-
-  o Minor bugfixes (C correctness):
-    - Add a missing lock acquisition in the shutdown code of the control
-      subsystem. Fixes bug 25675; bugfix on 0.2.7.3-rc. Found by
-      Coverity; this is CID 1433643.
-
-  o Minor bugfixes (circuit path selection):
-    - Don't count path selection failures as circuit build failures.
-      This change should eliminate cases where Tor blames its guard or
-      the network for situations like insufficient microdescriptors
-      and/or overly restrictive torrc settings. Fixes bug 25705; bugfix
-      on 0.3.3.1-alpha.
-
-  o Minor bugfixes (client):
-    - Don't consider Tor running as a client if the ControlPort is open,
-      but no actual client ports are open. Fixes bug 26062; bugfix
-      on 0.2.9.4-alpha.
-
-  o Minor bugfixes (code style):
-    - Fixed multiple includes of transports.h in src/or/connection.c
-      Fixes bug 25261; bugfix on 0.2.5.1-alpha.
-    - Remove the unused variable n_possible from the function
-      channel_get_for_extend(). Fixes bug 25645; bugfix on 0.2.4.4-alpha
-
-  o Minor bugfixes (control interface):
-    - Respond with more human-readable error messages to GETINFO exit-
-      policy/* requests. Also, let controller know if an error is
-      transient (response code 551) or not (response code 552). Fixes
-      bug 25852; bugfix on 0.2.8.1-alpha.
-
-  o Minor bugfixes (controller):
-    - Make CIRC_BW event reflect the total of all data sent on a
-      circuit, including padding and dropped cells. Also fix a mis-
-      counting bug when STREAM_BW events were enabled. Fixes bug 25400;
-      bugfix on 0.2.5.2-alpha.
-
-  o Minor bugfixes (correctness, client):
-    - Upon receiving a malformed connected cell, stop processing the cell
-      immediately.  Previously we would mark the connection for close, but
-      continue processing the cell as if the connection were open. Fixes bug
-      26072; bugfix on 0.2.4.7-alpha.
-
-  o Minor bugfixes (directory client):
-    - When unverified-consensus is verified, rename it to cached-
-      consenus. Fixes bug 4187; bugfix on 0.2.0.3-alpha.
-    - Fixed launching a certificate fetch always during the scheduled
-      periodic consensus fetch by fetching only in those cases when
-      consensus are waiting for certs. Fixes bug 24740; bugfix
-      on 0.2.9.1-alpha.
-
-  o Minor bugfixes (documentation):
-    - Stop saying in the manual that clients cache ipv4 dns answers from
-      exit relays. We haven't used them since 0.2.6.3-alpha, and in
-      ticket 24050 we stopped even caching them as of 0.3.2.6-alpha, but
-      we forgot to say so in the man page. Fixes bug 26052; bugfix
-      on 0.3.2.6-alpha.
-
-  o Minor bugfixes (error reporting):
-    - Improve tolerance for directory authorities with skewed clocks.
-      Previously, an authority with a clock more than 60 seconds ahead
-      could cause a client with a correct clock to warn that the
-      client's clock was behind. Now the clocks of a majority of
-      directory authorities have to be ahead of the client before this
-      warning will occur. Fixes bug 25756; bugfix on 0.2.2.25-alpha.
-
-  o Minor bugfixes (Linux seccomp2 sandbox):
-    - Allow the nanosleep() system call, which glibc uses to implement
-      sleep() and usleep(). Fixes bug 24969; bugfix on 0.2.5.1-alpha.
-
-  o Minor bugfixes (onion service):
-    - Fix a memory leak when a v3 onion service is configured and gets a
-      SIGHUP signal. Fixes bug 25901; bugfix on 0.3.2.1-alpha.
-    - When parsing the descriptor signature, look for the token plus an
-      extra white-space at the end. This is more correct but also will
-      allow us to support new fields that might start with "signature".
-      Fixes bug 26069; bugfix on 0.3.0.1-alpha.
-
-  o Minor bugfixes (path selection):
-    - Only select relays when they have the descriptors we prefer to use
-      for them. This change fixes a bug where we could select a relay
-      because it had _some_ descriptor, but reject it later with a
-      nonfatal assertion error because it didn't have the exact one we
-      wanted. Fixes bugs 25691 and 25692; bugfix on 0.3.3.4-alpha.
-
-  o Minor bugfixes (portability):
-    - Do not align mmap length, as it is not required by POSIX, and the
-      getpagesize function is deprecated. Fixes bug 25399; bugfix
-      on 0.1.1.23.
-
-  o Minor bugfixes (portability, FreeBSD):
-    - In have_enough_mem_for_dircache(), the variable DIRCACHE_MIN_MEM_MB
-      does not stringify on FreeBSD, so we switch to tor_asprintf().
-      Fixes bug 20887; bugfix on 0.2.8.1-alpha. Patch by Neel Chauhan.
-
-  o Minor bugfixes (relay statistics):
-    - When a relay is collecting internal statistics about how many
-      create cell requests it has seen of each type, accurately count
-      the requests from relays that temporarily fall out of the
-      consensus. (To be extra conservative, we were already ignoring
-      requests from clients in our counts, and we continue ignoring them
-      here.) Fixes bug 24910; bugfix on 0.2.4.17-rc.
-
-  o Minor bugfixes (relay, crash):
-    - Avoid a crash when running with DirPort set but ORPort tuned off.
-      Fixes a case of bug 23693; bugfix on 0.3.1.1-alpha.
-
-  o Minor bugfixes (restart-in-process):
-    - When shutting down, Tor now clears all the flags in the control.c
-      module. This should prevent a bug where authentication cookies are
-      not generated on restart. Fixes bug 25512; bugfix on 0.3.3.1-alpha.
-
-  o Minor bugfixes (testing):
-    - When testing workqueue event-cancellation, make sure that we
-      actually cancel an event, and that cancel each event with equal
-      probability. (It was previously possible, though extremely
-      unlikely, for our event-canceling test not to cancel any events.)
-      Fixes bug 26008; bugfix on 0.2.6.3-alpha.
-    - Repeat part of the test in test_client_pick_intro() a number of
-      times, to give it consistent coverage. Fixes bug 25996; bugfix
-      on 0.3.2.1-alpha.
-    - Remove randomness from the hs_common/responsible_hsdirs test, so
-      that it always takes the same path through the function it tests.
-      Fixes bug 25997; bugfix on 0.3.2.1-alpha.
-    - Change the behavior of the "channel/outbound" test so that it
-      never causes a 10-second rollover for the EWMA circuitmux code.
-      Previously, this behavior would happen randomly, and result in
-      fluctuating test coverage. Fixes bug 25994; bugfix
-      on 0.3.3.1-alpha.
-    - Use X509_new() to allocate certificates that will be freed later
-      with X509_free(). Previously, some parts of the unit tests had
-      used tor_malloc_zero(), which is incorrect, and which caused test
-      failures on Windows when they were built with extra hardening.
-      Fixes bugs 25943 and 25944; bugfix on 0.2.8.1-alpha. Patch by
-      Marcin Cieślak.
-    - While running the circuit_timeout test, fix the PRNG to a
-      deterministic AES stream, so that the test coverage from this test
-      will itself be deterministic. Fixes bug 25995; bugfix
-      on 0.2.2.2-alpha.
-
-  o Minor bugfixes (vanguards):
-    - Allow the last hop in a vanguard circuit to be the same as our
-      first, to prevent the adversary from influencing guard node choice
-      by choice of last hop. Also prevent the creation of A - B - A
-      paths, or A - A paths, which are forbidden by relays. Fixes bug
-      25870; bugfix on 0.3.3.1-alpha.
-
-  o Code simplification and refactoring:
-    - Remove duplicate code in parse_{c,s}method_line and bootstrap
-      their functionalities into a single function. Fixes bug 6236;
-      bugfix on 0.2.3.6-alpha.
-    - We remove the PortForwsrding and PortForwardingHelper options,
-      related functions, and the port_forwarding tests. These options
-      were used by the now-deprecated Vidalia to help ordinary users
-      become Tor relays or bridges. Closes ticket 25409. Patch by
-      Neel Chauhan.
-    - In order to make the OR and dir checking function in router.c less
-      confusing we renamed some functions and
-      consider_testing_reachability() has been split into
-      router_should_check_reachability() and
-      router_do_reachability_checks(). Also we improved the documentation
-      in some functions. Closes ticket 18918.
-    - Initial work to isolate Libevent usage to a handful of modules in
-      our codebase, to simplify our call structure, and so that we can
-      more easily change event loops in the future if needed. Closes
-      ticket 23750.
-    - Introduce a function to call getsockname() and return tor_addr_t,
-      to save a little complexity throughout the codebase. Closes
-      ticket 18105.
-    - Make hsdir_index in node_t a hsdir_index_t rather than a pointer
-      as hsdir_index is always present. Also, we move hsdir_index_t into
-      or.h. Closes ticket 23094. Patch by Neel Chauhan.
-    - Merge functions used for describing nodes and suppress the
-      functions that do not allocate memory for the output buffer
-      string. NODE_DESC_BUF_LEN constant and format_node_description()
-      function cannot be used externally from router.c module anymore.
-      Closes ticket 25432. Patch by valentecaio.
-    - Our main loop has been simplified so that all important operations
-      happen inside events. Previously, some operations had to happen
-      outside the event loop, to prevent infinite sequences of event
-      activations. Closes ticket 25374.
-    - Put a SHA1 public key digest in hs_service_intro_point_t, and use
-      it in register_intro_circ() and service_intro_point_new(). This
-      prevents the digest from being re-calculated each time. Closes
-      ticket 23107. Patch by Neel Chauhan.
-    - Refactor token-bucket implementations to use a common backend.
-      Closes ticket 25766.
-    - Remove extern declaration of stats_n_seconds_working variable from
-      main, protecting its accesses with get_uptime() and reset_uptime()
-      functions. Closes ticket 25081, patch by “valentecaio”.
-    - Remove our previous logic for "cached gettimeofday()" -- our
-      coarse monotonic timers are fast enough for this purpose, and far
-      less error-prone. Implements part of ticket 25927.
-    - Remove the return value for fascist_firewall_choose_address_base(),
-      and sister functions such as fascist_firewall_choose_address_node()
-      and fascist_firewall_choose_address_rs(). Also, while we're here,
-      initialize the ap argument as leaving it uninitialized can pose a
-      security hazard. Closes ticket 24734. Patch by Neel Chauhan.
-    - Rename two fields of connection_t struct. timestamp_lastwritten is
-      renamed to timestamp_last_write_allowed and timestamp_lastread is
-      renamed to timestamp_last_read_allowed. Closes ticket 24714, patch
-      by "valentecaio".
-    - Since Tor requires C99, remove our old workaround code for libc
-      implementations where free(NULL) doesn't work. Closes ticket 24484.
-    - Use our standard rate-limiting code to deal with excessive
-      libevent failures, rather than the hand-rolled logic we had
-      before. Closes ticket 26016.
-    - We remove the return value of node_get_prim_orport() and
-      node_get_prim_dirport(), and introduce node_get_prim_orport() in
-      node_ipv6_or_preferred() and node_ipv6_dir_preferred() in order to
-      check for a null address. Closes ticket 23873. Patch by
-      Neel Chauhan.
-    - We switch to should_record_bridge_info() in
-      geoip_note_client_seen() and options_need_geoip_info() instead of
-      accessing the configuration values directly. Fixes bug 25290;
-      bugfix on 0.2.1.6-alpha. Patch by Neel Chauhan.
-
-  o Deprecated features:
-    - As we are not recommending 0.2.5 anymore, we require relays that
-      once had an ed25519 key associated with their RSA key to always
-      have that key, instead of allowing them to drop back to a version
-      that didn't support ed25519. This means they need to use a new RSA
-      key if the want to downgrade to an older version of tor without
-      ed25519. Closes ticket 20522.
-
-  o Documentation:
-    - Correct an IPv6 error in the documentation for ExitPolicy. Closes
-      ticket 25857. Patch from "CTassisF".
-
-  o Removed features:
-    - Directory authorities will no longer support voting according to
-      any consensus method before consensus method 25. This keeps
-      authorities compatible with all authorities running 0.2.9.8 and
-      later, and does not break any clients or relays. Implements ticket
-      24378 and proposal 290.
-    - The PortForwarding and PortForwardingHelper features have been
-      removed. The reasoning is, given that implementations of NAT
-      traversal protocols within common consumer grade routers are
-      frequently buggy, and that the target audience for a NAT punching
-      feature is a perhaps less-technically-inclined relay operator,
-      when the helper fails to setup traversal the problems are usually
-      deep, ugly, and very router specific, making them horrendously
-      impossible for technical support to reliable assist with, and thus
-      resulting in frustration all around. Unfortunately, relay
-      operators who would like to run relays behind NATs will need to
-      become more familiar with the port forwarding configurations on
-      their local router. Closes 25409.
-    - The TestingEnableTbEmptyEvent option has been removed. It was used
-      in testing simulations to measure how often connection buckets
-      were emptied, in order to improve our scheduling, but it has not
-      been actively used in years. Closes ticket 25760.
-    - The old "round-robin" circuit multiplexer (circuitmux)
-      implementation has been removed, along with a fairly large set of
-      code that existed to support it. It has not been the default
-      circuitmux since we introduced the "EWMA" circuitmux in 0.2.4.x,
-      but it still required an unreasonable amount of memory and CPU.
-      Closes ticket 25268.
-
-
 Changes in version 0.3.3.5-rc - 2018-04-15
   Tor 0.3.3.5-rc fixes various bugs in earlier versions of Tor,
   including some that could affect reliability or correctness.

Makefile.am

@@ -38,7 +38,6 @@ include contrib/include.am
 
 EXTRA_DIST+= \
 	ChangeLog					\
-	CONTRIBUTING					\
 	INSTALL						\
 	LICENSE						\
 	Makefile.nmake					\
@@ -52,14 +51,14 @@ AM_ETAGSFLAGS=--regex='{c}/MOCK_IMPL([^,]+,\W*\([a-zA-Z0-9_]+\)\W*,/\1/s'
 if COVERAGE_ENABLED
 TEST_CFLAGS=-fno-inline -fprofile-arcs -ftest-coverage
 if DISABLE_ASSERTS_IN_UNIT_TESTS
-TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE -DDISABLE_ASSERTS_IN_UNIT_TESTS @TOR_MODULES_ALL_ENABLED@
+TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE -DDISABLE_ASSERTS_IN_UNIT_TESTS
 else
-TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE @TOR_MODULES_ALL_ENABLED@
+TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE
 endif
 TEST_NETWORK_FLAGS=--coverage --hs-multi-client 1
 else
 TEST_CFLAGS=
-TEST_CPPFLAGS=-DTOR_UNIT_TESTS @TOR_MODULES_ALL_ENABLED@
+TEST_CPPFLAGS=-DTOR_UNIT_TESTS
 TEST_NETWORK_FLAGS=--hs-multi-client 1
 endif
 TEST_NETWORK_WARNING_FLAGS=--quiet --only-warnings
@@ -218,26 +217,6 @@ check-logs:
 	$(top_srcdir)/scripts/maint/checkLogs.pl \
 		$(top_srcdir)/src/*/*.[ch] | sort -n
 
-.PHONY: check-typos
-check-typos:
-	@if test -x "`which misspell 2>&1;true`"; then \
-		echo "Checking for Typos ..."; \
-		(misspell \
-			$(top_srcdir)/src/[^e]*/*.[ch] \
-			$(top_srcdir)/doc \
-			$(top_srcdir)/contrib \
-			$(top_srcdir)/scripts \
-			$(top_srcdir)/README \
-			$(top_srcdir)/ChangeLog \
-			$(top_srcdir)/INSTALL \
-			$(top_srcdir)/ReleaseNotes \
-			$(top_srcdir)/LICENSE); \
-	else \
-		echo "Tor can use misspell to check for typos."; \
-		echo "It seems that you don't have misspell installed."; \
-		echo "You can install the latest version of misspell here: https://github.com/client9/misspell#install"; \
-	fi
-
 .PHONY: check-changes
 check-changes:
 if USEPYTHON

ReleaseNotes

@@ -529,7 +529,7 @@ Changes in version 0.3.3.6 - 2018-05-22
       would call the Rust implementation of
       protover_get_supported_protocols(). This was due to the C version
       returning a static string, whereas the Rust version newly allocated
-      a CString to pass across the FFI boundary. Consequently, the C
+      a CString to pass accross the FFI boundary. Consequently, the C
       code was not expecting to need to free() what it was given. Fixes
       bug 25127; bugfix on 0.3.2.1-alpha.
 

changes/TROVE-2018-005

@@ -1,6 +0,0 @@
-  o Major bugfixes (security, directory authority, denial-of-service):
-    - Fix a bug that could have allowed an attacker to force a
-      directory authority to use up all its RAM by passing it a
-      maliciously crafted protocol versions string. Fixes bug 25517;
-      bugfix on 0.2.9.4-alpha.  This issue is also tracked as
-      TROVE-2018-005.

changes/bug25691_again

@@ -0,0 +1,6 @@
+  o Minor bugfixes (path selection):
+    - Only select relays when they have the descriptors we prefer to
+      use for them. This change fixes a bug where we could select
+      a relay because it had _some_ descriptor, but reject it later with
+      a nonfatal assertion error because it didn't have the exact one we
+      wanted. Fixes bugs 25691 and 25692; bugfix on 0.3.3.4-alpha.

changes/bug25939

@@ -1,3 +0,0 @@
-  o Minor bugfixes (onion services):
-    - Fix a bug that blocked the creation of ephemeral v3 onion services. Fixes
-      bug 25939; bugfix on 0.3.4.1-alpha.

changes/bug26101_26102

@@ -1,5 +0,0 @@
-  o Minor bugfixes (test coverage tools):
-    - Update our "cov-diff" script to handle output from the latest
-      version of gcov, and to remove extraneous timestamp information
-      from its output. Fixes bugs 26101 and 26102; bugfix on
-      0.2.5.1-alpha.

changes/bug26156

@@ -1,3 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Fix compilation when building with OpenSSL 1.1.0 with the
-      "no-deprecated" flag enabled. Fixes bug 26156; bugfix on 0.3.4.1-alpha.

changes/bug26259

@@ -1,4 +0,0 @@
-  o Minor bugfixes (control port):
-    - Do not count 0-length RELAY_COMMAND_DATA cells as valid data in CIRC_BW
-      events. Previously, such cells were counted entirely in the OVERHEAD
-      field. Now they are not. Fixes bug 26259; bugfix on 0.3.4.1-alpha.

changes/doc25237

@@ -1,4 +0,0 @@
-  o Documentation:
-    - In code comment, point the reader to the exact section
-      in Tor specification that specifies circuit close error
-      code values. Resolves ticket 25237.

changes/ticket25549

@@ -1,4 +0,0 @@
-  o Minor features (continuous integration):
-    - Add the necessary configuration files for continuous integration
-      testing on Windows, via the Appveyor platform. Closes ticket 25549.
-      Patches from Marcin Cieślak and Isis Lovecruft.

configure.ac

@@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2017, The Tor Project, Inc.
 dnl See LICENSE for licensing information
 
 AC_PREREQ([2.63])
-AC_INIT([tor],[0.3.4.1-alpha-dev])
+AC_INIT([tor],[0.3.3.6-dev])
 AC_CONFIG_SRCDIR([src/or/main.c])
 AC_CONFIG_MACRO_DIR([m4])
 
@@ -61,8 +61,6 @@ AC_ARG_ENABLE(cargo-online-mode,
    AS_HELP_STRING(--enable-cargo-online-mode, [Allow cargo to make network requests to fetch crates. For builds with rust only.]))
 AC_ARG_ENABLE(restart-debugging,
    AS_HELP_STRING(--enable-restart-debugging, [Build Tor with support for debugging in-process restart. Developers only.]))
-AC_ARG_ENABLE(zstd-advanced-apis,
-   AS_HELP_STRING(--disable-zstd-advanced-apis, [Build without support for zstd's "static-only" APIs.]))
 
 if test "x$enable_coverage" != "xyes" -a "x$enable_asserts_in_tests" = "xno" ; then
     AC_MSG_ERROR([Can't disable assertions outside of coverage build])
@@ -116,10 +114,6 @@ if test "$enable_restart_debugging" = "yes"; then
             [Defined if we're building with support for in-process restart debugging.])
 fi
 
-if test "$enable_zstd_advanced_apis" != "no"; then
-   AC_DEFINE(ENABLE_ZSTD_ADVANCED_APIS, 1,
-             [Defined if we're going to try to use zstd's "static-only" APIs.])
-fi
 
 # systemd support
 if test "x$enable_systemd" = "xno"; then
@@ -230,31 +224,6 @@ if test "x$enable_android" = "xyes"; then
 
 fi
 
-dnl ---
-dnl Tor modules options. These options are namespaced with --disable-module-XXX
-dnl ---
-
-dnl All our modules.
-m4_define(MODULES, dirauth)
-
-dnl Directory Authority module.
-AC_ARG_ENABLE([module-dirauth],
-              AS_HELP_STRING([--disable-module-dirauth],
-                             [Do not build tor with the dirauth module]),
-              [], dnl Action if-given
-              AC_DEFINE([HAVE_MODULE_DIRAUTH], [1],
-                        [Compile with Directory Authority feature support]))
-AM_CONDITIONAL(BUILD_MODULE_DIRAUTH, [test "x$enable_module_dirauth" != "xno"])
-
-dnl Helper variables.
-TOR_MODULES_ALL_ENABLED=
-AC_DEFUN([ADD_MODULE], [
-    MODULE=m4_toupper($1)
-    TOR_MODULES_ALL_ENABLED="${TOR_MODULES_ALL_ENABLED} -DHAVE_MODULE_${MODULE}=1"
-])
-m4_foreach_w([module], MODULES, [ADD_MODULE([module])])
-AC_SUBST(TOR_MODULES_ALL_ENABLED)
-
 dnl check for the correct "ar" when cross-compiling.
 dnl   (AM_PROG_AR was new in automake 1.11.2, which we do not yet require,
 dnl    so kludge up a replacement for the case where it isn't there yet.)
@@ -306,10 +275,7 @@ fi
 AM_CONDITIONAL(USEPYTHON, [test "x$PYTHON" != "x"])
 
 dnl List all external rust crates we depend on here. Include the version
-rust_crates=" \
-    digest-0.7.2 \
-    libc-0.2.39 \
-"
+rust_crates="libc-0.2.39"
 AC_SUBST(rust_crates)
 
 ifdef([AC_C_FLEXIBLE_ARRAY_MEMBER], [
@@ -581,7 +547,6 @@ AC_CHECK_FUNCS(
 	mach_approximate_time \
         memmem \
         memset_s \
-        mmap \
 	pipe \
 	pipe2 \
         prctl \
@@ -650,21 +615,6 @@ fi
 AM_CONDITIONAL(BUILD_READPASSPHRASE_C,
   test "x$ac_cv_func_readpassphrase" = "xno" && test "$bwin32" = "false")
 
-AC_MSG_CHECKING([whether free(NULL) works])
-AC_RUN_IFELSE([AC_LANG_PROGRAM([
-  #include <stdlib.h>
-], [
-char *p = NULL;
-free(p);
-])],
-[free_null_ok=true; AC_MSG_RESULT(yes)],
-[free_null_ok=false; AC_MSG_RESULT(no)],
-[free_null_ok=cross; AC_MSG_RESULT(cross)])
-
-if test "$free_null_ok" = "false"; then
-   AC_MSG_ERROR([Your libc implementation doesn't allow free(NULL), as required by C99.])
-fi
-
 dnl ------------------------------------------------------
 dnl Where do you live, libevent?  And how do we call you?
 
@@ -816,18 +766,9 @@ AC_ARG_WITH(ssl-dir,
 
 AC_MSG_NOTICE([Now, we'll look for OpenSSL >= 1.0.1])
 TOR_SEARCH_LIBRARY(openssl, $tryssldir, [-lssl -lcrypto $TOR_LIB_GDI $TOR_LIB_WS32],
-    [#include <openssl/ssl.h>
-     char *getenv(const char *);],
-    [struct ssl_cipher_st;
-     unsigned SSL_CIPHER_get_id(const struct ssl_cipher_st *);
-     char *getenv(const char *);],
-    dnl This funny-looking test program calls getenv, so that the compiler
-    dnl will neither make code that call SSL_CIPHER_get_id(NULL) [producing
-    dnl a crash], nor optimize out the call to SSL_CIPHER_get_id().
-    dnl We look for SSL_cipher_get_id() because it is present in
-    dnl OpenSSL >=1.0.1, because it is not deprecated, and because Tor
-    dnl depends on it.
-    [if (getenv("THIS_SHOULDNT_BE_SET_X201803")) SSL_CIPHER_get_id((void *)0);], [],
+    [#include <openssl/ssl.h>],
+    [struct ssl_method_st; const struct ssl_method_st *TLSv1_1_method(void);],
+    [TLSv1_1_method();], [],
     [/usr/local/opt/openssl /usr/local/openssl /usr/lib/openssl /usr/local/ssl /usr/lib/ssl /usr/local /opt/openssl])
 
 dnl XXXX check for OPENSSL_VERSION_NUMBER == SSLeay()
@@ -894,10 +835,6 @@ AC_CHECK_MEMBERS([SSL.state], , ,
 [#include <openssl/ssl.h>
 ])
 
-AC_CHECK_SIZEOF(SHA_CTX, , [AC_INCLUDES_DEFAULT()
-#include <openssl/sha.h>
-])
-
 dnl Define the set of checks for KIST scheduler support.
 AC_DEFUN([CHECK_KIST_SUPPORT],[
   dnl KIST needs struct tcp_info and for certain members to exist.
@@ -1016,16 +953,6 @@ if test "x$have_zstd" = "xyes"; then
     AC_DEFINE(HAVE_ZSTD,1,[Have Zstd])
     TOR_ZSTD_CFLAGS="${ZSTD_CFLAGS}"
     TOR_ZSTD_LIBS="${ZSTD_LIBS}"
-
-    dnl now check for zstd functions
-    save_LIBS="$LIBS"
-    save_CFLAGS="$CFLAGS"
-    LIBS="$LIBS $ZSTD_LIBS"
-    CFLAGS="$CFLAGS $ZSTD_CFLAGS"
-    AC_CHECK_FUNCS(ZSTD_estimateCStreamSize \
-                   ZSTD_estimateDCtxSize)
-    LIBS="$save_LIBS"
-    CFLAGS="$save_CFLAGS"
 fi
 AC_SUBST(TOR_ZSTD_CFLAGS)
 AC_SUBST(TOR_ZSTD_LIBS)
@@ -1778,6 +1705,14 @@ AC_CHECK_DECLS([mlockall], , , [
 #include <sys/mman.h>
 #endif])
 
+# Some MinGW environments don't have getpagesize in unistd.h. We don't use
+# AC_CHECK_FUNCS(getpagesize), because other environments rename getpagesize
+# using macros
+AC_CHECK_DECLS([getpagesize], , , [
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif])
+
 # Allow user to specify an alternate syslog facility
 AC_ARG_WITH(syslog-facility,
 AS_HELP_STRING(--with-syslog-facility=LOG, [syslog facility to use (default=LOG_DAEMON)]),
@@ -1915,12 +1850,6 @@ AC_SUBST(BUILDDIR)
 AH_TEMPLATE([BUILDDIR],[tor's build directory])
 AC_DEFINE_UNQUOTED(BUILDDIR,"$BUILDDIR")
 
-if test "x$SRCDIR" = "x"; then
-  SRCDIR=$(cd "$srcdir"; pwd)
-fi
-AH_TEMPLATE([SRCDIR],[tor's sourcedir directory])
-AC_DEFINE_UNQUOTED(SRCDIR,"$SRCDIR")
-
 if test "x$CONFDIR" = "x"; then
   CONFDIR=`eval echo $sysconfdir/tor`
 fi

contrib/win32build/tor-mingw.nsi.in

@@ -8,7 +8,7 @@
 !include "LogicLib.nsh"
 !include "FileFunc.nsh"
 !insertmacro GetParameters
-!define VERSION "0.3.4.1-alpha-dev"
+!define VERSION "0.3.3.6-dev"
 !define INSTALLER "tor-${VERSION}-win32.exe"
 !define WEBSITE "https://www.torproject.org/"
 !define LICENSE "LICENSE"

doc/HACKING/CodingStandards.md

@@ -42,23 +42,6 @@ If you have changed build system components:
    - For example, if you have changed Makefiles, autoconf files, or anything
      else that affects the build system.
 
-License issues
-==============
-
-Tor is distributed under the license terms in the LICENSE -- in
-brief, the "3-clause BSD license".  If you send us code to
-distribute with Tor, it needs to be code that we can distribute
-under those terms.  Please don't send us patches unless you agree
-to allow this.
-
-Some compatible licenses include:
-
-  - 3-clause BSD
-  - 2-clause BSD
-  - CC0 Public Domain Dedication
-
-
-
 How we use Git branches
 =======================
 
@@ -434,4 +417,3 @@ the functions that call your function rely on it doing something, then your
 function should mention that it does that something in the documentation.  If
 you rely on a function doing something beyond what is in its documentation,
 then you should watch out, or it might do something else later.
-

doc/HACKING/CodingStandardsRust.md

@@ -324,26 +324,12 @@ Here are some additional bits of advice and rules:
             }
         }
 
-3. Pass only C-compatible primitive types and bytes over the boundary
+3. Pass only integer types and bytes over the boundary
 
-   Rust's C-compatible primitive types are integers and floats.
-   These types are declared in the [libc crate](https://doc.rust-lang.org/libc/x86_64-unknown-linux-gnu/libc/index.html#types).
-   Most Rust objects have different [representations](https://doc.rust-lang.org/libc/x86_64-unknown-linux-gnu/libc/index.html#types)
-   in C and Rust, so they can't be passed using FFI.
-
-   Tor currently uses the following Rust primitive types from libc for FFI:
-   * defined-size integers: `uint32_t`
-   * native-sized integers: `c_int`
-   * native-sized floats: `c_double`
-   * native-sized raw pointers: `* c_void`, `* c_char`, `** c_char`
-
-   TODO: C smartlist to Stringlist conversion using FFI
-
-   The only non-primitive type which may cross the FFI boundary is
+   The only non-integer type which may cross the FFI boundary is
    bytes, e.g. `&[u8]`.  This SHOULD be done on the Rust side by
-   passing a pointer (`*mut libc::c_char`). The length can be passed
-   explicitly (`libc::size_t`), or the string can be NUL-byte terminated
-   C string.
+   passing a pointer (`*mut libc::c_char`) and a length
+   (`libc::size_t`).
 
    One might be tempted to do this via doing
    `CString::new("blah").unwrap().into_raw()`. This has several problems:

doc/HACKING/HelpfulTools.md

@@ -4,22 +4,6 @@ Useful tools
 These aren't strictly necessary for hacking on Tor, but they can help track
 down bugs.
 
-Travis CI
----------
-It's CI. Looks like this: https://travis-ci.org/torproject/tor.
-
-Runs automatically on Pull Requests sent to torproject/tor. You can set it up
-for your fork to build commits outside of PRs too:
-
-1. sign up for GitHub: https://github.com/join
-2. fork https://github.com/torproject/tor:
-   https://help.github.com/articles/fork-a-repo/
-3. follow https://docs.travis-ci.com/user/getting-started/#To-get-started-with-Travis-CI.
-   skip steps involving `.travis.yml` (we already have one).
-
-Builds should show up on the web at travis-ci.com and on IRC at #tor-ci on
-OFTC. If they don't, ask #tor-dev (also on OFTC).
-
 Jenkins
 -------
 

doc/HACKING/Module.md

@@ -1,111 +0,0 @@
-# Modules in Tor #
-
-This document describes the build system and coding standards when writing a
-module in Tor.
-
-## What is a module? ##
-
-In the context of the tor code base, a module is a subsystem that we can
-selectively enable or disable, at `configure` time.
-
-Currently, there is only one module:
-
-  - Directory Authority subsystem (dirauth)
-
-It is located in its own directory in `src/or/dirauth/`. To disable it, one
-need to pass `--disable-module-dirauth` at configure time. All modules are
-currently enabled by default.
-
-## Build System ##
-
-The changes to the build system are pretty straightforward.
-
-1. Locate in the `configure.ac` file this define: `m4_define(MODULES`. It
-   contains a list (white-space separated) of the module in tor. Add yours to
-   the list.
-
-2. Use the `AC_ARG_ENABLE([module-dirauth]` template for your new module. We
-   use the "disable module" approach instead of enabling them one by one. So,
-   by default, tor will build all the modules.
-
-   This will define the `HAVE_MODULE_<name>` statement which can be used in
-   the C code to conditionally compile things for your module. And the
-   `BUILD_MODULE_<name>` is also defined for automake files (e.g: include.am).
-
-3. In the `src/or/include.am` file, locate the `MODULE_DIRAUTH_SOURCES` value.
-   You need to create your own `_SOURCES` variable for your module and then
-   conditionally add the it to `LIBTOR_A_SOURCES` if you should build the
-   module.
-
-   It is then **very** important to add your SOURCES variable to
-   `src_or_libtor_testing_a_SOURCES` so the tests can build it.
-
-4. Do the same for header files, locate `ORHEADERS +=` which always add all
-   headers of all modules so the symbol can be found for the module entry
-   points.
-
-Finally, your module will automatically be included in the
-`TOR_MODULES_ALL_ENABLED` variable which is used to build the unit tests. They
-always build everything in order to tests everything.
-
-## Coding ##
-
-As mentioned above, a module must be isolated in its own directory (name of
-the module) in `src/or/`.
-
-There are couples of "rules" you want to follow:
-
-* Minimize as much as you can the number of entry points into your module.
-  Less is always better but of course that doesn't work out for every use
-  case. However, it is a good thing to always keep that in mind.
-
-* Do **not** use the `HAVE_MODULE_<name>` define outside of the module code
-  base. Every entry point should have a second definition if the module is
-  disabled. For instance:
-
-  ```
-  #ifdef HAVE_MODULE_DIRAUTH
-
-  int sr_init(int save_to_disk);
-
-  #else /* HAVE_MODULE_DIRAUTH */
-
-  static inline int
-  sr_init(int save_to_disk)
-  {
-    (void) save_to_disk;
-    return 0;
-  }
-
-  #endif /* HAVE_MODULE_DIRAUTH */
-
-  ```
-
-  The main reason for this approach is to avoid having conditional code
-  everywhere in the code base. It should be centralized as much as possible
-  which helps maintainability but also avoids conditional spaghetti code
-  making the code much more difficult to follow/understand.
-
-* It is possible that you end up with code that needs to be used by the rest
-  of the code base but is still part of your module. As a good example, if you
-  look at `src/or/shared_random_client.c`: it contains code needed by the hidden
-  service subsystem but mainly related to the shared random subsystem very
-  specific to the dirauth module.
-
-  This is fine but try to keep it as lean as possible and never use the same
-  filename as the one in the module. For example, this is a bad idea and
-  should never be done:
-
-    - `src/or/shared_random.c`
-    - `src/or/dirauth/shared_random.c`
-
-* When you include headers from the module, **always** use the full module
-  path in your statement. Example:
-
-  `#include "dirauth/dirvote.h"`
-
-  The main reason is that we do **not** add the module include path by default
-  so it needs to be specified. But also, it helps our human brain understand
-  which part comes from a module or not.
-
-  Even **in** the module itself, use the full include path like above.

doc/HACKING/ReleasingTor.md

@@ -34,7 +34,7 @@ new Tor release:
 
    What about Coverity Scan?
 
-   What about clang scan-build?
+   What about clan scan-build?
 
    Does 'make distcheck' complain?
 

doc/include.am

@@ -35,15 +35,10 @@ EXTRA_DIST+= doc/asciidoc-helper.sh			\
 	     doc/TUNING						\
 	     doc/HACKING/README.1st.md				\
 	     doc/HACKING/CodingStandards.md 			\
-	     doc/HACKING/CodingStandardsRust.md			\
-	     doc/HACKING/Fuzzing.md				\
 	     doc/HACKING/GettingStarted.md 			\
-	     doc/HACKING/GettingStartedRust.md 			\
 	     doc/HACKING/HelpfulTools.md 			\
 	     doc/HACKING/HowToReview.md  			\
-	     doc/HACKING/Module.md				\
 	     doc/HACKING/ReleasingTor.md                        \
-	     doc/HACKING/Tracing.md				\
 	     doc/HACKING/WritingTests.md
 
 docdir = @docdir@

doc/tor.1.txt

@@ -311,9 +311,7 @@ GENERAL OPTIONS
     other than controller connections, and we close (and don't reattempt)
     any outbound
     connections.  Controllers sometimes use this option to avoid using
-    the network until Tor is fully configured.  Tor will make still certain
-    network-related calls (like DNS lookups) as a part of its configuration
-    process, even if DisableNetwork is set. (Default: 0)
+    the network until Tor is fully configured. (Default: 0)
 
 [[ConstrainedSockets]] **ConstrainedSockets** **0**|**1**::
     If set, Tor will tell the kernel to attempt to shrink the buffers for all
@@ -368,8 +366,7 @@ GENERAL OPTIONS
 
 [[ControlSocket]] **ControlSocket** __Path__::
     Like ControlPort, but listens on a Unix domain socket, rather than a TCP
-    socket. '0' disables ControlSocket. (Unix and Unix-like systems only.)
-    (Default: 0)
+    socket. '0' disables ControlSocket (Unix and Unix-like systems only.)
 
 [[ControlSocketsGroupWritable]] **ControlSocketsGroupWritable** **0**|**1**::
     If this option is set to 0, don't allow the filesystem group to read and
@@ -789,15 +786,17 @@ GENERAL OPTIONS
     This is useful when running on flash memory or other media that support
     only a limited number of writes. (Default: 0)
 
-[[CircuitPriorityHalflife]] **CircuitPriorityHalflife** __NUM__::
+[[CircuitPriorityHalflife]] **CircuitPriorityHalflife** __NUM1__::
     If this value is set, we override the default algorithm for choosing which
-    circuit's cell to deliver or relay next. It is delivered first to the
-    circuit that has the lowest weighted cell count, where cells are weighted
-    exponentially according to this value (in seconds). If the value is -1, it
-    is taken from the consensus if possible else it will fallback to the
-    default value of 30. Minimum: 1, Maximum: 2147483647. This can be defined
-    as a float value. This is an advanced option; you generally shouldn't have
-    to mess with it. (Default: -1)
+    circuit's cell to deliver or relay next. When the value is 0, we
+    round-robin between the active circuits on a connection, delivering one
+    cell from each in turn. When the value is positive, we prefer delivering
+    cells from whichever connection has the lowest weighted cell count, where
+    cells are weighted exponentially according to the supplied
+    CircuitPriorityHalflife value (in seconds). If this option is not set at
+    all, we use the behavior recommended in the current consensus
+    networkstatus. This is an advanced option; you generally shouldn't have
+    to mess with it. (Default: not set)
 
 [[CountPrivateBandwidth]] **CountPrivateBandwidth** **0**|**1**::
     If this option is set, then Tor's rate-limiting applies not only to
@@ -814,9 +813,10 @@ GENERAL OPTIONS
 
 [[NoExec]] **NoExec** **0**|**1**::
     If this option is set to 1, then Tor will never launch another
-    executable, regardless of the settings of ClientTransportPlugin
-    or ServerTransportPlugin.  Once this option has been set to 1,
-    it cannot be set back to 0 without restarting Tor. (Default: 0)
+    executable, regardless of the settings of PortForwardingHelper,
+    ClientTransportPlugin, or ServerTransportPlugin.  Once this
+    option has been set to 1, it cannot be set back to 0 without
+    restarting Tor. (Default: 0)
 
 [[Schedulers]] **Schedulers** **KIST**|**KISTLite**|**Vanilla**::
     Specify the scheduler type that tor should use. The scheduler is
@@ -1294,11 +1294,9 @@ The following options are useful only for clients (that is, if
     2 minutes)
 
 [[TokenBucketRefillInterval]] **TokenBucketRefillInterval** __NUM__ [**msec**|**second**]::
-    Set the refill delay interval of Tor's token bucket to NUM milliseconds.
-    NUM must be between 1 and 1000, inclusive.  When Tor is out of bandwidth,
-    on a connection or globally, it will wait up to this long before it tries
-    to use that connection again.
-    Note that bandwidth limits are still expressed in bytes per second: this
+    Set the refill interval of Tor's token bucket to NUM milliseconds.
+    NUM must be between 1 and 1000, inclusive.  Note that the configured
+    bandwidth limits are still expressed in bytes per second: this
     option only affects the frequency with which Tor checks to see whether
     previously exhausted connections may read again.
     Can not be changed while tor is running. (Default: 100 msec)
@@ -1355,13 +1353,6 @@ The following options are useful only for clients (that is, if
     number from the guard-n-primary-guards-to-use consensus parameter, and
     default to 1 if the consensus parameter isn't set. (Default: 0)
 
-[[NumPrimaryGuards]] **NumPrimaryGuards** __NUM__::
-    If UseEntryGuards is set to 1, we will try to pick NUM routers for our
-    primary guard list, which is the set of routers we strongly prefer when
-    connecting to the Tor network. If NUM is 0, we try to learn the number from
-    the guard-n-primary-guards consensus parameter, and default to 3 if the
-    consensus parameter isn't set. (Default: 0)
-
 [[NumDirectoryGuards]] **NumDirectoryGuards** __NUM__::
     If UseEntryGuards is set to 1, we try to make sure we have at least NUM
     routers to use as directory guards. If this option is set to 0, use the
@@ -1415,7 +1406,7 @@ The following options are useful only for clients (that is, if
 
 [[HTTPTunnelPort]] **HTTPTunnelPort**  \['address':]__port__|**auto** [_isolation flags_]::
     Open this port to listen for proxy connections using the "HTTP CONNECT"
-    protocol instead of SOCKS. Set this to
+    protocol instead of SOCKS. Set this to 0
     0 if you don't want to allow "HTTP CONNECT" connections. Set the port
     to "auto" to have Tor pick a port for you. This directive can be
     specified multiple times to bind to multiple addresses/ports.  See
@@ -1455,7 +1446,7 @@ The following options are useful only for clients (that is, if
     Set this to "default", or leave it unconfigured, to use regular IPTables
     on Linux, or to use pf +rdr-to+ rules on *BSD systems. +
  +
-    (Default: "default")
+    (Default: "default".)
 
 [[NATDPort]] **NATDPort** \['address':]__port__|**auto** [_isolation flags_]::
     Open this port to listen for connections from old versions of ipfw (as
@@ -1591,14 +1582,6 @@ The following options are useful only for clients (that is, if
     ExcludeNodes have higher priority than HSLayer2Nodes,
     which means that nodes specified in ExcludeNodes will not be
     picked.
- +
-    When either this option or HSLayer3Nodes are set, the /16 subnet
-    and node family restrictions are removed for hidden service
-    circuits. Additionally, we allow the guard node to be present
-    as the Rend, HSDir, and IP node, and as the hop before it. This
-    is done to prevent the adversary from inferring information
-    about our guard, layer2, and layer3 node choices at later points
-    in the path.
  +
     This option is meant to be managed by a Tor controller such as
     https://github.com/mikeperry-tor/vanguards that selects and
@@ -1645,14 +1628,6 @@ The following options are useful only for clients (that is, if
     ExcludeNodes have higher priority than HSLayer3Nodes,
     which means that nodes specified in ExcludeNodes will not be
     picked.
- +
-    When either this option or HSLayer2Nodes are set, the /16 subnet
-    and node family restrictions are removed for hidden service
-    circuits. Additionally, we allow the guard node to be present
-    as the Rend, HSDir, and IP node, and as the hop before it. This
-    is done to prevent the adversary from inferring information
-    about our guard, layer2, and layer3 node choices at later points
-    in the path.
   +
     This option is meant to be managed by a Tor controller such as
     https://github.com/mikeperry-tor/vanguards that selects and
@@ -1763,31 +1738,34 @@ The following options are useful only for clients (that is, if
     prevent your Tor client from bootstrapping.  If this option is negative,
     Tor will use a default value chosen by the directory authorities. If the
     directory authorities do not choose a value, Tor will default to 0.6.
-    (Default: -1)
+    (Default: -1.)
 
-[[ClientBootstrapConsensusAuthorityDownloadInitialDelay]] **ClientBootstrapConsensusAuthorityDownloadInitialDelay** __N__::
-    Initial delay in seconds for when clients should download consensuses from authorities
+[[ClientBootstrapConsensusAuthorityDownloadSchedule]] **ClientBootstrapConsensusAuthorityDownloadSchedule** __N__,__N__,__...__::
+    Schedule for when clients should download consensuses from authorities
     if they are bootstrapping (that is, they don't have a usable, reasonably
     live consensus). Only used by clients fetching from a list of fallback
     directory mirrors. This schedule is advanced by (potentially concurrent)
     connection attempts, unlike other schedules, which are advanced by
-    connection failures. (Default: 6)
+    connection failures. (Default: 6, 11, 3600, 10800, 25200, 54000, 111600,
+    262800)
 
-[[ClientBootstrapConsensusFallbackDownloadInitialDelay]] **ClientBootstrapConsensusFallbackDownloadInitialDelay** __N__::
-    Initial delay in seconds for when clients should download consensuses from fallback
+[[ClientBootstrapConsensusFallbackDownloadSchedule]] **ClientBootstrapConsensusFallbackDownloadSchedule** __N__,__N__,__...__::
+    Schedule for when clients should download consensuses from fallback
     directory mirrors if they are bootstrapping (that is, they don't have a
     usable, reasonably live consensus). Only used by clients fetching from a
     list of fallback directory mirrors. This schedule is advanced by
     (potentially concurrent) connection attempts, unlike other schedules,
-    which are advanced by connection failures. (Default: 0)
+    which are advanced by connection failures. (Default: 0, 1, 4, 11, 3600,
+    10800, 25200, 54000, 111600, 262800)
 
-[[ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay]] **ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay** __N__::
-    Initial delay in seconds for when clients should download consensuses from authorities
+[[ClientBootstrapConsensusAuthorityOnlyDownloadSchedule]] **ClientBootstrapConsensusAuthorityOnlyDownloadSchedule** __N__,__N__,__...__::
+    Schedule for when clients should download consensuses from authorities
     if they are bootstrapping (that is, they don't have a usable, reasonably
     live consensus). Only used by clients which don't have or won't fetch
     from a list of fallback directory mirrors. This schedule is advanced by
     (potentially concurrent) connection attempts, unlike other schedules,
-    which are advanced by connection failures. (Default: 0)
+    which are advanced by connection failures. (Default: 0, 3, 7, 3600,
+    10800, 25200, 54000, 111600, 262800)
 
 [[ClientBootstrapConsensusMaxInProgressTries]] **ClientBootstrapConsensusMaxInProgressTries** __NUM__::
     Try this many simultaneous connections to download a consensus before
@@ -1918,7 +1896,7 @@ is non-zero):
     If you want to use a reduced exit policy rather than the default exit
     policy, set "ReducedExitPolicy 1". If you want to _replace_ the default
     exit policy with your custom exit policy, end your exit policy with either
-    a reject *:* or an accept *:*. Otherwise, you're _augmenting_ (prepending
+    a reject *:* or an accept *:*. Otherwise, you’re _augmenting_ (prepending
     to) the default or reduced exit policy. +
  +
     The default exit policy is:
@@ -2081,8 +2059,6 @@ is non-zero):
 [[Nickname]] **Nickname** __name__::
     Set the server's nickname to \'name'. Nicknames must be between 1 and 19
     characters inclusive, and must contain only the characters [a-zA-Z0-9].
-    If not set, **Unnamed** will be used. Relays can always be uniquely identified
-    by their identity fingerprints.
 
 [[NumCPUs]] **NumCPUs** __num__::
     How many processes to use at once for decrypting onionskins and other
@@ -2118,6 +2094,18 @@ is non-zero):
     For obvious reasons, NoAdvertise and NoListen are mutually exclusive, and
     IPv4Only and IPv6Only are mutually exclusive.
 
+[[PortForwarding]] **PortForwarding** **0**|**1**::
+    Attempt to automatically forward the DirPort and ORPort on a NAT router
+    connecting this Tor server to the Internet. If set, Tor will try both
+    NAT-PMP (common on Apple routers) and UPnP (common on routers from other
+    manufacturers). (Default: 0)
+
+[[PortForwardingHelper]] **PortForwardingHelper** __filename__|__pathname__::
+    If PortForwarding is set, use this executable to configure the forwarding.
+    If set to a filename, the system path will be searched for the executable.
+    If set to a path, only the specified path will be executed.
+    (Default: tor-fw-helper)
+
 [[PublishServerDescriptor]] **PublishServerDescriptor** **0**|**1**|**v3**|**bridge**,**...**::
     This option specifies which descriptors Tor will publish when acting as
     a relay. You can
@@ -2281,8 +2269,7 @@ is non-zero):
     sent and received by this relay, in addition to total cell counts.
     These statistics are rounded, and omitted if traffic is low. This
     information is important for load balancing decisions related to padding.
-    If ExtraInfoStatistics is enabled, it will be published
-    as a part of extra-info document. (Default: 1)
+    (Default: 1)
 
 [[DirReqStatistics]] **DirReqStatistics** **0**|**1**::
     Relays and bridges only.
@@ -2381,11 +2368,6 @@ is non-zero):
     KeywDirectory. If the option is set to 1, make the KeyDirectory readable
     by the default GID. (Default: 0)
 
-[[RephistTrackTime]] **RephistTrackTime** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**::
-    Tells an authority, or other node tracking node reliability and history,
-    that fine-grained information about nodes can be discarded when it hasn't
-    changed for a given amount of time.  (Default: 24 hours)
-
 
 DIRECTORY SERVER OPTIONS
 ------------------------
@@ -2755,6 +2737,11 @@ on the public Tor network.
     different identity.  This feature is used to migrate directory authority
     keys in the event of a compromise.  (Default: 0)
 
+[[RephistTrackTime]] **RephistTrackTime** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**::
+    Tells an authority, or other node tracking node reliability and history,
+    that fine-grained information about nodes can be discarded when it hasn't
+    changed for a given amount of time.  (Default: 24 hours)
+
 [[AuthDirHasIPv6Connectivity]] **AuthDirHasIPv6Connectivity** **0**|**1**::
     Authoritative directories only. When set to 0, OR ports with an
     IPv6 address are not included in the authority's votes. When set to 1,
@@ -2922,9 +2909,12 @@ The following options are used for running a testing Tor network.
        AssumeReachable 1
        AuthDirMaxServersPerAddr 0
        AuthDirMaxServersPerAuthAddr 0
-       ClientBootstrapConsensusAuthorityDownloadInitialDelay 0
-       ClientBootstrapConsensusFallbackDownloadInitialDelay 0
-       ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay 0
+       ClientBootstrapConsensusAuthorityDownloadSchedule 0, 2,
+          4 (for 40 seconds), 8, 16, 32, 60
+       ClientBootstrapConsensusFallbackDownloadSchedule 0, 1,
+          4 (for 40 seconds), 8, 16, 32, 60
+       ClientBootstrapConsensusAuthorityOnlyDownloadSchedule 0, 1,
+          4 (for 40 seconds), 8, 16, 32, 60
        ClientDNSRejectInternalAddresses 0
        ClientRejectInternalAddresses 0
        CountPrivateBandwidth 1
@@ -2939,16 +2929,17 @@ The following options are used for running a testing Tor network.
        TestingV3AuthInitialDistDelay 20 seconds
        TestingAuthDirTimeToLearnReachability 0 minutes
        TestingEstimatedDescriptorPropagationTime 0 minutes
-       TestingServerDownloadInitialDelay 0
-       TestingClientDownloadInitialDelay 0
-       TestingServerConsensusDownloadInitialDelay 0
-       TestingClientConsensusDownloadInitialDelay 0
-       TestingBridgeDownloadInitialDelay 10
-       TestingBridgeBootstrapDownloadInitialDelay 0
+       TestingServerDownloadSchedule 0, 0, 0, 5, 10, 15, 20, 30, 60
+       TestingClientDownloadSchedule 0, 0, 5, 10, 15, 20, 30, 60
+       TestingServerConsensusDownloadSchedule 0, 0, 5, 10, 15, 20, 30, 60
+       TestingClientConsensusDownloadSchedule 0, 0, 5, 10, 15, 20, 30, 60
+       TestingBridgeDownloadSchedule 10, 30, 60
+       TestingBridgeBootstrapDownloadSchedule 0, 0, 5, 10, 15, 20, 30, 60
        TestingClientMaxIntervalWithoutRequest 5 seconds
        TestingDirConnectionMaxStall 30 seconds
        TestingEnableConnBwEvent 1
        TestingEnableCellStatsEvent 1
+       TestingEnableTbEmptyEvent 1
 
 [[TestingV3AuthInitialVotingInterval]] **TestingV3AuthInitialVotingInterval** __N__ **minutes**|**hours**::
     Like V3AuthVotingInterval, but for initial voting interval before the first
@@ -2983,31 +2974,37 @@ The following options are used for running a testing Tor network.
     Minimum value for the Fast flag.  Overrides the ordinary minimum taken
     from the consensus when TestingTorNetwork is set. (Default: 0.)
 
-[[TestingServerDownloadInitialDelay]] **TestingServerDownloadInitialDelay** __N__::
-    Initial delay in seconds for when servers should download things in general. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 0)
+[[TestingServerDownloadSchedule]] **TestingServerDownloadSchedule** __N__,__N__,__...__::
+    Schedule for when servers should download things in general. Changing this
+    requires that **TestingTorNetwork** is set. (Default: 0, 0, 0, 60, 60, 120,
+    300, 900, 2147483647)
 
-[[TestingClientDownloadInitialDelay]] **TestingClientDownloadInitialDelay** __N__::
-    Initial delay in seconds for when clients should download things in general. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 0)
+[[TestingClientDownloadSchedule]] **TestingClientDownloadSchedule** __N__,__N__,__...__::
+    Schedule for when clients should download things in general. Changing this
+    requires that **TestingTorNetwork** is set. (Default: 0, 0, 60, 300, 600,
+    2147483647)
 
-[[TestingServerConsensusDownloadInitialDelay]] **TestingServerConsensusDownloadInitialDelay** __N__::
-    Initial delay in seconds for when servers should download consensuses. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 0)
+[[TestingServerConsensusDownloadSchedule]] **TestingServerConsensusDownloadSchedule** __N__,__N__,__...__::
+    Schedule for when servers should download consensuses. Changing this
+    requires that **TestingTorNetwork** is set. (Default: 0, 0, 60, 300, 600,
+    1800, 1800, 1800, 1800, 1800, 3600, 7200)
 
-[[TestingClientConsensusDownloadInitialDelay]] **TestingClientConsensusDownloadInitialDelay** __N__::
-    Initial delay in seconds for when clients should download consensuses. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 0)
+[[TestingClientConsensusDownloadSchedule]] **TestingClientConsensusDownloadSchedule** __N__,__N__,__...__::
+    Schedule for when clients should download consensuses. Changing this
+    requires that **TestingTorNetwork** is set. (Default: 0, 0, 60, 300, 600,
+    1800, 3600, 3600, 3600, 10800, 21600, 43200)
 
-[[TestingBridgeDownloadInitialDelay]] **TestingBridgeDownloadInitialDelay** __N__::
-    Initial delay in seconds for when clients should download each bridge descriptor when they
+[[TestingBridgeDownloadSchedule]] **TestingBridgeDownloadSchedule** __N__,__N__,__...__::
+    Schedule for when clients should download each bridge descriptor when they
     know that one or more of their configured bridges are running. Changing
-    this requires that **TestingTorNetwork** is set. (Default: 10800)
+    this requires that **TestingTorNetwork** is set. (Default: 10800, 25200,
+    54000, 111600, 262800)
 
-[[TestingBridgeBootstrapDownloadInitialDelay]] **TestingBridgeBootstrapDownloadInitialDelay** __N__::
-    Initial delay in seconds for when clients should download each bridge descriptor when they
+[[TestingBridgeBootstrapDownloadSchedule]] **TestingBridgeBootstrapDownloadSchedule** __N__,__N__,__...__::
+    Schedule for when clients should download each bridge descriptor when they
     have just started, or when they can not contact any of their bridges.
-    Changing this requires that **TestingTorNetwork** is set. (Default: 0)
+    Changing this requires that **TestingTorNetwork** is set. (Default: 0, 30,
+    90, 600, 3600, 10800, 25200, 54000, 111600, 262800)
 
 [[TestingClientMaxIntervalWithoutRequest]] **TestingClientMaxIntervalWithoutRequest** __N__ **seconds**|**minutes**::
     When directory clients have only a few descriptors to request, they batch
@@ -3080,6 +3077,11 @@ The following options are used for running a testing Tor network.
     events.  Changing this requires that **TestingTorNetwork** is set.
     (Default: 0)
 
+[[TestingEnableTbEmptyEvent]] **TestingEnableTbEmptyEvent** **0**|**1**::
+    If this option is set, then Tor controllers may register for TB_EMPTY
+    events.  Changing this requires that **TestingTorNetwork** is set.
+    (Default: 0)
+
 [[TestingMinExitFlagThreshold]] **TestingMinExitFlagThreshold**  __N__ **KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Sets a lower-bound for assigning an exit flag when running as an
     authority on a testing network. Overrides the usual default lower bound

scripts/maint/updateFallbackDirs.py

@@ -173,6 +173,12 @@ MAX_LIST_FILE_SIZE = 1024 * 1024
 # meant that we had to rebuild the list more often. We want fallbacks to be
 # stable for 2 years, so we set it to a few months.
 #
+# There was a bug in Tor 0.2.8.1-alpha and earlier where a relay temporarily
+# submits a 0 DirPort when restarted.
+# This causes OnionOO to (correctly) reset its stability timer.
+# Affected relays should upgrade to Tor 0.2.9 or later, which has a fix
+# for this issue.
+#
 # If a relay changes address or port, that's it, it's not useful any more,
 # because clients can't find it
 ADDRESS_AND_PORT_STABLE_DAYS = 90

scripts/test/appveyor-irc-notify.py

@@ -1,192 +0,0 @@
-# coding=utf8
-# Copyright (C) 2015-2016 Christopher R. Wood
-# Copyright (c) 2018 The Tor Project
-# Copyright (c) 2018 isis agora lovecruft
-#
-# From: https://raw.githubusercontent.com/gridsync/gridsync/def54f8166089b733d166665fdabcad4cdc526d8/misc/irc-notify.py
-# and: https://github.com/gridsync/gridsync
-#
-# Modified by nexB on October 2016:
-#  - rework the handling of environment variables.
-#  - made the script use functions
-#  - support only Appveyor loading its environment variable to craft IRC notices.
-#
-# Modified by isis agora lovecruft <isis@torproject.org> in 2018:
-#  - Make IRC server configurable.
-#  - Make bot IRC nick deterministic.
-#  - Make bot join the channel rather than sending NOTICE messages externally.
-#  - Fix a bug which always caused sys.exit() to be logged as a traceback.
-#  - Actually reset the IRC colour codes after printing.
-#
-# Modified by Marcin Cieślak in 2018:
-#  - Accept UTF-8
-#  - only guess github URLs
-#  - stop using ANSI colors
-
-# This program is free software; you can redistribute it and/or modify it under the
-# terms of the GNU General Public License as published by the Free Software Foundation;
-# either version 2 of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT ANY
-# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
-# PARTICULAR PURPOSE. See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with this
-# program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street,
-# Fifth Floor, Boston, MA 02110-1301 USA.
-
-"""Simple AppVeyor IRC notification script.
-
-The first argument is an IRC server and port; the second is the channel. Other
-arguments passed to the script will be sent as notice messages content and any
-{var}-formatted environment variables will be expanded automatically, replaced
-with a corresponding Appveyor environment variable value. Use commas to
-delineate multiple messages.
-
-
-Example:
-export APPVEYOR_URL=https://ci.appveyor.com
-export APPVEYOR_PROJECT_NAME=tor
-export APPVEYOR_REPO_COMMIT_AUTHOR=isislovecruft
-export APPVEYOR_REPO_COMMIT_TIMESTAMP=2018-04-23
-export APPVEYOR_REPO_PROVIDER=gihub
-export APPVEYOR_REPO_BRANCH=repo_branch
-export APPVEYOR_PULL_REQUEST_TITLE=pull_request_title
-export APPVEYOR_BUILD_VERSION=1
-export APPVEYOR_REPO_COMMIT=22c95b72e29248dc4de9b85e590ee18f6f587de8
-export APPVEYOR_REPO_COMMIT_MESSAGE="some IRC test"
-export APPVEYOR_ACCOUNT_NAME=isislovecruft
-export APPVEYOR_PULL_REQUEST_NUMBER=pull_request_number
-export APPVEYOR_REPO_NAME=isislovecruft/tor
-python ./appveyor-irc-notify.py irc.oftc.net:6697 tor-ci '{repo_name} {repo_branch} {short_commit} - {repo_commit_author}: {repo_commit_message}','Build #{build_version} passed. Details: {build_url} |  Commit: {commit_url}
-
-See also https://github.com/gridsync/gridsync/blob/master/appveyor.yml for examples
-in Appveyor's YAML:
-
-    on_success:
-      - "python scripts/test/appveyor-irc-notify.py irc.oftc.net:6697 tor-ci success
-    on_failure:
-      - "python scripts/test/appveyor-irc-notify.py irc.oftc.net:6697 tor-ci failure
-"""
-
-from __future__ import print_function
-from __future__ import absolute_import
-
-import os
-import random
-import socket
-import ssl
-import sys
-import time
-
-
-def appveyor_vars():
-    """
-    Return a dict of key value carfted from appveyor environment variables.
-    """
-
-    vars = dict([
-            (
-                v.replace('APPVEYOR_', '').lower(),
-                os.getenv(v, '').decode('utf-8')
-            ) for v in [
-                'APPVEYOR_URL',
-                'APPVEYOR_REPO_COMMIT_MESSAGE_EXTENDED',
-                'APPVEYOR_REPO_BRANCH',
-                'APPVEYOR_REPO_COMMIT_AUTHOR',
-                'APPVEYOR_REPO_COMMIT_AUTHOR_EMAIL',
-                'APPVEYOR_REPO_COMMIT_TIMESTAMP',
-                'APPVEYOR_REPO_PROVIDER',
-                'APPVEYOR_PROJECT_NAME',
-                'APPVEYOR_PULL_REQUEST_TITLE',
-                'APPVEYOR_BUILD_VERSION',
-                'APPVEYOR_REPO_COMMIT',
-                'APPVEYOR_REPO_COMMIT_MESSAGE',
-                'APPVEYOR_ACCOUNT_NAME',
-                'APPVEYOR_PULL_REQUEST_NUMBER',
-                'APPVEYOR_REPO_NAME'
-            ]
-    ])
-
-    BUILD_FMT = u'{url}/project/{account_name}/{project_name}/build/{build_version}'
-
-    if vars["repo_provider"] == 'github':
-        COMMIT_FMT = u'https://{repo_provider}.com/{repo_name}/commit/{repo_commit}'
-        vars.update(commit_url=COMMIT_FMT.format(**vars))
-
-    vars.update(
-        build_url=BUILD_FMT.format(**vars),
-        short_commit=vars["repo_commit"][:7],
-    )
-    return vars
-
-
-def notify():
-    """
-    Send IRC notification
-    """
-    apvy_vars = appveyor_vars()
-
-    server, port = sys.argv[1].rsplit(":", 1)
-    channel = sys.argv[2]
-    success = sys.argv[3] == "success"
-    failure = sys.argv[3] == "failure"
-
-    if success or failure:
-        messages = []
-        messages.append(u"{repo_name} {repo_branch} {short_commit} - {repo_commit_author}: {repo_commit_message}")
-
-        if success:
-            m = u"Build #{build_version} passed. Details: {build_url}"
-        if failure:
-            m = u"Build #{build_version} failed. Details: {build_url}"
-
-        if "commit_url" in apvy_vars:
-            m += " Commit: {commit_url}"
-     
-        messages.append(m)
-    else:
-        messages = sys.argv[3:]
-        messages = ' '.join(messages)
-        messages = messages.decode("utf-8").split(',')
-
-    print(repr(apvy_vars))
-    messages = [msg.format(**apvy_vars).strip() for msg in messages]
-
-    irc_username = 'appveyor-ci'
-    irc_nick = irc_username
-
-    # establish connection
-    irc_sock = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM))
-    irc_sock.connect((socket.gethostbyname(server), int(port)))
-    irc_sock.send('NICK {0}\r\nUSER {0} * 0 :{0}\r\n'.format(irc_username).encode())
-    irc_sock.send('JOIN #{0}\r\n'.format(channel).encode())
-    irc_file = irc_sock.makefile()
-
-    while irc_file:
-        line = irc_file.readline()
-        print(line.rstrip())
-        response = line.split()
-
-        if response[0] == 'PING':
-            irc_file.send('PONG {}\r\n'.format(response[1]).encode())
-
-        elif response[1] == '433':
-            irc_sock.send('NICK {}\r\n'.format(irc_nick).encode())
-
-        elif response[1] == '001':
-            time.sleep(5)
-            # send notification
-            for msg in messages:
-                print(u'PRIVMSG #{} :{}'.format(channel, msg).encode("utf-8"))
-                irc_sock.send(u'PRIVMSG #{} :{}\r\n'.format(channel, msg).encode("utf-8"))
-            time.sleep(5)
-            return
-
-
-if __name__ == '__main__':
-    try:
-        notify()
-    except:
-        import traceback
-        print('ERROR: Failed to send notification: \n' + traceback.format_exc())

scripts/test/cov-diff

@@ -10,12 +10,12 @@ DIRB="$2"
 for B in $DIRB/*; do
   A=$DIRA/`basename $B`
   if [ -f $A ]; then
-      perl -pe 's/^\s*\!*\d+(\*?):/        1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp"
+      perl -pe 's/^\s*\!*\d+:/        1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp"
   else
       cat /dev/null > "$A.tmp"
   fi
-  perl -pe 's/^\s*\!*\d+(\*?):/        1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp"
-  diff -u "$A.tmp" "$B.tmp" |perl -pe 's/^((?:\+\+\+|---)(?:.*tmp))\s+.*/$1/;'
+  perl -pe 's/^\s*\!*\d+:/        1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp"
+  diff -u "$A.tmp" "$B.tmp"
   rm "$A.tmp" "$B.tmp"
 done
 

src/common/address.c

@@ -1697,7 +1697,7 @@ get_interface_address6_via_udp_socket_hack,(int severity,
                                             sa_family_t family,
                                             tor_addr_t *addr))
 {
-  struct sockaddr_storage target_addr;
+  struct sockaddr_storage my_addr, target_addr;
   int sock=-1, r=-1;
   socklen_t addr_len;
 
@@ -1740,19 +1740,21 @@ get_interface_address6_via_udp_socket_hack,(int severity,
     goto err;
   }
 
-  if (tor_addr_from_getsockname(addr, sock) < 0) {
+  if (tor_getsockname(sock,(struct sockaddr*)&my_addr, &addr_len)) {
     int e = tor_socket_errno(sock);
     log_fn(severity, LD_NET, "getsockname() to determine interface failed: %s",
            tor_socket_strerror(e));
     goto err;
   }
 
-  if (tor_addr_is_loopback(addr) || tor_addr_is_multicast(addr)) {
-    log_fn(severity, LD_NET, "Address that we determined via UDP socket"
-           " magic is unsuitable for public comms.");
-  } else {
-    r=0;
-  }
+ if (tor_addr_from_sockaddr(addr, (struct sockaddr*)&my_addr, NULL) == 0) {
+    if (tor_addr_is_loopback(addr) || tor_addr_is_multicast(addr)) {
+      log_fn(severity, LD_NET, "Address that we determined via UDP socket"
+                               " magic is unsuitable for public comms.");
+    } else {
+      r=0;
+    }
+ }
 
  err:
   if (sock >= 0)

src/common/address_set.c

@@ -15,7 +15,7 @@
 #include "address.h"
 #include "compat.h"
 #include "container.h"
-#include "crypto_rand.h"
+#include "crypto.h"
 #include "util.h"
 #include "siphash.h"
 

src/common/aes.c

@@ -16,8 +16,8 @@
   #include <ws2tcpip.h>
 #endif
 
-#include "compat_openssl.h"
 #include <openssl/opensslv.h>
+#include "crypto.h"
 #include "crypto_openssl_mgt.h"
 
 #if OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,0,0)
@@ -116,11 +116,7 @@ aes_cipher_free_(aes_cnt_cipher_t *cipher_)
   if (!cipher_)
     return;
   EVP_CIPHER_CTX *cipher = (EVP_CIPHER_CTX *) cipher_;
-#ifdef OPENSSL_1_1_API
-  EVP_CIPHER_CTX_reset(cipher);
-#else
   EVP_CIPHER_CTX_cleanup(cipher);
-#endif
   EVP_CIPHER_CTX_free(cipher);
 }
 void

src/common/buffers.h

@@ -12,6 +12,7 @@
 #ifndef TOR_BUFFERS_H
 #define TOR_BUFFERS_H
 
+#include "compat.h"
 #include "compat.h"
 #include "torint.h"
 #include "testsupport.h"

src/common/compat.c

@@ -100,6 +100,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt)
 /* Only use the linux prctl;  the IRIX prctl is totally different */
 #include <sys/prctl.h>
 #elif defined(__APPLE__)
+#include <sys/types.h>
 #include <sys/ptrace.h>
 #endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) || ... */
 
@@ -115,7 +116,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt)
 #ifdef HAVE_SIGNAL_H
 #include <signal.h>
 #endif
-#ifdef HAVE_MMAP
+#ifdef HAVE_SYS_MMAN_H
 #include <sys/mman.h>
 #endif
 #ifdef HAVE_SYS_SYSLIMITS_H
@@ -203,17 +204,25 @@ tor_rename(const char *path_old, const char *path_new)
                 sandbox_intern_string(path_new));
 }
 
-#if defined(HAVE_MMAP) || defined(RUNNING_DOXYGEN)
+/* Some MinGW builds have sys/mman.h, but not the corresponding symbols.
+ * Other configs rename the symbols using macros (including getpagesize).
+ * So check for sys/mman.h and unistd.h, and a getpagesize declaration. */
+#if (defined(HAVE_SYS_MMAN_H) && defined(HAVE_UNISTD_H) && \
+     defined(HAVE_DECL_GETPAGESIZE))
+#define COMPAT_HAS_MMAN_AND_PAGESIZE
+#endif
+
+#if defined(COMPAT_HAS_MMAN_AND_PAGESIZE) || \
+  defined(RUNNING_DOXYGEN)
 /** Try to create a memory mapping for <b>filename</b> and return it.  On
- * failure, return NULL. Sets errno properly, using ERANGE to mean
- * "empty file". Must only be called on trusted Tor-owned files, as changing
- * the underlying file's size causes unspecified behavior. */
+ * failure, return NULL.  Sets errno properly, using ERANGE to mean
+ * "empty file". */
 tor_mmap_t *
 tor_mmap_file(const char *filename)
 {
   int fd; /* router file */
   char *string;
-  int result;
+  int page_size, result;
   tor_mmap_t *res;
   size_t size, filesize;
   struct stat st;
@@ -242,6 +251,13 @@ tor_mmap_file(const char *filename)
     return NULL;
   }
   size = filesize = (size_t)(st.st_size);
+  /*
+   * Should we check for weird crap like mmapping a named pipe here,
+   * or just wait for if (!size) below to fail?
+   */
+  /* ensure page alignment */
+  page_size = getpagesize();
+  size += (size%page_size) ? page_size-(size%page_size) : 0;
 
   if (st.st_size > SSIZE_T_CEILING || (off_t)size < st.st_size) {
     log_warn(LD_FS, "File \"%s\" is too large. Ignoring.",filename);
@@ -402,8 +418,40 @@ tor_munmap_file(tor_mmap_t *handle)
   return 0;
 }
 #else
-#error "cannot implement tor_mmap_file"
-#endif /* defined(HAVE_MMAP) || ... || ... */
+tor_mmap_t *
+tor_mmap_file(const char *filename)
+{
+  struct stat st;
+  char *res = read_file_to_str(filename, RFTS_BIN|RFTS_IGNORE_MISSING, &st);
+  tor_mmap_t *handle;
+  if (! res)
+    return NULL;
+  handle = tor_malloc_zero(sizeof(tor_mmap_t));
+  handle->data = res;
+  handle->size = st.st_size;
+  return handle;
+}
+
+/** Unmap the file mapped with tor_mmap_file(), and return 0 for success
+ * or -1 for failure.
+ */
+
+int
+tor_munmap_file(tor_mmap_t *handle)
+{
+  char *d = NULL;
+  if (handle == NULL)
+    return 0;
+
+  d = (char*)handle->data;
+  tor_free(d);
+  memwipe(handle, 0, sizeof(tor_mmap_t));
+  tor_free(handle);
+
+  /* Can't fail in this mmap()/munmap()-free case */
+  return 0;
+}
+#endif /* defined(COMPAT_HAS_MMAN_AND_PAGESIZE) || ... || ... */
 
 /** Replacement for snprintf.  Differs from platform snprintf in two
  * ways: First, always NUL-terminates its output.  Second, always
@@ -1344,24 +1392,6 @@ tor_getsockname,(tor_socket_t sock, struct sockaddr *address,
    return getsockname(sock, address, address_len);
 }
 
-/**
- * Find the local address associated with the socket <b>sock</b>, and
- * place it in *<b>addr_out</b>.  Return 0 on success, -1 on failure.
- *
- * (As tor_getsockname, but instead places the result in a tor_addr_t.) */
-int
-tor_addr_from_getsockname(tor_addr_t *addr_out, tor_socket_t sock)
-{
-  struct sockaddr_storage ss;
-  socklen_t ss_len = sizeof(ss);
-  memset(&ss, 0, sizeof(ss));
-
-  if (tor_getsockname(sock, (struct sockaddr *) &ss, &ss_len) < 0)
-    return -1;
-
-  return tor_addr_from_sockaddr(addr_out, (struct sockaddr *)&ss, NULL);
-}
-
 /** Turn <b>socket</b> into a nonblocking socket. Return 0 on success, -1
  * on failure.
  */

src/common/compat.h

@@ -318,12 +318,12 @@ typedef struct tor_mmap_t {
   size_t size; /**< Size of the file. */
 
   /* None of the fields below should be accessed from outside compat.c */
-#ifdef HAVE_MMAP
+#ifdef HAVE_SYS_MMAN_H
   size_t mapping_size; /**< Size of the actual mapping. (This is this file
                         * size, rounded up to the nearest page.) */
 #elif defined _WIN32
   HANDLE mmap_handle;
-#endif /* defined(HAVE_MMAP) || ... */
+#endif /* defined(HAVE_SYS_MMAN_H) || ... */
 
 } tor_mmap_t;
 
@@ -510,8 +510,6 @@ int get_n_open_sockets(void);
 MOCK_DECL(int,
 tor_getsockname,(tor_socket_t socket, struct sockaddr *address,
                  socklen_t *address_len));
-struct tor_addr_t;
-int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock);
 
 #define tor_socket_send(s, buf, len, flags) send(s, buf, len, flags)
 #define tor_socket_recv(s, buf, len, flags) recv(s, buf, len, flags)

src/common/compat_libevent.c

@@ -11,7 +11,7 @@
 #define COMPAT_LIBEVENT_PRIVATE
 #include "compat_libevent.h"
 
-#include "crypto_rand.h"
+#include "crypto.h"
 
 #include "util.h"
 #include "torlog.h"
@@ -79,43 +79,6 @@ tor_event_free_(struct event *ev)
 /** Global event base for use by the main thread. */
 static struct event_base *the_event_base = NULL;
 
-/**
- * @defgroup postloop post-loop event helpers
- *
- * If we're not careful, Libevent can susceptible to infinite event chains:
- * one event can activate another, whose callback activates another, whose
- * callback activates another, ad infinitum.  While this is happening,
- * Libevent won't be checking timeouts, socket-based events, signals, and so
- * on.
- *
- * We solve this problem by marking some events as "post-loop".  A post-loop
- * event behaves like any ordinary event, but any events that _it_ activates
- * cannot run until Libevent has checked for other events at least once.
- *
- * @{ */
-
-/**
- * An event that stops Libevent from running any more events on the current
- * iteration of its loop, until it has re-checked for socket events, signal
- * events, timeouts, etc.
- */
-static struct event *rescan_mainloop_ev = NULL;
-
-/**
- * Callback to implement rescan_mainloop_ev: it simply exits the mainloop,
- * and relies on Tor to re-enter the mainloop since no error has occurred.
- */
-static void
-rescan_mainloop_cb(evutil_socket_t fd, short events, void *arg)
-{
-  (void)fd;
-  (void)events;
-  struct event_base *the_base = arg;
-  event_base_loopbreak(the_base);
-}
-
-/** @} */
-
 /* This is what passes for version detection on OSX.  We set
  * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before
  * 10.4.0 (aka 1040). */
@@ -167,15 +130,6 @@ tor_libevent_initialize(tor_libevent_cfg *torcfg)
     /* LCOV_EXCL_STOP */
   }
 
-  rescan_mainloop_ev = event_new(the_event_base, -1, 0,
-                                 rescan_mainloop_cb, the_event_base);
-  if (!rescan_mainloop_ev) {
-    /* LCOV_EXCL_START */
-    log_err(LD_GENERAL, "Unable to create rescan event: cannot continue.");
-    exit(1); // exit ok: libevent is broken.
-    /* LCOV_EXCL_STOP */
-  }
-
   log_info(LD_GENERAL,
       "Initialized libevent version %s using method %s. Good.",
       event_get_version(), tor_libevent_get_method());
@@ -253,39 +207,10 @@ periodic_timer_new(struct event_base *base,
   }
   timer->cb = cb;
   timer->data = data;
-  periodic_timer_launch(timer, tv);
+  event_add(timer->ev, (struct timeval *)tv); /*drop const for old libevent*/
   return timer;
 }
 
-/**
- * Launch the timer <b>timer</b> to run at <b>tv</b> from now, and every
- * <b>tv</b> thereafter.
- *
- * If the timer is already enabled, this function does nothing.
- */
-void
-periodic_timer_launch(periodic_timer_t *timer, const struct timeval *tv)
-{
-  tor_assert(timer);
-  if (event_pending(timer->ev, EV_TIMEOUT, NULL))
-    return;
-  event_add(timer->ev, tv);
-}
-
-/**
- * Disable the provided <b>timer</b>, but do not free it.
- *
- * You can reenable the same timer later with periodic_timer_launch.
- *
- * If the timer is already disabled, this function does nothing.
- */
-void
-periodic_timer_disable(periodic_timer_t *timer)
-{
-  tor_assert(timer);
-  (void) event_del(timer->ev);
-}
-
 /** Stop and free a periodic timer */
 void
 periodic_timer_free_(periodic_timer_t *timer)
@@ -296,173 +221,6 @@ periodic_timer_free_(periodic_timer_t *timer)
   tor_free(timer);
 }
 
-/**
- * Type used to represent events that run directly from the main loop,
- * either because they are activated from elsewhere in the code, or
- * because they have a simple timeout.
- *
- * We use this type to avoid exposing Libevent's API throughout the rest
- * of the codebase.
- *
- * This type can't be used for all events: it doesn't handle events that
- * are triggered by signals or by sockets.
- */
-struct mainloop_event_t {
-  struct event *ev;
-  void (*cb)(mainloop_event_t *, void *);
-  void *userdata;
-};
-
-/**
- * Internal: Implements mainloop event using a libevent event.
- */
-static void
-mainloop_event_cb(evutil_socket_t fd, short what, void *arg)
-{
-  (void)fd;
-  (void)what;
-  mainloop_event_t *mev = arg;
-  mev->cb(mev, mev->userdata);
-}
-
-/**
- * As mainloop_event_cb, but implements a post-loop event.
- */
-static void
-mainloop_event_postloop_cb(evutil_socket_t fd, short what, void *arg)
-{
-  (void)fd;
-  (void)what;
-
-  /* Note that if rescan_mainloop_ev is already activated,
-   * event_active() will do nothing: only the first post-loop event that
-   * happens each time through the event loop will cause it to be
-   * activated.
-   *
-   * Because event_active() puts events on a FIFO queue, every event
-   * that is made active _after_ rescan_mainloop_ev will get its
-   * callback run after rescan_mainloop_cb is called -- that is, on the
-   * next iteration of the loop.
-   */
-  event_active(rescan_mainloop_ev, EV_READ, 1);
-
-  mainloop_event_t *mev = arg;
-  mev->cb(mev, mev->userdata);
-}
-
-/**
- * Helper for mainloop_event_new() and mainloop_event_postloop_new().
- */
-static mainloop_event_t *
-mainloop_event_new_impl(int postloop,
-                        void (*cb)(mainloop_event_t *, void *),
-                        void *userdata)
-{
-  tor_assert(cb);
-
-  struct event_base *base = tor_libevent_get_base();
-  mainloop_event_t *mev = tor_malloc_zero(sizeof(mainloop_event_t));
-  mev->ev = tor_event_new(base, -1, 0,
-                  postloop ? mainloop_event_postloop_cb : mainloop_event_cb,
-                  mev);
-  tor_assert(mev->ev);
-  mev->cb = cb;
-  mev->userdata = userdata;
-  return mev;
-}
-
-/**
- * Create and return a new mainloop_event_t to run the function <b>cb</b>.
- *
- * When run, the callback function will be passed the mainloop_event_t
- * and <b>userdata</b> as its arguments.  The <b>userdata</b> pointer
- * must remain valid for as long as the mainloop_event_t event exists:
- * it is your responsibility to free it.
- *
- * The event is not scheduled by default: Use mainloop_event_activate()
- * or mainloop_event_schedule() to make it run.
- */
-mainloop_event_t *
-mainloop_event_new(void (*cb)(mainloop_event_t *, void *),
-                   void *userdata)
-{
-  return mainloop_event_new_impl(0, cb, userdata);
-}
-
-/**
- * As mainloop_event_new(), but create a post-loop event.
- *
- * A post-loop event behaves like any ordinary event, but any events
- * that _it_ activates cannot run until Libevent has checked for other
- * events at least once.
- */
-mainloop_event_t *
-mainloop_event_postloop_new(void (*cb)(mainloop_event_t *, void *),
-                            void *userdata)
-{
-  return mainloop_event_new_impl(1, cb, userdata);
-}
-
-/**
- * Schedule <b>event</b> to run in the main loop, immediately.  If it is
- * not scheduled, it will run anyway. If it is already scheduled to run
- * later, it will run now instead.  This function will have no effect if
- * the event is already scheduled to run.
- *
- * This function may only be called from the main thread.
- */
-void
-mainloop_event_activate(mainloop_event_t *event)
-{
-  tor_assert(event);
-  event_active(event->ev, EV_READ, 1);
-}
-
-/** Schedule <b>event</b> to run in the main loop, after a delay of <b>tv</b>.
- *
- * If the event is scheduled for a different time, cancel it and run
- * after this delay instead.  If the event is currently pending to run
- * <em>now</b>, has no effect.
- *
- * Do not call this function with <b>tv</b> == NULL -- use
- * mainloop_event_activate() instead.
- *
- * This function may only be called from the main thread.
- */
-int
-mainloop_event_schedule(mainloop_event_t *event, const struct timeval *tv)
-{
-  tor_assert(event);
-  if (BUG(tv == NULL)) {
-    // LCOV_EXCL_START
-    mainloop_event_activate(event);
-    return 0;
-    // LCOV_EXCL_STOP
-  }
-  return event_add(event->ev, tv);
-}
-
-/** Cancel <b>event</b> if it is currently active or pending. (Do nothing if
- * the event is not currently active or pending.) */
-void
-mainloop_event_cancel(mainloop_event_t *event)
-{
-  if (!event)
-    return;
-  (void) event_del(event->ev);
-}
-
-/** Cancel <b>event</b> and release all storage associated with it. */
-void
-mainloop_event_free_(mainloop_event_t *event)
-{
-  if (!event)
-    return;
-  tor_event_free(event->ev);
-  memset(event, 0xb8, sizeof(*event));
-  tor_free(event);
-}
-
 int
 tor_init_libevent_rng(void)
 {
@@ -485,45 +243,56 @@ tor_init_libevent_rng(void)
 void
 tor_libevent_free_all(void)
 {
-  tor_event_free(rescan_mainloop_ev);
   if (the_event_base)
     event_base_free(the_event_base);
   the_event_base = NULL;
 }
 
-/**
- * Run the event loop for the provided event_base, handling events until
- * something stops it.  If <b>once</b> is set, then just poll-and-run
- * once, then exit.  Return 0 on success, -1 if an error occurred, or 1
- * if we exited because no events were pending or active.
- *
- * This isn't reentrant or multithreaded.
- */
-int
-tor_libevent_run_event_loop(struct event_base *base, int once)
-{
-  const int flags = once ? EVLOOP_ONCE : 0;
-  return event_base_loop(base, flags);
-}
-
-/** Tell the event loop to exit after <b>delay</b>.  If <b>delay</b> is NULL,
- * instead exit after we're done running the currently active events. */
+#if defined(LIBEVENT_VERSION_NUMBER) &&         \
+  LIBEVENT_VERSION_NUMBER >= V(2,1,1) &&        \
+  !defined(TOR_UNIT_TESTS)
 void
-tor_libevent_exit_loop_after_delay(struct event_base *base,
-                                   const struct timeval *delay)
+tor_gettimeofday_cached(struct timeval *tv)
 {
-  event_base_loopexit(base, delay);
+  event_base_gettimeofday_cached(the_event_base, tv);
 }
-
-/** Tell the event loop to exit after running whichever callback is currently
- * active. */
 void
-tor_libevent_exit_loop_after_callback(struct event_base *base)
+tor_gettimeofday_cache_clear(void)
 {
-  event_base_loopbreak(base);
+  event_base_update_cache_time(the_event_base);
+}
+#else /* !(defined(LIBEVENT_VERSION_NUMBER) &&         ...) */
+/** Cache the current hi-res time; the cache gets reset when libevent
+ * calls us. */
+static struct timeval cached_time_hires = {0, 0};
+
+/** Return a fairly recent view of the current time. */
+void
+tor_gettimeofday_cached(struct timeval *tv)
+{
+  if (cached_time_hires.tv_sec == 0) {
+    tor_gettimeofday(&cached_time_hires);
+  }
+  *tv = cached_time_hires;
+}
+
+/** Reset the cached view of the current time, so that the next time we try
+ * to learn it, we will get an up-to-date value. */
+void
+tor_gettimeofday_cache_clear(void)
+{
+  cached_time_hires.tv_sec = 0;
+}
+
+#ifdef TOR_UNIT_TESTS
+/** For testing: force-update the cached time to a given value. */
+void
+tor_gettimeofday_cache_set(const struct timeval *tv)
+{
+  tor_assert(tv);
+  memcpy(&cached_time_hires, tv, sizeof(*tv));
 }
 
-#if defined(TOR_UNIT_TESTS)
 /** For testing: called post-fork to make libevent reinitialize
  * kernel structures. */
 void
@@ -533,4 +302,5 @@ tor_libevent_postfork(void)
   tor_assert(r == 0);
 }
 #endif /* defined(TOR_UNIT_TESTS) */
+#endif /* defined(LIBEVENT_VERSION_NUMBER) &&         ... */
 

src/common/compat_libevent.h

@@ -7,6 +7,8 @@
 #include "orconfig.h"
 #include "testsupport.h"
 
+#include <event2/event.h>
+
 void configure_libevent_logging(void);
 void suppress_libevent_log_msg(const char *msg);
 
@@ -17,9 +19,6 @@ void suppress_libevent_log_msg(const char *msg);
   evdns_add_server_port_with_base(tor_libevent_get_base(), \
   (sock),(tcp),(cb),(data));
 
-struct event;
-struct event_base;
-
 void tor_event_free_(struct event *ev);
 #define tor_event_free(ev) \
   FREE_AND_NULL(struct event, tor_event_free_, (ev))
@@ -31,24 +30,11 @@ periodic_timer_t *periodic_timer_new(struct event_base *base,
              void (*cb)(periodic_timer_t *timer, void *data),
              void *data);
 void periodic_timer_free_(periodic_timer_t *);
-void periodic_timer_launch(periodic_timer_t *, const struct timeval *tv);
-void periodic_timer_disable(periodic_timer_t *);
 #define periodic_timer_free(t) \
   FREE_AND_NULL(periodic_timer_t, periodic_timer_free_, (t))
 
-typedef struct mainloop_event_t mainloop_event_t;
-mainloop_event_t *mainloop_event_new(void (*cb)(mainloop_event_t *, void *),
-                                     void *userdata);
-mainloop_event_t * mainloop_event_postloop_new(
-                                     void (*cb)(mainloop_event_t *, void *),
-                                     void *userdata);
-void mainloop_event_activate(mainloop_event_t *event);
-int mainloop_event_schedule(mainloop_event_t *event,
-                            const struct timeval *delay);
-void mainloop_event_cancel(mainloop_event_t *event);
-void mainloop_event_free_(mainloop_event_t *event);
-#define mainloop_event_free(event) \
-  FREE_AND_NULL(mainloop_event_t, mainloop_event_free_, (event))
+#define tor_event_base_loopexit event_base_loopexit
+#define tor_event_base_loopbreak event_base_loopbreak
 
 /** Defines a configuration for using libevent with Tor: passed as an argument
  * to tor_libevent_initialize() to describe how we want to set up. */
@@ -70,15 +56,13 @@ void tor_libevent_free_all(void);
 
 int tor_init_libevent_rng(void);
 
+void tor_gettimeofday_cached(struct timeval *tv);
+void tor_gettimeofday_cache_clear(void);
 #ifdef TOR_UNIT_TESTS
+void tor_gettimeofday_cache_set(const struct timeval *tv);
 void tor_libevent_postfork(void);
 #endif
 
-int tor_libevent_run_event_loop(struct event_base *base, int once);
-void tor_libevent_exit_loop_after_delay(struct event_base *base,
-                                        const struct timeval *delay);
-void tor_libevent_exit_loop_after_callback(struct event_base *base);
-
 #ifdef COMPAT_LIBEVENT_PRIVATE
 
 /** Macro: returns the number of a Libevent version as a 4-byte number,

src/common/compat_time.c

@@ -71,8 +71,8 @@ tor_sleep_msec(int msec)
 /** Set *timeval to the current time of day.  On error, log and terminate.
  * (Same as gettimeofday(timeval,NULL), but never returns -1.)
  */
-MOCK_IMPL(void,
-tor_gettimeofday, (struct timeval *timeval))
+void
+tor_gettimeofday(struct timeval *timeval)
 {
 #ifdef _WIN32
   /* Epoch bias copied from perl: number of units between windows epoch and
@@ -279,7 +279,6 @@ monotime_reset_ratchets_for_testing(void)
  * nanoseconds.
  */
 static struct mach_timebase_info mach_time_info;
-static struct mach_timebase_info mach_time_info_msec_cvt;
 static int monotime_shift = 0;
 
 static void
@@ -297,14 +296,6 @@ monotime_init_internal(void)
     // requires that tor_log2(0) == 0.
     monotime_shift = tor_log2(ms_per_tick);
   }
-  {
-    // For converting ticks to milliseconds in a 32-bit-friendly way, we
-    // will first right-shift by 20, and then multiply by 20/19, since
-    // (1<<20) * 19/20 is about 1e6.  We precompute a new numerate and
-    // denominator here to avoid multiple multiplies.
-    mach_time_info_msec_cvt.numer = mach_time_info.numer * 20;
-    mach_time_info_msec_cvt.denom = mach_time_info.denom * 19;
-  }
 }
 
 /**
@@ -354,22 +345,6 @@ monotime_diff_nsec(const monotime_t *start,
   return diff_nsec;
 }
 
-int32_t
-monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
-                             const monotime_coarse_t *end)
-{
-  if (BUG(mach_time_info.denom == 0)) {
-    monotime_init();
-  }
-  const int64_t diff_ticks = end->abstime_ - start->abstime_;
-
-  /* We already require in di_ops.c that right-shift performs a sign-extend. */
-  const int32_t diff_microticks = (int32_t)(diff_ticks >> 20);
-
-  return (diff_microticks * mach_time_info_msec_cvt.numer) /
-    mach_time_info_msec_cvt.denom;
-}
-
 uint32_t
 monotime_coarse_to_stamp(const monotime_coarse_t *t)
 {
@@ -468,15 +443,6 @@ monotime_diff_nsec(const monotime_t *start,
   return diff_nsec;
 }
 
-int32_t
-monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
-                             const monotime_coarse_t *end)
-{
-  const int32_t diff_sec = (int32_t)(end->ts_.tv_sec - start->ts_.tv_sec);
-  const int32_t diff_nsec = (int32_t)(end->ts_.tv_nsec - start->ts_.tv_nsec);
-  return diff_sec * 1000 + diff_nsec / ONE_MILLION;
-}
-
 /* This value is ONE_BILLION >> 20. */
 static const uint32_t STAMP_TICKS_PER_SECOND = 953;
 
@@ -626,13 +592,6 @@ monotime_coarse_diff_msec(const monotime_coarse_t *start,
   return diff_ticks;
 }
 
-int32_t
-monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
-                             const monotime_coarse_t *end)
-{
-  return (int32_t)monotime_coarse_diff_msec(start, end);
-}
-
 int64_t
 monotime_coarse_diff_usec(const monotime_coarse_t *start,
                           const monotime_coarse_t *end)
@@ -718,15 +677,6 @@ monotime_diff_nsec(const monotime_t *start,
   return (diff.tv_sec * ONE_BILLION + diff.tv_usec * 1000);
 }
 
-int32_t
-monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
-                             const monotime_coarse_t *end)
-{
-  struct timeval diff;
-  timersub(&end->tv_, &start->tv_, &diff);
-  return diff.tv_sec * 1000 + diff.tv_usec / 1000;
-}
-
 /* This value is ONE_MILLION >> 10. */
 static const uint32_t STAMP_TICKS_PER_SECOND = 976;
 
@@ -880,24 +830,11 @@ monotime_coarse_stamp_units_to_approx_msec(uint64_t units)
   return (abstime_diff * mach_time_info.numer) /
     (mach_time_info.denom * ONE_MILLION);
 }
-uint64_t
-monotime_msec_to_approx_coarse_stamp_units(uint64_t msec)
-{
-  uint64_t abstime_val =
-    (((uint64_t)msec) * ONE_MILLION * mach_time_info.denom) /
-    mach_time_info.numer;
-  return abstime_val >> monotime_shift;
-}
 #else
 uint64_t
 monotime_coarse_stamp_units_to_approx_msec(uint64_t units)
 {
   return (units * 1000) / STAMP_TICKS_PER_SECOND;
 }
-uint64_t
-monotime_msec_to_approx_coarse_stamp_units(uint64_t msec)
-{
-  return (msec * STAMP_TICKS_PER_SECOND) / 1000;
-}
 #endif
 

src/common/compat_time.h

@@ -150,7 +150,6 @@ uint32_t monotime_coarse_to_stamp(const monotime_coarse_t *t);
  * into an approximate number of milliseconds.
  */
 uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units);
-uint64_t monotime_msec_to_approx_coarse_stamp_units(uint64_t msec);
 uint32_t monotime_coarse_get_stamp(void);
 
 #if defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)
@@ -173,34 +172,7 @@ void monotime_coarse_add_msec(monotime_coarse_t *out,
 #define monotime_coarse_add_msec monotime_add_msec
 #endif /* defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */
 
-/**
- * As monotime_coarse_diff_msec, but avoid 64-bit division.
- *
- * Requires that the difference fit into an int32_t; not for use with
- * large time differences.
- */
-int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
-                                     const monotime_coarse_t *end);
-
-/**
- * As monotime_coarse_diff_msec, but avoid 64-bit division if it is expensive.
- *
- * Requires that the difference fit into an int32_t; not for use with
- * large time differences.
- */
-static inline int32_t
-monotime_coarse_diff_msec32(const monotime_coarse_t *start,
-                            const monotime_coarse_t *end)
-{
-#if SIZEOF_VOID_P == 8
-  // on a 64-bit platform, let's assume 64/64 division is cheap.
-  return (int32_t) monotime_coarse_diff_msec(start, end);
-#else
-  return monotime_coarse_diff_msec32_(start, end);
-#endif
-}
-
-MOCK_DECL(void, tor_gettimeofday, (struct timeval *timeval));
+void tor_gettimeofday(struct timeval *timeval);
 
 #ifdef TOR_UNIT_TESTS
 void tor_sleep_msec(int msec);

src/common/compat_winthreads.c

@@ -18,6 +18,7 @@
 #include "util.h"
 #include "container.h"
 #include "torlog.h"
+#include <process.h>
 
 /* This value is more or less total cargo-cult */
 #define SPIN_COUNT 2000

src/common/compress.c

@@ -663,13 +663,3 @@ tor_compress_init(void)
   tor_zstd_init();
 }
 
-/** Warn if we had any problems while setting up our compression libraries.
- *
- * (This isn't part of tor_compress_init, since the logs aren't set up yet.)
- */
-void
-tor_compress_log_init_warnings(void)
-{
-  tor_zstd_warn_if_version_mismatched();
-}
-

src/common/compress.h

@@ -87,7 +87,6 @@ void tor_compress_free_(tor_compress_state_t *state);
 size_t tor_compress_state_size(const tor_compress_state_t *state);
 
 void tor_compress_init(void);
-void tor_compress_log_init_warnings(void);
 
 #endif /* !defined(TOR_COMPRESS_H) */
 

src/common/compress_zstd.c

@@ -18,13 +18,6 @@
 #include "compress.h"
 #include "compress_zstd.h"
 
-#ifdef ENABLE_ZSTD_ADVANCED_APIS
-/* This is a lie, but we make sure it doesn't get us in trouble by wrapping
- * all invocations of zstd's static-only functions in a check to make sure
- * that the compile-time version matches the run-time version. */
-#define ZSTD_STATIC_LINKING_ONLY
-#endif
-
 #ifdef HAVE_ZSTD
 #include <zstd.h>
 #endif
@@ -58,31 +51,21 @@ tor_zstd_method_supported(void)
 #endif
 }
 
-#ifdef HAVE_ZSTD
-/** Format a zstd version number as a string in <b>buf</b>. */
-static void
-tor_zstd_format_version(char *buf, size_t buflen, unsigned version_number)
-{
-  tor_snprintf(buf, buflen,
-               "%u.%u.%u",
-               version_number / 10000 % 100,
-               version_number / 100 % 100,
-               version_number % 100);
-}
-#endif
-
-#define VERSION_STR_MAX_LEN 16 /* more than enough space for 99.99.99 */
-
 /** Return a string representation of the version of the currently running
  * version of libzstd. Returns NULL if Zstandard is unsupported. */
 const char *
 tor_zstd_get_version_str(void)
 {
 #ifdef HAVE_ZSTD
-  static char version_str[VERSION_STR_MAX_LEN];
+  static char version_str[16];
+  size_t version_number;
 
-  tor_zstd_format_version(version_str, sizeof(version_str),
-                          ZSTD_versionNumber());
+  version_number = ZSTD_versionNumber();
+  tor_snprintf(version_str, sizeof(version_str),
+               "%d.%d.%d",
+               (int) version_number / 10000 % 100,
+               (int) version_number / 100 % 100,
+               (int) version_number % 100);
 
   return version_str;
 #else /* !(defined(HAVE_ZSTD)) */
@@ -102,26 +85,6 @@ tor_zstd_get_header_version_str(void)
 #endif
 }
 
-#ifdef TOR_UNIT_TESTS
-static int static_apis_disable_for_testing = 0;
-#endif
-
-/** Return true iff we can use the "static-only" APIs. */
-int
-tor_zstd_can_use_static_apis(void)
-{
-#if defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD)
-#ifdef TOR_UNIT_TESTS
-  if (static_apis_disable_for_testing) {
-    return 0;
-  }
-#endif
-  return (ZSTD_VERSION_NUMBER == ZSTD_versionNumber());
-#else
-  return 0;
-#endif
-}
-
 /** Internal Zstandard state for incremental compression/decompression.
  * The body of this struct is not exposed. */
 struct tor_zstd_compress_state_t {
@@ -149,11 +112,9 @@ struct tor_zstd_compress_state_t {
 
 #ifdef HAVE_ZSTD
 /** Return an approximate number of bytes stored in memory to hold the
- * Zstandard compression/decompression state. This is a fake estimate
- * based on inspecting the zstd source: tor_zstd_state_size_precalc() is
- * more accurate when it's allowed to use "static-only" functions */
+ * Zstandard compression/decompression state. */
 static size_t
-tor_zstd_state_size_precalc_fake(int compress, int preset)
+tor_zstd_state_size_precalc(int compress, int preset)
 {
   tor_assert(preset > 0);
 
@@ -210,28 +171,6 @@ tor_zstd_state_size_precalc_fake(int compress, int preset)
 
   return memory_usage;
 }
-
-/** Return an approximate number of bytes stored in memory to hold the
- * Zstandard compression/decompression state. */
-static size_t
-tor_zstd_state_size_precalc(int compress, int preset)
-{
-#ifdef ZSTD_STATIC_LINKING_ONLY
-  if (tor_zstd_can_use_static_apis()) {
-    if (compress) {
-#ifdef HAVE_ZSTD_ESTIMATECSTREAMSIZE
-      return ZSTD_estimateCStreamSize(preset);
-#endif
-    } else {
-#ifdef HAVE_ZSTD_ESTIMATEDCTXSIZE
-      /* Could use DStream, but that takes a windowSize. */
-      return ZSTD_estimateDCtxSize();
-#endif
-    }
-  }
-#endif
-  return tor_zstd_state_size_precalc_fake(compress, preset);
-}
 #endif /* defined(HAVE_ZSTD) */
 
 /** Construct and return a tor_zstd_compress_state_t object using
@@ -501,34 +440,3 @@ tor_zstd_init(void)
   atomic_counter_init(&total_zstd_allocation);
 }
 
-/** Warn if the header and library versions don't match. */
-void
-tor_zstd_warn_if_version_mismatched(void)
-{
-#if defined(HAVE_ZSTD) && defined(ENABLE_ZSTD_ADVANCED_APIS)
-  if (! tor_zstd_can_use_static_apis()) {
-    char header_version[VERSION_STR_MAX_LEN];
-    char runtime_version[VERSION_STR_MAX_LEN];
-    tor_zstd_format_version(header_version, sizeof(header_version),
-                            ZSTD_VERSION_NUMBER);
-    tor_zstd_format_version(runtime_version, sizeof(runtime_version),
-                            ZSTD_versionNumber());
-
-    log_warn(LD_GENERAL,
-             "Tor was compiled with zstd %s, but is running with zstd %s. "
-             "For safety, we'll avoid using advanced zstd functionality.",
-             header_version, runtime_version);
-  }
-#endif
-}
-
-#ifdef TOR_UNIT_TESTS
-/** Testing only: disable usage of static-only APIs, so we can make sure that
- * we still work without them. */
-void
-tor_zstd_set_static_apis_disabled_for_testing(int disabled)
-{
-  static_apis_disable_for_testing = disabled;
-}
-#endif
-

src/common/compress_zstd.h

@@ -17,8 +17,6 @@ const char *tor_zstd_get_version_str(void);
 
 const char *tor_zstd_get_header_version_str(void);
 
-int tor_zstd_can_use_static_apis(void);
-
 /** Internal state for an incremental Zstandard compression/decompression. */
 typedef struct tor_zstd_compress_state_t tor_zstd_compress_state_t;
 
@@ -43,11 +41,6 @@ size_t tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state);
 size_t tor_zstd_get_total_allocation(void);
 
 void tor_zstd_init(void);
-void tor_zstd_warn_if_version_mismatched(void);
-
-#ifdef TOR_UNIT_TESTS
-void tor_zstd_set_static_apis_disabled_for_testing(int disabled);
-#endif
 
 #endif /* !defined(TOR_COMPRESS_ZSTD_H) */
 

src/common/container.c

@@ -15,7 +15,7 @@
 #include "util.h"
 #include "torlog.h"
 #include "container.h"
-#include "crypto_digest.h"
+#include "crypto.h"
 
 #include <stdlib.h>
 #include <string.h>

src/common/crypto.h

@@ -17,10 +17,20 @@
 
 #include <stdio.h>
 #include "torint.h"
+#include "testsupport.h"
 #include "compat.h"
 #include "util.h"
 #include "crypto_rsa.h"
 
+#include "keccak-tiny/keccak-tiny.h"
+
+/** Length of the output of our message digest. */
+#define DIGEST_LEN 20
+/** Length of the output of our second (improved) message digests.  (For now
+ * this is just sha256, but it could be any other 256-bit digest.) */
+#define DIGEST256_LEN 32
+/** Length of the output of our 64-bit optimized message digests (SHA512). */
+#define DIGEST512_LEN 64
 /** Length of our symmetric cipher's keys of 128-bit. */
 #define CIPHER_KEY_LEN 16
 /** Length of our symmetric cipher's IV of 128-bit. */
@@ -30,15 +40,57 @@
 /** Length of our DH keys. */
 #define DH_BYTES (1024/8)
 
+/** Length of a sha1 message digest when encoded in base32 with trailing =
+ * signs removed. */
+#define BASE32_DIGEST_LEN 32
+/** Length of a sha1 message digest when encoded in base64 with trailing =
+ * signs removed. */
+#define BASE64_DIGEST_LEN 27
+/** Length of a sha256 message digest when encoded in base64 with trailing =
+ * signs removed. */
+#define BASE64_DIGEST256_LEN 43
+/** Length of a sha512 message digest when encoded in base64 with trailing =
+ * signs removed. */
+#define BASE64_DIGEST512_LEN 86
+
 /** Length of encoded public key fingerprints, including space; but not
  * including terminating NUL. */
 #define FINGERPRINT_LEN 49
+/** Length of hex encoding of SHA1 digest, not including final NUL. */
+#define HEX_DIGEST_LEN 40
+/** Length of hex encoding of SHA256 digest, not including final NUL. */
+#define HEX_DIGEST256_LEN 64
+/** Length of hex encoding of SHA512 digest, not including final NUL. */
+#define HEX_DIGEST512_LEN 128
+
+typedef enum {
+  DIGEST_SHA1 = 0,
+  DIGEST_SHA256 = 1,
+  DIGEST_SHA512 = 2,
+  DIGEST_SHA3_256 = 3,
+  DIGEST_SHA3_512 = 4,
+} digest_algorithm_t;
+#define  N_DIGEST_ALGORITHMS (DIGEST_SHA3_512+1)
+#define  N_COMMON_DIGEST_ALGORITHMS (DIGEST_SHA256+1)
+
+/** A set of all the digests we commonly compute, taken on a single
+ * string.  Any digests that are shorter than 512 bits are right-padded
+ * with 0 bits.
+ *
+ * Note that this representation wastes 44 bytes for the SHA1 case, so
+ * don't use it for anything where we need to allocate a whole bunch at
+ * once.
+ **/
+typedef struct {
+  char d[N_COMMON_DIGEST_ALGORITHMS][DIGEST256_LEN];
+} common_digests_t;
 
 typedef struct aes_cnt_cipher crypto_cipher_t;
+typedef struct crypto_digest_t crypto_digest_t;
+typedef struct crypto_xof_t crypto_xof_t;
 typedef struct crypto_dh_t crypto_dh_t;
 
 /* global state */
-int crypto_init_siphash_key(void);
 int crypto_early_init(void) ATTR_WUR;
 int crypto_global_init(int hardwareAccel,
                        const char *accelName,
@@ -62,6 +114,24 @@ void crypto_cipher_free_(crypto_cipher_t *env);
 #define crypto_cipher_free(c) \
   FREE_AND_NULL(crypto_cipher_t, crypto_cipher_free_, (c))
 
+/* public key crypto  */
+MOCK_DECL(int, crypto_pk_public_checksig_digest,(crypto_pk_t *env,
+                                         const char *data, size_t datalen,
+                                         const char *sig, size_t siglen));
+int crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen,
+                                  const char *from, size_t fromlen);
+int crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env, char *to,
+                                    size_t tolen,
+                                    const char *from, size_t fromlen,
+                                    int padding, int force);
+int crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env, char *to,
+                                     size_t tolen,
+                                     const char *from, size_t fromlen,
+                                     int padding, int warnOnFailure);
+int crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out);
+int crypto_pk_get_common_digests(crypto_pk_t *pk,
+                                 common_digests_t *digests_out);
+
 /* symmetric crypto */
 const char *crypto_cipher_get_key(crypto_cipher_t *env);
 
@@ -78,6 +148,52 @@ int crypto_cipher_decrypt_with_iv(const char *key,
                                   char *to, size_t tolen,
                                   const char *from, size_t fromlen);
 
+/* SHA-1 and other digests. */
+int crypto_digest(char *digest, const char *m, size_t len);
+int crypto_digest256(char *digest, const char *m, size_t len,
+                     digest_algorithm_t algorithm);
+int crypto_digest512(char *digest, const char *m, size_t len,
+                     digest_algorithm_t algorithm);
+int crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len);
+struct smartlist_t;
+void crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
+                                    const char *prepend,
+                                    const struct smartlist_t *lst,
+                                    const char *append,
+                                    digest_algorithm_t alg);
+void crypto_digest_smartlist(char *digest_out, size_t len_out,
+                             const struct smartlist_t *lst, const char *append,
+                             digest_algorithm_t alg);
+const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg);
+size_t crypto_digest_algorithm_get_length(digest_algorithm_t alg);
+int crypto_digest_algorithm_parse_name(const char *name);
+crypto_digest_t *crypto_digest_new(void);
+crypto_digest_t *crypto_digest256_new(digest_algorithm_t algorithm);
+crypto_digest_t *crypto_digest512_new(digest_algorithm_t algorithm);
+void crypto_digest_free_(crypto_digest_t *digest);
+#define crypto_digest_free(d) \
+  FREE_AND_NULL(crypto_digest_t, crypto_digest_free_, (d))
+void crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
+                             size_t len);
+void crypto_digest_get_digest(crypto_digest_t *digest,
+                              char *out, size_t out_len);
+crypto_digest_t *crypto_digest_dup(const crypto_digest_t *digest);
+void crypto_digest_assign(crypto_digest_t *into,
+                          const crypto_digest_t *from);
+void crypto_hmac_sha256(char *hmac_out,
+                        const char *key, size_t key_len,
+                        const char *msg, size_t msg_len);
+void crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out,
+                         const uint8_t *key, size_t key_len,
+                         const uint8_t *msg, size_t msg_len);
+
+crypto_xof_t *crypto_xof_new(void);
+void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len);
+void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len);
+void crypto_xof_free_(crypto_xof_t *xof);
+#define crypto_xof_free(xof) \
+  FREE_AND_NULL(crypto_xof_t, crypto_xof_free_, (xof))
+
 /* Key negotiation */
 #define DH_TYPE_CIRCUIT 1
 #define DH_TYPE_REND 2
@@ -103,6 +219,31 @@ int crypto_expand_key_material_rfc5869_sha256(
                                     const uint8_t *info_in, size_t info_in_len,
                                     uint8_t *key_out, size_t key_out_len);
 
+/* random numbers */
+int crypto_seed_rng(void) ATTR_WUR;
+MOCK_DECL(void,crypto_rand,(char *to, size_t n));
+void crypto_rand_unmocked(char *to, size_t n);
+void crypto_strongest_rand(uint8_t *out, size_t out_len);
+int crypto_rand_int(unsigned int max);
+int crypto_rand_int_range(unsigned int min, unsigned int max);
+uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max);
+time_t crypto_rand_time_range(time_t min, time_t max);
+uint64_t crypto_rand_uint64(uint64_t max);
+double crypto_rand_double(void);
+struct tor_weak_rng_t;
+void crypto_seed_weak_rng(struct tor_weak_rng_t *rng);
+int crypto_init_siphash_key(void);
+
+char *crypto_random_hostname(int min_rand_len, int max_rand_len,
+                             const char *prefix, const char *suffix);
+
+struct smartlist_t;
+void *smartlist_choose(const struct smartlist_t *sl);
+void smartlist_shuffle(struct smartlist_t *sl);
+
+/** OpenSSL-based utility functions. */
+void memwipe(void *mem, uint8_t byte, size_t sz);
+
 /* Prototypes for private functions only used by tortls.c, crypto.c, and the
  * unit tests. */
 struct dh_st;
@@ -110,5 +251,20 @@ struct dh_st *crypto_dh_get_dh_(crypto_dh_t *dh);
 
 void crypto_add_spaces_to_fp(char *out, size_t outlen, const char *in);
 
+#ifdef CRYPTO_PRIVATE
+
+STATIC int crypto_force_rand_ssleay(void);
+STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len);
+
+#ifdef TOR_UNIT_TESTS
+extern int break_strongest_rng_syscall;
+extern int break_strongest_rng_fallback;
+#endif
+#endif /* defined(CRYPTO_PRIVATE) */
+
+#ifdef TOR_UNIT_TESTS
+digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest);
+#endif
+
 #endif /* !defined(TOR_CRYPTO_H) */
 

src/common/crypto_curve25519.c

@@ -21,11 +21,9 @@
 #include <sys/stat.h>
 #endif
 #include "container.h"
+#include "crypto.h"
 #include "crypto_curve25519.h"
-#include "crypto_digest.h"
 #include "crypto_format.h"
-#include "crypto_rand.h"
-#include "crypto_util.h"
 #include "util.h"
 #include "torlog.h"
 

src/common/crypto_curve25519.h

@@ -6,7 +6,6 @@
 
 #include "testsupport.h"
 #include "torint.h"
-#include "crypto_digest.h"
 #include "crypto_openssl_mgt.h"
 
 /** Length of a curve25519 public key when encoded. */

src/common/crypto_digest.c

@@ -1,583 +0,0 @@
-/* Copyright (c) 2001, Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2017, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file crypto_digest.c
- * \brief Block of functions related with digest and xof utilities and
- * operations.
- **/
-
-#include "container.h"
-#include "crypto_digest.h"
-#include "crypto_openssl_mgt.h"
-#include "crypto_util.h"
-#include "torlog.h"
-
-#include "keccak-tiny/keccak-tiny.h"
-
-DISABLE_GCC_WARNING(redundant-decls)
-
-#include <openssl/hmac.h>
-#include <openssl/sha.h>
-
-ENABLE_GCC_WARNING(redundant-decls)
-
-/* Crypto digest functions */
-
-/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
- * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
- * Return 0 on success, -1 on failure.
- */
-int
-crypto_digest(char *digest, const char *m, size_t len)
-{
-  tor_assert(m);
-  tor_assert(digest);
-  if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL)
-    return -1;
-  return 0;
-}
-
-/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
- * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
- * into <b>digest</b>.  Return 0 on success, -1 on failure. */
-int
-crypto_digest256(char *digest, const char *m, size_t len,
-                 digest_algorithm_t algorithm)
-{
-  tor_assert(m);
-  tor_assert(digest);
-  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
-
-  int ret = 0;
-  if (algorithm == DIGEST_SHA256)
-    ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL);
-  else
-    ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
-           > -1);
-
-  if (!ret)
-    return -1;
-  return 0;
-}
-
-/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
- * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
- * into <b>digest</b>.  Return 0 on success, -1 on failure. */
-int
-crypto_digest512(char *digest, const char *m, size_t len,
-                 digest_algorithm_t algorithm)
-{
-  tor_assert(m);
-  tor_assert(digest);
-  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
-
-  int ret = 0;
-  if (algorithm == DIGEST_SHA512)
-    ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest)
-           != NULL);
-  else
-    ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
-           > -1);
-
-  if (!ret)
-    return -1;
-  return 0;
-}
-
-/** Set the common_digests_t in <b>ds_out</b> to contain every digest on the
- * <b>len</b> bytes in <b>m</b> that we know how to compute.  Return 0 on
- * success, -1 on failure. */
-int
-crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len)
-{
-  tor_assert(ds_out);
-  memset(ds_out, 0, sizeof(*ds_out));
-  if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0)
-    return -1;
-  if (crypto_digest256(ds_out->d[DIGEST_SHA256], m, len, DIGEST_SHA256) < 0)
-    return -1;
-
-  return 0;
-}
-
-/** Return the name of an algorithm, as used in directory documents. */
-const char *
-crypto_digest_algorithm_get_name(digest_algorithm_t alg)
-{
-  switch (alg) {
-    case DIGEST_SHA1:
-      return "sha1";
-    case DIGEST_SHA256:
-      return "sha256";
-    case DIGEST_SHA512:
-      return "sha512";
-    case DIGEST_SHA3_256:
-      return "sha3-256";
-    case DIGEST_SHA3_512:
-      return "sha3-512";
-      // LCOV_EXCL_START
-    default:
-      tor_fragile_assert();
-      return "??unknown_digest??";
-      // LCOV_EXCL_STOP
-  }
-}
-
-/** Given the name of a digest algorithm, return its integer value, or -1 if
- * the name is not recognized. */
-int
-crypto_digest_algorithm_parse_name(const char *name)
-{
-  if (!strcmp(name, "sha1"))
-    return DIGEST_SHA1;
-  else if (!strcmp(name, "sha256"))
-    return DIGEST_SHA256;
-  else if (!strcmp(name, "sha512"))
-    return DIGEST_SHA512;
-  else if (!strcmp(name, "sha3-256"))
-    return DIGEST_SHA3_256;
-  else if (!strcmp(name, "sha3-512"))
-    return DIGEST_SHA3_512;
-  else
-    return -1;
-}
-
-/** Given an algorithm, return the digest length in bytes. */
-size_t
-crypto_digest_algorithm_get_length(digest_algorithm_t alg)
-{
-  switch (alg) {
-    case DIGEST_SHA1:
-      return DIGEST_LEN;
-    case DIGEST_SHA256:
-      return DIGEST256_LEN;
-    case DIGEST_SHA512:
-      return DIGEST512_LEN;
-    case DIGEST_SHA3_256:
-      return DIGEST256_LEN;
-    case DIGEST_SHA3_512:
-      return DIGEST512_LEN;
-    default:
-      tor_assert(0);              // LCOV_EXCL_LINE
-      return 0; /* Unreachable */ // LCOV_EXCL_LINE
-  }
-}
-
-/** Intermediate information about the digest of a stream of data. */
-struct crypto_digest_t {
-  digest_algorithm_t algorithm; /**< Which algorithm is in use? */
-   /** State for the digest we're using.  Only one member of the
-    * union is usable, depending on the value of <b>algorithm</b>. Note also
-    * that space for other members might not even be allocated!
-    */
-  union {
-    SHA_CTX sha1; /**< state for SHA1 */
-    SHA256_CTX sha2; /**< state for SHA256 */
-    SHA512_CTX sha512; /**< state for SHA512 */
-    keccak_state sha3; /**< state for SHA3-[256,512] */
-  } d;
-};
-
-#ifdef TOR_UNIT_TESTS
-
-digest_algorithm_t
-crypto_digest_get_algorithm(crypto_digest_t *digest)
-{
-  tor_assert(digest);
-
-  return digest->algorithm;
-}
-
-#endif /* defined(TOR_UNIT_TESTS) */
-
-/**
- * Return the number of bytes we need to malloc in order to get a
- * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
- * when we free one.
- */
-static size_t
-crypto_digest_alloc_bytes(digest_algorithm_t alg)
-{
-  /* Helper: returns the number of bytes in the 'f' field of 'st' */
-#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
-  /* Gives the length of crypto_digest_t through the end of the field 'd' */
-#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \
-                         STRUCT_FIELD_SIZE(crypto_digest_t, f))
-  switch (alg) {
-    case DIGEST_SHA1:
-      return END_OF_FIELD(d.sha1);
-    case DIGEST_SHA256:
-      return END_OF_FIELD(d.sha2);
-    case DIGEST_SHA512:
-      return END_OF_FIELD(d.sha512);
-    case DIGEST_SHA3_256:
-    case DIGEST_SHA3_512:
-      return END_OF_FIELD(d.sha3);
-    default:
-      tor_assert(0); // LCOV_EXCL_LINE
-      return 0;      // LCOV_EXCL_LINE
-  }
-#undef END_OF_FIELD
-#undef STRUCT_FIELD_SIZE
-}
-
-/**
- * Internal function: create and return a new digest object for 'algorithm'.
- * Does not typecheck the algorithm.
- */
-static crypto_digest_t *
-crypto_digest_new_internal(digest_algorithm_t algorithm)
-{
-  crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm));
-  r->algorithm = algorithm;
-
-  switch (algorithm)
-    {
-    case DIGEST_SHA1:
-      SHA1_Init(&r->d.sha1);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Init(&r->d.sha2);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Init(&r->d.sha512);
-      break;
-    case DIGEST_SHA3_256:
-      keccak_digest_init(&r->d.sha3, 256);
-      break;
-    case DIGEST_SHA3_512:
-      keccak_digest_init(&r->d.sha3, 512);
-      break;
-    default:
-      tor_assert_unreached();
-    }
-
-  return r;
-}
-
-/** Allocate and return a new digest object to compute SHA1 digests.
- */
-crypto_digest_t *
-crypto_digest_new(void)
-{
-  return crypto_digest_new_internal(DIGEST_SHA1);
-}
-
-/** Allocate and return a new digest object to compute 256-bit digests
- * using <b>algorithm</b>.
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new`
- * C_RUST_COUPLED: `crypto::digest::Sha256::default`
- */
-crypto_digest_t *
-crypto_digest256_new(digest_algorithm_t algorithm)
-{
-  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
-  return crypto_digest_new_internal(algorithm);
-}
-
-/** Allocate and return a new digest object to compute 512-bit digests
- * using <b>algorithm</b>. */
-crypto_digest_t *
-crypto_digest512_new(digest_algorithm_t algorithm)
-{
-  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
-  return crypto_digest_new_internal(algorithm);
-}
-
-/** Deallocate a digest object.
- */
-void
-crypto_digest_free_(crypto_digest_t *digest)
-{
-  if (!digest)
-    return;
-  size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  memwipe(digest, 0, bytes);
-  tor_free(digest);
-}
-
-/** Add <b>len</b> bytes from <b>data</b> to the digest object.
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess`
- * C_RUST_COUPLED: `crypto::digest::Sha256::process`
- */
-void
-crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
-                        size_t len)
-{
-  tor_assert(digest);
-  tor_assert(data);
-  /* Using the SHA*_*() calls directly means we don't support doing
-   * SHA in hardware. But so far the delay of getting the question
-   * to the hardware, and hearing the answer, is likely higher than
-   * just doing it ourselves. Hashes are fast.
-   */
-  switch (digest->algorithm) {
-    case DIGEST_SHA1:
-      SHA1_Update(&digest->d.sha1, (void*)data, len);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Update(&digest->d.sha2, (void*)data, len);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Update(&digest->d.sha512, (void*)data, len);
-      break;
-    case DIGEST_SHA3_256: /* FALLSTHROUGH */
-    case DIGEST_SHA3_512:
-      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
-      break;
-    default:
-      /* LCOV_EXCL_START */
-      tor_fragile_assert();
-      break;
-      /* LCOV_EXCL_STOP */
-  }
-}
-
-/** Compute the hash of the data that has been passed to the digest
- * object; write the first out_len bytes of the result to <b>out</b>.
- * <b>out_len</b> must be \<= DIGEST512_LEN.
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest`
- * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256`
- */
-void
-crypto_digest_get_digest(crypto_digest_t *digest,
-                         char *out, size_t out_len)
-{
-  unsigned char r[DIGEST512_LEN];
-  crypto_digest_t tmpenv;
-  tor_assert(digest);
-  tor_assert(out);
-  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
-
-  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
-   * short output buffers by truncating appropriately. */
-  if (digest->algorithm == DIGEST_SHA3_256 ||
-      digest->algorithm == DIGEST_SHA3_512) {
-    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
-    return;
-  }
-
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  /* memcpy into a temporary ctx, since SHA*_Final clears the context */
-  memcpy(&tmpenv, digest, alloc_bytes);
-  switch (digest->algorithm) {
-    case DIGEST_SHA1:
-      SHA1_Final(r, &tmpenv.d.sha1);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Final(r, &tmpenv.d.sha2);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Final(r, &tmpenv.d.sha512);
-      break;
-//LCOV_EXCL_START
-    case DIGEST_SHA3_256: /* FALLSTHROUGH */
-    case DIGEST_SHA3_512:
-    default:
-      log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm);
-      /* This is fatal, because it should never happen. */
-      tor_assert_unreached();
-      break;
-//LCOV_EXCL_STOP
-  }
-  memcpy(out, r, out_len);
-  memwipe(r, 0, sizeof(r));
-}
-
-/** Allocate and return a new digest object with the same state as
- * <b>digest</b>
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup`
- * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256`
- */
-crypto_digest_t *
-crypto_digest_dup(const crypto_digest_t *digest)
-{
-  tor_assert(digest);
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  return tor_memdup(digest, alloc_bytes);
-}
-
-/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>.
- * Asserts that <b>digest</b> is a SHA1 digest object.
- */
-void
-crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
-                         const crypto_digest_t *digest)
-{
-  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  tor_assert(bytes <= sizeof(checkpoint->mem));
-  memcpy(checkpoint->mem, digest, bytes);
-}
-
-/** Restore the state of  <b>digest</b> from <b>checkpoint</b>.
- * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the
- * state was previously stored with crypto_digest_checkpoint() */
-void
-crypto_digest_restore(crypto_digest_t *digest,
-                      const crypto_digest_checkpoint_t *checkpoint)
-{
-  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  memcpy(digest, checkpoint->mem, bytes);
-}
-
-/** Replace the state of the digest object <b>into</b> with the state
- * of the digest object <b>from</b>.  Requires that 'into' and 'from'
- * have the same digest type.
- */
-void
-crypto_digest_assign(crypto_digest_t *into,
-                     const crypto_digest_t *from)
-{
-  tor_assert(into);
-  tor_assert(from);
-  tor_assert(into->algorithm == from->algorithm);
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm);
-  memcpy(into,from,alloc_bytes);
-}
-
-/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
- * at <b>digest_out</b> to the hash of the concatenation of those strings,
- * plus the optional string <b>append</b>, computed with the algorithm
- * <b>alg</b>.
- * <b>out_len</b> must be \<= DIGEST512_LEN. */
-void
-crypto_digest_smartlist(char *digest_out, size_t len_out,
-                        const smartlist_t *lst,
-                        const char *append,
-                        digest_algorithm_t alg)
-{
-  crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg);
-}
-
-/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
- * at <b>digest_out</b> to the hash of the concatenation of: the
- * optional string <b>prepend</b>, those strings,
- * and the optional string <b>append</b>, computed with the algorithm
- * <b>alg</b>.
- * <b>len_out</b> must be \<= DIGEST512_LEN. */
-void
-crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
-                        const char *prepend,
-                        const smartlist_t *lst,
-                        const char *append,
-                        digest_algorithm_t alg)
-{
-  crypto_digest_t *d = crypto_digest_new_internal(alg);
-  if (prepend)
-    crypto_digest_add_bytes(d, prepend, strlen(prepend));
-  SMARTLIST_FOREACH(lst, const char *, cp,
-                    crypto_digest_add_bytes(d, cp, strlen(cp)));
-  if (append)
-    crypto_digest_add_bytes(d, append, strlen(append));
-  crypto_digest_get_digest(d, digest_out, len_out);
-  crypto_digest_free(d);
-}
-
-/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using
- * the <b>key</b> of length <b>key_len</b>.  Store the DIGEST256_LEN-byte
- * result in <b>hmac_out</b>. Asserts on failure.
- */
-void
-crypto_hmac_sha256(char *hmac_out,
-                   const char *key, size_t key_len,
-                   const char *msg, size_t msg_len)
-{
-  unsigned char *rv = NULL;
-  /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */
-  tor_assert(key_len < INT_MAX);
-  tor_assert(msg_len < INT_MAX);
-  tor_assert(hmac_out);
-  rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len,
-            (unsigned char*)hmac_out, NULL);
-  tor_assert(rv);
-}
-
-/** Compute a MAC using SHA3-256 of <b>msg_len</b> bytes in <b>msg</b> using a
- * <b>key</b> of length <b>key_len</b> and a <b>salt</b> of length
- * <b>salt_len</b>. Store the result of <b>len_out</b> bytes in in
- * <b>mac_out</b>. This function can't fail. */
-void
-crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out,
-                    const uint8_t *key, size_t key_len,
-                    const uint8_t *msg, size_t msg_len)
-{
-  crypto_digest_t *digest;
-
-  const uint64_t key_len_netorder = tor_htonll(key_len);
-
-  tor_assert(mac_out);
-  tor_assert(key);
-  tor_assert(msg);
-
-  digest = crypto_digest256_new(DIGEST_SHA3_256);
-
-  /* Order matters here that is any subsystem using this function should
-   * expect this very precise ordering in the MAC construction. */
-  crypto_digest_add_bytes(digest, (const char *) &key_len_netorder,
-                          sizeof(key_len_netorder));
-  crypto_digest_add_bytes(digest, (const char *) key, key_len);
-  crypto_digest_add_bytes(digest, (const char *) msg, msg_len);
-  crypto_digest_get_digest(digest, (char *) mac_out, len_out);
-  crypto_digest_free(digest);
-}
-
-/* xof functions  */
-
-/** Internal state for a eXtendable-Output Function (XOF). */
-struct crypto_xof_t {
-  keccak_state s;
-};
-
-/** Allocate a new XOF object backed by SHAKE-256.  The security level
- * provided is a function of the length of the output used.  Read and
- * understand FIPS-202 A.2 "Additional Consideration for Extendable-Output
- * Functions" before using this construct.
- */
-crypto_xof_t *
-crypto_xof_new(void)
-{
-  crypto_xof_t *xof;
-  xof = tor_malloc(sizeof(crypto_xof_t));
-  keccak_xof_init(&xof->s, 256);
-  return xof;
-}
-
-/** Absorb bytes into a XOF object.  Must not be called after a call to
- * crypto_xof_squeeze_bytes() for the same instance, and will assert
- * if attempted.
- */
-void
-crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len)
-{
-  int i = keccak_xof_absorb(&xof->s, data, len);
-  tor_assert(i == 0);
-}
-
-/** Squeeze bytes out of a XOF object.  Calling this routine will render
- * the XOF instance ineligible to absorb further data.
- */
-void
-crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len)
-{
-  int i = keccak_xof_squeeze(&xof->s, out, len);
-  tor_assert(i == 0);
-}
-
-/** Cleanse and deallocate a XOF object. */
-void
-crypto_xof_free_(crypto_xof_t *xof)
-{
-  if (!xof)
-    return;
-  memwipe(xof, 0, sizeof(crypto_xof_t));
-  tor_free(xof);
-}
-

src/common/crypto_digest.h

@@ -1,136 +0,0 @@
-/* Copyright (c) 2001, Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2017, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file crypto_digest.h
- *
- * \brief Headers for crypto_digest.c
- **/
-
-#ifndef TOR_CRYPTO_DIGEST_H
-#define TOR_CRYPTO_DIGEST_H
-
-#include <stdio.h>
-
-#include "container.h"
-#include "torint.h"
-
-/** Length of the output of our message digest. */
-#define DIGEST_LEN 20
-/** Length of the output of our second (improved) message digests.  (For now
- * this is just sha256, but it could be any other 256-bit digest.) */
-#define DIGEST256_LEN 32
-/** Length of the output of our 64-bit optimized message digests (SHA512). */
-#define DIGEST512_LEN 64
-
-/** Length of a sha1 message digest when encoded in base32 with trailing =
- * signs removed. */
-#define BASE32_DIGEST_LEN 32
-/** Length of a sha1 message digest when encoded in base64 with trailing =
- * signs removed. */
-#define BASE64_DIGEST_LEN 27
-/** Length of a sha256 message digest when encoded in base64 with trailing =
- * signs removed. */
-#define BASE64_DIGEST256_LEN 43
-/** Length of a sha512 message digest when encoded in base64 with trailing =
- * signs removed. */
-#define BASE64_DIGEST512_LEN 86
-
-/** Length of hex encoding of SHA1 digest, not including final NUL. */
-#define HEX_DIGEST_LEN 40
-/** Length of hex encoding of SHA256 digest, not including final NUL. */
-#define HEX_DIGEST256_LEN 64
-/** Length of hex encoding of SHA512 digest, not including final NUL. */
-#define HEX_DIGEST512_LEN 128
-
-typedef enum {
-  DIGEST_SHA1 = 0,
-  DIGEST_SHA256 = 1,
-  DIGEST_SHA512 = 2,
-  DIGEST_SHA3_256 = 3,
-  DIGEST_SHA3_512 = 4,
-} digest_algorithm_t;
-#define  N_DIGEST_ALGORITHMS (DIGEST_SHA3_512+1)
-#define  N_COMMON_DIGEST_ALGORITHMS (DIGEST_SHA256+1)
-
-#define DIGEST_CHECKPOINT_BYTES (SIZEOF_VOID_P + 512)
-/** Structure used to temporarily save the a digest object. Only implemented
- * for SHA1 digest for now. */
-typedef struct crypto_digest_checkpoint_t {
-  uint8_t mem[DIGEST_CHECKPOINT_BYTES];
-} crypto_digest_checkpoint_t;
-
-/** A set of all the digests we commonly compute, taken on a single
- * string.  Any digests that are shorter than 512 bits are right-padded
- * with 0 bits.
- *
- * Note that this representation wastes 44 bytes for the SHA1 case, so
- * don't use it for anything where we need to allocate a whole bunch at
- * once.
- **/
-typedef struct {
-  char d[N_COMMON_DIGEST_ALGORITHMS][DIGEST256_LEN];
-} common_digests_t;
-
-typedef struct crypto_digest_t crypto_digest_t;
-typedef struct crypto_xof_t crypto_xof_t;
-
-/* SHA-1 and other digests */
-int crypto_digest(char *digest, const char *m, size_t len);
-int crypto_digest256(char *digest, const char *m, size_t len,
-                     digest_algorithm_t algorithm);
-int crypto_digest512(char *digest, const char *m, size_t len,
-                     digest_algorithm_t algorithm);
-int crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len);
-void crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
-                                    const char *prepend,
-                                    const struct smartlist_t *lst,
-                                    const char *append,
-                                    digest_algorithm_t alg);
-void crypto_digest_smartlist(char *digest_out, size_t len_out,
-                             const struct smartlist_t *lst, const char *append,
-                             digest_algorithm_t alg);
-const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg);
-size_t crypto_digest_algorithm_get_length(digest_algorithm_t alg);
-int crypto_digest_algorithm_parse_name(const char *name);
-crypto_digest_t *crypto_digest_new(void);
-crypto_digest_t *crypto_digest256_new(digest_algorithm_t algorithm);
-crypto_digest_t *crypto_digest512_new(digest_algorithm_t algorithm);
-void crypto_digest_free_(crypto_digest_t *digest);
-#define crypto_digest_free(d) \
-  FREE_AND_NULL(crypto_digest_t, crypto_digest_free_, (d))
-void crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
-                             size_t len);
-void crypto_digest_get_digest(crypto_digest_t *digest,
-                              char *out, size_t out_len);
-crypto_digest_t *crypto_digest_dup(const crypto_digest_t *digest);
-void crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
-                              const crypto_digest_t *digest);
-void crypto_digest_restore(crypto_digest_t *digest,
-                           const crypto_digest_checkpoint_t *checkpoint);
-void crypto_digest_assign(crypto_digest_t *into,
-                          const crypto_digest_t *from);
-void crypto_hmac_sha256(char *hmac_out,
-                        const char *key, size_t key_len,
-                        const char *msg, size_t msg_len);
-void crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out,
-                         const uint8_t *key, size_t key_len,
-                         const uint8_t *msg, size_t msg_len);
-
-/* xof functions*/
-crypto_xof_t *crypto_xof_new(void);
-void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len);
-void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len);
-void crypto_xof_free_(crypto_xof_t *xof);
-#define crypto_xof_free(xof) \
-  FREE_AND_NULL(crypto_xof_t, crypto_xof_free_, (xof))
-
-#ifdef TOR_UNIT_TESTS
-digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest);
-#endif
-
-#endif /* !defined(TOR_CRYPTO_DIGEST_H) */
-

src/common/crypto_ed25519.c

@@ -21,12 +21,11 @@
 #include <sys/stat.h>
 #endif
 
+#include "crypto.h"
+
 #include "crypto_curve25519.h"
-#include "crypto_digest.h"
 #include "crypto_ed25519.h"
 #include "crypto_format.h"
-#include "crypto_rand.h"
-#include "crypto_util.h"
 #include "torlog.h"
 #include "util.h"
 #include "util_format.h"

src/common/crypto_format.c

@@ -15,11 +15,10 @@
 #include <sys/stat.h>
 #endif
 #include "container.h"
+#include "crypto.h"
 #include "crypto_curve25519.h"
-#include "crypto_digest.h"
 #include "crypto_ed25519.h"
 #include "crypto_format.h"
-#include "crypto_util.h"
 #include "util.h"
 #include "util_format.h"
 #include "torlog.h"

src/common/crypto_pwbox.c

@@ -9,11 +9,8 @@
  */
 
 #include "crypto.h"
-#include "crypto_digest.h"
-#include "crypto_pwbox.h"
-#include "crypto_rand.h"
 #include "crypto_s2k.h"
-#include "crypto_util.h"
+#include "crypto_pwbox.h"
 #include "di_ops.h"
 #include "util.h"
 #include "pwbox.h"

src/common/crypto_rand.c

@@ -1,615 +0,0 @@
-/* Copyright (c) 2001, Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2018, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file crypto_rand.c
- *
- * \brief Functions for initialising and seeding (pseudo-)random
- * number generators, and working with randomness.
- **/
-
-#ifndef CRYPTO_RAND_PRIVATE
-#define CRYPTO_RAND_PRIVATE
-
-#include "crypto_rand.h"
-
-#ifdef _WIN32
-#include <windows.h>
-#include <wincrypt.h>
-#endif /* defined(_WIN32) */
-
-#include "container.h"
-#include "compat.h"
-#include "compat_openssl.h"
-#include "crypto_util.h"
-#include "sandbox.h"
-#include "testsupport.h"
-#include "torlog.h"
-#include "util.h"
-#include "util_format.h"
-
-DISABLE_GCC_WARNING(redundant-decls)
-#include <openssl/rand.h>
-ENABLE_GCC_WARNING(redundant-decls)
-
-#if __GNUC__ && GCC_VERSION >= 402
-#if GCC_VERSION >= 406
-#pragma GCC diagnostic pop
-#else
-#pragma GCC diagnostic warning "-Wredundant-decls"
-#endif
-#endif /* __GNUC__ && GCC_VERSION >= 402 */
-
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-#ifdef HAVE_SYS_FCNTL_H
-#include <sys/fcntl.h>
-#endif
-#ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SYS_SYSCALL_H
-#include <sys/syscall.h>
-#endif
-#ifdef HAVE_SYS_RANDOM_H
-#include <sys/random.h>
-#endif
-
-/**
- * How many bytes of entropy we add at once.
- *
- * This is how much entropy OpenSSL likes to add right now, so maybe it will
- * work for us too.
- **/
-#define ADD_ENTROPY 32
-
-/**
- * Longest recognized DNS query.
- **/
-#define MAX_DNS_LABEL_SIZE 63
-
-/**
- * Largest strong entropy request permitted.
- **/
-#define MAX_STRONGEST_RAND_SIZE 256
-
-/**
- * Set the seed of the weak RNG to a random value.
- **/
-void
-crypto_seed_weak_rng(tor_weak_rng_t *rng)
-{
-  unsigned seed;
-  crypto_rand((void*)&seed, sizeof(seed));
-  tor_init_weak_random(rng, seed);
-}
-
-#ifdef TOR_UNIT_TESTS
-int break_strongest_rng_syscall = 0;
-int break_strongest_rng_fallback = 0;
-#endif
-
-/**
- * Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
- * via system calls, storing it into <b>out</b>. Return 0 on success, -1 on
- * failure.  A maximum request size of 256 bytes is imposed.
- **/
-static int
-crypto_strongest_rand_syscall(uint8_t *out, size_t out_len)
-{
-  tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE);
-
-  /* We only log at notice-level here because in the case that this function
-   * fails the crypto_strongest_rand_raw() caller will log with a warning-level
-   * message and let crypto_strongest_rand() error out and finally terminating
-   * Tor with an assertion error.
-   */
-
-#ifdef TOR_UNIT_TESTS
-  if (break_strongest_rng_syscall)
-    return -1;
-#endif
-
-#if defined(_WIN32)
-  static int provider_set = 0;
-  static HCRYPTPROV provider;
-
-  if (!provider_set) {
-    if (!CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL,
-                             CRYPT_VERIFYCONTEXT)) {
-      log_notice(LD_CRYPTO, "Unable to set Windows CryptoAPI provider [1].");
-      return -1;
-    }
-    provider_set = 1;
-  }
-  if (!CryptGenRandom(provider, out_len, out)) {
-    log_notice(LD_CRYPTO, "Unable get entropy from the Windows CryptoAPI.");
-    return -1;
-  }
-
-  return 0;
-#elif defined(__linux__) && defined(SYS_getrandom)
-  static int getrandom_works = 1; /* Be optimistic about our chances... */
-
-  /* getrandom() isn't as straightforward as getentropy(), and has
-   * no glibc wrapper.
-   *
-   * As far as I can tell from getrandom(2) and the source code, the
-   * requests we issue will always succeed (though it will block on the
-   * call if /dev/urandom isn't seeded yet), since we are NOT specifying
-   * GRND_NONBLOCK and the request is <= 256 bytes.
-   *
-   * The manpage is unclear on what happens if a signal interrupts the call
-   * while the request is blocked due to lack of entropy....
-   *
-   * We optimistically assume that getrandom() is available and functional
-   * because it is the way of the future, and 2 branch mispredicts pale in
-   * comparison to the overheads involved with failing to open
-   * /dev/srandom followed by opening and reading from /dev/urandom.
-   */
-  if (PREDICT_LIKELY(getrandom_works)) {
-    long ret;
-    /* A flag of '0' here means to read from '/dev/urandom', and to
-     * block if insufficient entropy is available to service the
-     * request.
-     */
-    const unsigned int flags = 0;
-    do {
-      ret = syscall(SYS_getrandom, out, out_len, flags);
-    } while (ret == -1 && ((errno == EINTR) ||(errno == EAGAIN)));
-
-    if (PREDICT_UNLIKELY(ret == -1)) {
-      /* LCOV_EXCL_START we can't actually make the syscall fail in testing. */
-      tor_assert(errno != EAGAIN);
-      tor_assert(errno != EINTR);
-
-      /* Useful log message for errno. */
-      if (errno == ENOSYS) {
-        log_notice(LD_CRYPTO, "Can't get entropy from getrandom()."
-                   " You are running a version of Tor built to support"
-                   " getrandom(), but the kernel doesn't implement this"
-                   " function--probably because it is too old?"
-                   " Trying fallback method instead.");
-      } else {
-        log_notice(LD_CRYPTO, "Can't get entropy from getrandom(): %s."
-                              " Trying fallback method instead.",
-                   strerror(errno));
-      }
-
-      getrandom_works = 0; /* Don't bother trying again. */
-      return -1;
-      /* LCOV_EXCL_STOP */
-    }
-
-    tor_assert(ret == (long)out_len);
-    return 0;
-  }
-
-  return -1; /* getrandom() previously failed unexpectedly. */
-#elif defined(HAVE_GETENTROPY)
-  /* getentropy() is what Linux's getrandom() wants to be when it grows up.
-   * the only gotcha is that requests are limited to 256 bytes.
-   */
-  return getentropy(out, out_len);
-#else
-  (void) out;
-#endif /* defined(_WIN32) || ... */
-
-  /* This platform doesn't have a supported syscall based random. */
-  return -1;
-}
-
-/**
- * Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
- * via the per-platform fallback mechanism, storing it into <b>out</b>.
- * Return 0 on success, -1 on failure.  A maximum request size of 256 bytes
- * is imposed.
- **/
-static int
-crypto_strongest_rand_fallback(uint8_t *out, size_t out_len)
-{
-#ifdef TOR_UNIT_TESTS
-  if (break_strongest_rng_fallback)
-    return -1;
-#endif
-
-#ifdef _WIN32
-  /* Windows exclusively uses crypto_strongest_rand_syscall(). */
-  (void)out;
-  (void)out_len;
-  return -1;
-#else /* !(defined(_WIN32)) */
-  static const char *filenames[] = {
-    "/dev/srandom", "/dev/urandom", "/dev/random", NULL
-  };
-  int fd, i;
-  size_t n;
-
-  for (i = 0; filenames[i]; ++i) {
-    log_debug(LD_FS, "Considering %s as entropy source", filenames[i]);
-    fd = open(sandbox_intern_string(filenames[i]), O_RDONLY, 0);
-    if (fd<0) continue;
-    log_info(LD_CRYPTO, "Reading entropy from \"%s\"", filenames[i]);
-    n = read_all(fd, (char*)out, out_len, 0);
-    close(fd);
-    if (n != out_len) {
-      /* LCOV_EXCL_START
-       * We can't make /dev/foorandom actually fail. */
-      log_notice(LD_CRYPTO,
-                 "Error reading from entropy source %s (read only %lu bytes).",
-                 filenames[i],
-                 (unsigned long)n);
-      return -1;
-      /* LCOV_EXCL_STOP */
-    }
-
-    return 0;
-  }
-
-  return -1;
-#endif /* defined(_WIN32) */
-}
-
-/**
- * Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
- * storing it into <b>out</b>. Return 0 on success, -1 on failure.  A maximum
- * request size of 256 bytes is imposed.
- **/
-STATIC int
-crypto_strongest_rand_raw(uint8_t *out, size_t out_len)
-{
-  static const size_t sanity_min_size = 16;
-  static const int max_attempts = 3;
-  tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE);
-
-  /* For buffers >= 16 bytes (128 bits), we sanity check the output by
-   * zero filling the buffer and ensuring that it actually was at least
-   * partially modified.
-   *
-   * Checking that any individual byte is non-zero seems like it would
-   * fail too often (p = out_len * 1/256) for comfort, but this is an
-   * "adjust according to taste" sort of check.
-   */
-  memwipe(out, 0, out_len);
-  for (int i = 0; i < max_attempts; i++) {
-    /* Try to use the syscall/OS favored mechanism to get strong entropy. */
-    if (crypto_strongest_rand_syscall(out, out_len) != 0) {
-      /* Try to use the less-favored mechanism to get strong entropy. */
-      if (crypto_strongest_rand_fallback(out, out_len) != 0) {
-        /* Welp, we tried.  Hopefully the calling code terminates the process
-         * since we're basically boned without good entropy.
-         */
-        log_warn(LD_CRYPTO,
-                 "Cannot get strong entropy: no entropy source found.");
-        return -1;
-      }
-    }
-
-    if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len))
-      return 0;
-  }
-
-  /* LCOV_EXCL_START
-   *
-   * We tried max_attempts times to fill a buffer >= 128 bits long,
-   * and each time it returned all '0's.  Either the system entropy
-   * source is busted, or the user should go out and buy a ticket to
-   * every lottery on the planet.
-   */
-  log_warn(LD_CRYPTO, "Strong OS entropy returned all zero buffer.");
-
-  return -1;
-  /* LCOV_EXCL_STOP */
-}
-
-/**
- * Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
- * storing it into <b>out</b>.
- **/
-void
-crypto_strongest_rand(uint8_t *out, size_t out_len)
-{
-#define DLEN SHA512_DIGEST_LENGTH
-  /* We're going to hash DLEN bytes from the system RNG together with some
-   * bytes from the openssl PRNG, in order to yield DLEN bytes.
-   */
-  uint8_t inp[DLEN*2];
-  uint8_t tmp[DLEN];
-  tor_assert(out);
-  while (out_len) {
-    crypto_rand((char*) inp, DLEN);
-    if (crypto_strongest_rand_raw(inp+DLEN, DLEN) < 0) {
-      // LCOV_EXCL_START
-      log_err(LD_CRYPTO, "Failed to load strong entropy when generating an "
-              "important key. Exiting.");
-      /* Die with an assertion so we get a stack trace. */
-      tor_assert(0);
-      // LCOV_EXCL_STOP
-    }
-    if (out_len >= DLEN) {
-      SHA512(inp, sizeof(inp), out);
-      out += DLEN;
-      out_len -= DLEN;
-    } else {
-      SHA512(inp, sizeof(inp), tmp);
-      memcpy(out, tmp, out_len);
-      break;
-    }
-  }
-  memwipe(tmp, 0, sizeof(tmp));
-  memwipe(inp, 0, sizeof(inp));
-#undef DLEN
-}
-
-/**
- * Seed OpenSSL's random number generator with bytes from the operating
- * system.  Return 0 on success, -1 on failure.
- **/
-int
-crypto_seed_rng(void)
-{
-  int rand_poll_ok = 0, load_entropy_ok = 0;
-  uint8_t buf[ADD_ENTROPY];
-
-  /* OpenSSL has a RAND_poll function that knows about more kinds of
-   * entropy than we do.  We'll try calling that, *and* calling our own entropy
-   * functions.  If one succeeds, we'll accept the RNG as seeded. */
-  rand_poll_ok = RAND_poll();
-  if (rand_poll_ok == 0)
-    log_warn(LD_CRYPTO, "RAND_poll() failed."); // LCOV_EXCL_LINE
-
-  load_entropy_ok = !crypto_strongest_rand_raw(buf, sizeof(buf));
-  if (load_entropy_ok) {
-    RAND_seed(buf, sizeof(buf));
-  }
-
-  memwipe(buf, 0, sizeof(buf));
-
-  if ((rand_poll_ok || load_entropy_ok) && RAND_status() == 1)
-    return 0;
-  else
-    return -1;
-}
-
-/**
- * Write <b>n</b> bytes of strong random data to <b>to</b>. Supports mocking
- * for unit tests.
- *
- * This function is not allowed to fail; if it would fail to generate strong
- * entropy, it must terminate the process instead.
- **/
-MOCK_IMPL(void,
-crypto_rand, (char *to, size_t n))
-{
-  crypto_rand_unmocked(to, n);
-}
-
-/**
- * Write <b>n</b> bytes of strong random data to <b>to</b>.  Most callers
- * will want crypto_rand instead.
- *
- * This function is not allowed to fail; if it would fail to generate strong
- * entropy, it must terminate the process instead.
- **/
-void
-crypto_rand_unmocked(char *to, size_t n)
-{
-  int r;
-  if (n == 0)
-    return;
-
-  tor_assert(n < INT_MAX);
-  tor_assert(to);
-  r = RAND_bytes((unsigned char*)to, (int)n);
-  /* We consider a PRNG failure non-survivable. Let's assert so that we get a
-   * stack trace about where it happened.
-   */
-  tor_assert(r >= 0);
-}
-
-/**
- * Return a pseudorandom integer, chosen uniformly from the values
- * between 0 and <b>max</b>-1 inclusive.  <b>max</b> must be between 1 and
- * INT_MAX+1, inclusive.
- */
-int
-crypto_rand_int(unsigned int max)
-{
-  unsigned int val;
-  unsigned int cutoff;
-  tor_assert(max <= ((unsigned int)INT_MAX)+1);
-  tor_assert(max > 0); /* don't div by 0 */
-
-  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
-   * distribution with clipping at the upper end of unsigned int's
-   * range.
-   */
-  cutoff = UINT_MAX - (UINT_MAX%max);
-  while (1) {
-    crypto_rand((char*)&val, sizeof(val));
-    if (val < cutoff)
-      return val % max;
-  }
-}
-
-/**
- * Return a pseudorandom integer, chosen uniformly from the values i such
- * that min <= i < max.
- *
- * <b>min</b> MUST be in range [0, <b>max</b>).
- * <b>max</b> MUST be in range (min, INT_MAX].
- **/
-int
-crypto_rand_int_range(unsigned int min, unsigned int max)
-{
-  tor_assert(min < max);
-  tor_assert(max <= INT_MAX);
-
-  /* The overflow is avoided here because crypto_rand_int() returns a value
-   * between 0 and (max - min) inclusive. */
-  return min + crypto_rand_int(max - min);
-}
-
-/**
- * As crypto_rand_int_range, but supports uint64_t.
- **/
-uint64_t
-crypto_rand_uint64_range(uint64_t min, uint64_t max)
-{
-  tor_assert(min < max);
-  return min + crypto_rand_uint64(max - min);
-}
-
-/**
- * As crypto_rand_int_range, but supports time_t.
- **/
-time_t
-crypto_rand_time_range(time_t min, time_t max)
-{
-  tor_assert(min < max);
-  return min + (time_t)crypto_rand_uint64(max - min);
-}
-
-/**
- * Return a pseudorandom 64-bit integer, chosen uniformly from the values
- * between 0 and <b>max</b>-1 inclusive.
- **/
-uint64_t
-crypto_rand_uint64(uint64_t max)
-{
-  uint64_t val;
-  uint64_t cutoff;
-  tor_assert(max < UINT64_MAX);
-  tor_assert(max > 0); /* don't div by 0 */
-
-  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
-   * distribution with clipping at the upper end of unsigned int's
-   * range.
-   */
-  cutoff = UINT64_MAX - (UINT64_MAX%max);
-  while (1) {
-    crypto_rand((char*)&val, sizeof(val));
-    if (val < cutoff)
-      return val % max;
-  }
-}
-
-/**
- * Return a pseudorandom double d, chosen uniformly from the range
- * 0.0 <= d < 1.0.
- **/
-double
-crypto_rand_double(void)
-{
-  /* We just use an unsigned int here; we don't really care about getting
-   * more than 32 bits of resolution */
-  unsigned int u;
-  crypto_rand((char*)&u, sizeof(u));
-#if SIZEOF_INT == 4
-#define UINT_MAX_AS_DOUBLE 4294967296.0
-#elif SIZEOF_INT == 8
-#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19
-#else
-#error SIZEOF_INT is neither 4 nor 8
-#endif /* SIZEOF_INT == 4 || ... */
-  return ((double)u) / UINT_MAX_AS_DOUBLE;
-}
-
-/**
- * Generate and return a new random hostname starting with <b>prefix</b>,
- * ending with <b>suffix</b>, and containing no fewer than
- * <b>min_rand_len</b> and no more than <b>max_rand_len</b> random base32
- * characters. Does not check for failure.
- *
- * Clip <b>max_rand_len</b> to MAX_DNS_LABEL_SIZE.
- **/
-char *
-crypto_random_hostname(int min_rand_len, int max_rand_len, const char *prefix,
-                       const char *suffix)
-{
-  char *result, *rand_bytes;
-  int randlen, rand_bytes_len;
-  size_t resultlen, prefixlen;
-
-  if (max_rand_len > MAX_DNS_LABEL_SIZE)
-    max_rand_len = MAX_DNS_LABEL_SIZE;
-  if (min_rand_len > max_rand_len)
-    min_rand_len = max_rand_len;
-
-  randlen = crypto_rand_int_range(min_rand_len, max_rand_len+1);
-
-  prefixlen = strlen(prefix);
-  resultlen = prefixlen + strlen(suffix) + randlen + 16;
-
-  rand_bytes_len = ((randlen*5)+7)/8;
-  if (rand_bytes_len % 5)
-    rand_bytes_len += 5 - (rand_bytes_len%5);
-  rand_bytes = tor_malloc(rand_bytes_len);
-  crypto_rand(rand_bytes, rand_bytes_len);
-
-  result = tor_malloc(resultlen);
-  memcpy(result, prefix, prefixlen);
-  base32_encode(result+prefixlen, resultlen-prefixlen,
-                rand_bytes, rand_bytes_len);
-  tor_free(rand_bytes);
-  strlcpy(result+prefixlen+randlen, suffix, resultlen-(prefixlen+randlen));
-
-  return result;
-}
-
-/**
- * Return a randomly chosen element of <b>sl</b>; or NULL if <b>sl</b>
- * is empty.
- **/
-void *
-smartlist_choose(const smartlist_t *sl)
-{
-  int len = smartlist_len(sl);
-  if (len)
-    return smartlist_get(sl,crypto_rand_int(len));
-  return NULL; /* no elements to choose from */
-}
-
-/**
- * Scramble the elements of <b>sl</b> into a random order.
- **/
-void
-smartlist_shuffle(smartlist_t *sl)
-{
-  int i;
-  /* From the end of the list to the front, choose at random from the
-     positions we haven't looked at yet, and swap that position into the
-     current position.  Remember to give "no swap" the same probability as
-     any other swap. */
-  for (i = smartlist_len(sl)-1; i > 0; --i) {
-    int j = crypto_rand_int(i+1);
-    smartlist_swap(sl, i, j);
-  }
-}
-
-/** Make sure that openssl is using its default PRNG. Return 1 if we had to
- * adjust it; 0 otherwise. */
-int
-crypto_force_rand_ssleay(void)
-{
-  RAND_METHOD *default_method;
-  default_method = RAND_OpenSSL();
-  if (RAND_get_rand_method() != default_method) {
-    log_notice(LD_CRYPTO, "It appears that one of our engines has provided "
-               "a replacement the OpenSSL RNG. Resetting it to the default "
-               "implementation.");
-    RAND_set_rand_method(default_method);
-    return 1;
-  }
-  return 0;
-}
-
-#endif /* !defined(CRYPTO_RAND_PRIVATE) */
-

src/common/crypto_rand.h

@@ -1,52 +0,0 @@
-/* Copyright (c) 2001, Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2018, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file crypto_rand.h
- *
- * \brief Common functions for using (pseudo-)random number generators.
- **/
-
-#ifndef TOR_CRYPTO_RAND_H
-#define TOR_CRYPTO_RAND_H
-
-#include "torint.h"
-#include "util.h"
-
-/* random numbers */
-int crypto_seed_rng(void) ATTR_WUR;
-MOCK_DECL(void,crypto_rand,(char *to, size_t n));
-void crypto_rand_unmocked(char *to, size_t n);
-void crypto_strongest_rand(uint8_t *out, size_t out_len);
-int crypto_rand_int(unsigned int max);
-int crypto_rand_int_range(unsigned int min, unsigned int max);
-uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max);
-time_t crypto_rand_time_range(time_t min, time_t max);
-uint64_t crypto_rand_uint64(uint64_t max);
-double crypto_rand_double(void);
-struct tor_weak_rng_t;
-void crypto_seed_weak_rng(struct tor_weak_rng_t *rng);
-
-char *crypto_random_hostname(int min_rand_len, int max_rand_len,
-                             const char *prefix, const char *suffix);
-
-struct smartlist_t;
-void *smartlist_choose(const struct smartlist_t *sl);
-void smartlist_shuffle(struct smartlist_t *sl);
-int crypto_force_rand_ssleay(void);
-
-#ifdef CRYPTO_RAND_PRIVATE
-
-STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len);
-
-#ifdef TOR_UNIT_TESTS
-extern int break_strongest_rng_syscall;
-extern int break_strongest_rng_fallback;
-#endif
-#endif /* defined(CRYPTO_RAND_PRIVATE) */
-
-#endif /* !defined(TOR_CRYPTO_RAND_H) */
-

src/common/crypto_rsa.c

@@ -9,14 +9,12 @@
  * \brief Block of functions related with RSA utilities and operations.
  **/
 
-#include "crypto.h"
-#include "crypto_curve25519.h"
-#include "crypto_digest.h"
-#include "crypto_format.h"
-#include "compat_openssl.h"
-#include "crypto_rand.h"
 #include "crypto_rsa.h"
-#include "crypto_util.h"
+#include "crypto.h"
+#include "compat_openssl.h"
+#include "crypto_curve25519.h"
+#include "crypto_ed25519.h"
+#include "crypto_format.h"
 
 DISABLE_GCC_WARNING(redundant-decls)
 
@@ -629,148 +627,6 @@ crypto_pk_copy_full(crypto_pk_t *env)
   return crypto_new_pk_from_rsa_(new_key);
 }
 
-/** Perform a hybrid (public/secret) encryption on <b>fromlen</b>
- * bytes of data from <b>from</b>, with padding type 'padding',
- * storing the results on <b>to</b>.
- *
- * Returns the number of bytes written on success, -1 on failure.
- *
- * The encrypted data consists of:
- *   - The source data, padded and encrypted with the public key, if the
- *     padded source data is no longer than the public key, and <b>force</b>
- *     is false, OR
- *   - The beginning of the source data prefixed with a 16-byte symmetric key,
- *     padded and encrypted with the public key; followed by the rest of
- *     the source data encrypted in AES-CTR mode with the symmetric key.
- *
- * NOTE that this format does not authenticate the symmetrically encrypted
- * part of the data, and SHOULD NOT BE USED for new protocols.
- */
-int
-crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env,
-                                char *to, size_t tolen,
-                                const char *from,
-                                size_t fromlen,
-                                int padding, int force)
-{
-  int overhead, outlen, r;
-  size_t pkeylen, symlen;
-  crypto_cipher_t *cipher = NULL;
-  char *buf = NULL;
-
-  tor_assert(env);
-  tor_assert(from);
-  tor_assert(to);
-  tor_assert(fromlen < SIZE_T_CEILING);
-
-  overhead = crypto_get_rsa_padding_overhead(crypto_get_rsa_padding(padding));
-  pkeylen = crypto_pk_keysize(env);
-
-  if (!force && fromlen+overhead <= pkeylen) {
-    /* It all fits in a single encrypt. */
-    return crypto_pk_public_encrypt(env,to,
-                                    tolen,
-                                    from,fromlen,padding);
-  }
-  tor_assert(tolen >= fromlen + overhead + CIPHER_KEY_LEN);
-  tor_assert(tolen >= pkeylen);
-
-  char key[CIPHER_KEY_LEN];
-  crypto_rand(key, sizeof(key)); /* generate a new key. */
-  cipher = crypto_cipher_new(key);
-
-  buf = tor_malloc(pkeylen+1);
-  memcpy(buf, key, CIPHER_KEY_LEN);
-  memcpy(buf+CIPHER_KEY_LEN, from, pkeylen-overhead-CIPHER_KEY_LEN);
-
-  /* Length of symmetrically encrypted data. */
-  symlen = fromlen-(pkeylen-overhead-CIPHER_KEY_LEN);
-
-  outlen = crypto_pk_public_encrypt(env,to,tolen,buf,pkeylen-overhead,padding);
-  if (outlen!=(int)pkeylen) {
-    goto err;
-  }
-  r = crypto_cipher_encrypt(cipher, to+outlen,
-                            from+pkeylen-overhead-CIPHER_KEY_LEN, symlen);
-
-  if (r<0) goto err;
-  memwipe(buf, 0, pkeylen);
-  memwipe(key, 0, sizeof(key));
-  tor_free(buf);
-  crypto_cipher_free(cipher);
-  tor_assert(outlen+symlen < INT_MAX);
-  return (int)(outlen + symlen);
- err:
-
-  memwipe(buf, 0, pkeylen);
-  memwipe(key, 0, sizeof(key));
-  tor_free(buf);
-  crypto_cipher_free(cipher);
-  return -1;
-}
-
-/** Invert crypto_pk_obsolete_public_hybrid_encrypt. Returns the number of
- * bytes written on success, -1 on failure.
- *
- * NOTE that this format does not authenticate the symmetrically encrypted
- * part of the data, and SHOULD NOT BE USED for new protocols.
- */
-int
-crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env,
-                                 char *to,
-                                 size_t tolen,
-                                 const char *from,
-                                 size_t fromlen,
-                                 int padding, int warnOnFailure)
-{
-  int outlen, r;
-  size_t pkeylen;
-  crypto_cipher_t *cipher = NULL;
-  char *buf = NULL;
-
-  tor_assert(fromlen < SIZE_T_CEILING);
-  pkeylen = crypto_pk_keysize(env);
-
-  if (fromlen <= pkeylen) {
-    return crypto_pk_private_decrypt(env,to,tolen,from,fromlen,padding,
-                                     warnOnFailure);
-  }
-
-  buf = tor_malloc(pkeylen);
-  outlen = crypto_pk_private_decrypt(env,buf,pkeylen,from,pkeylen,padding,
-                                     warnOnFailure);
-  if (outlen<0) {
-    log_fn(warnOnFailure?LOG_WARN:LOG_DEBUG, LD_CRYPTO,
-           "Error decrypting public-key data");
-    goto err;
-  }
-  if (outlen < CIPHER_KEY_LEN) {
-    log_fn(warnOnFailure?LOG_WARN:LOG_INFO, LD_CRYPTO,
-           "No room for a symmetric key");
-    goto err;
-  }
-  cipher = crypto_cipher_new(buf);
-  if (!cipher) {
-    goto err;
-  }
-  memcpy(to,buf+CIPHER_KEY_LEN,outlen-CIPHER_KEY_LEN);
-  outlen -= CIPHER_KEY_LEN;
-  tor_assert(tolen - outlen >= fromlen - pkeylen);
-  r = crypto_cipher_decrypt(cipher, to+outlen, from+pkeylen, fromlen-pkeylen);
-  if (r<0)
-    goto err;
-  memwipe(buf,0,pkeylen);
-  tor_free(buf);
-  crypto_cipher_free(cipher);
-  tor_assert(outlen + fromlen < INT_MAX);
-  return (int)(outlen + (fromlen-pkeylen));
- err:
-  memwipe(buf,0,pkeylen);
-  tor_free(buf);
-  crypto_cipher_free(cipher);
-  return -1;
-}
-
 /** Encrypt <b>fromlen</b> bytes from <b>from</b> with the public key
  * in <b>env</b>, using the padding method <b>padding</b>.  On success,
  * write the result to <b>to</b>, and return the number of bytes
@@ -993,122 +849,6 @@ crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out)
   return 0;
 }
 
-/** Check a siglen-byte long signature at <b>sig</b> against
- * <b>datalen</b> bytes of data at <b>data</b>, using the public key
- * in <b>env</b>. Return 0 if <b>sig</b> is a correct signature for
- * SHA1(data).  Else return -1.
- */
-MOCK_IMPL(int,
-crypto_pk_public_checksig_digest,(crypto_pk_t *env, const char *data,
-                                  size_t datalen, const char *sig,
-                                  size_t siglen))
-{
-  char digest[DIGEST_LEN];
-  char *buf;
-  size_t buflen;
-  int r;
-
-  tor_assert(env);
-  tor_assert(data);
-  tor_assert(sig);
-  tor_assert(datalen < SIZE_T_CEILING);
-  tor_assert(siglen < SIZE_T_CEILING);
-
-  if (crypto_digest(digest,data,datalen)<0) {
-    log_warn(LD_BUG, "couldn't compute digest");
-    return -1;
-  }
-  buflen = crypto_pk_keysize(env);
-  buf = tor_malloc(buflen);
-  r = crypto_pk_public_checksig(env,buf,buflen,sig,siglen);
-  if (r != DIGEST_LEN) {
-    log_warn(LD_CRYPTO, "Invalid signature");
-    tor_free(buf);
-    return -1;
-  }
-  if (tor_memneq(buf, digest, DIGEST_LEN)) {
-    log_warn(LD_CRYPTO, "Signature mismatched with digest.");
-    tor_free(buf);
-    return -1;
-  }
-  tor_free(buf);
-
-  return 0;
-}
-
-/** Compute a SHA1 digest of <b>fromlen</b> bytes of data stored at
- * <b>from</b>; sign the data with the private key in <b>env</b>, and
- * store it in <b>to</b>.  Return the number of bytes written on
- * success, and -1 on failure.
- *
- * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
- * at least the length of the modulus of <b>env</b>.
- */
-int
-crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen,
-                              const char *from, size_t fromlen)
-{
-  int r;
-  char digest[DIGEST_LEN];
-  if (crypto_digest(digest,from,fromlen)<0)
-    return -1;
-  r = crypto_pk_private_sign(env,to,tolen,digest,DIGEST_LEN);
-  memwipe(digest, 0, sizeof(digest));
-  return r;
-}
-
-/** Given a private or public key <b>pk</b>, put a SHA1 hash of the
- * public key into <b>digest_out</b> (must have DIGEST_LEN bytes of space).
- * Return 0 on success, -1 on failure.
- */
-int
-crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out)
-{
-  char *buf;
-  size_t buflen;
-  int len;
-  int rv = -1;
-
-  buflen = crypto_pk_keysize(pk)*2;
-  buf = tor_malloc(buflen);
-  len = crypto_pk_asn1_encode(pk, buf, buflen);
-  if (len < 0)
-    goto done;
-
-  if (crypto_digest(digest_out, buf, len) < 0)
-    goto done;
-
-  rv = 0;
-  done:
-  tor_free(buf);
-  return rv;
-}
-
-/** Compute all digests of the DER encoding of <b>pk</b>, and store them
- * in <b>digests_out</b>.  Return 0 on success, -1 on failure. */
-int
-crypto_pk_get_common_digests(crypto_pk_t *pk, common_digests_t *digests_out)
-{
-  char *buf;
-  size_t buflen;
-  int len;
-  int rv = -1;
-
-  buflen = crypto_pk_keysize(pk)*2;
-  buf = tor_malloc(buflen);
-  len = crypto_pk_asn1_encode(pk, buf, buflen);
-  if (len < 0)
-    goto done;
-
-  if (crypto_common_digests(digests_out, (char*)buf, len) < 0)
-    goto done;
-
-  rv = 0;
- done:
-  tor_free(buf);
-  return rv;
-}
-
 /** Given a crypto_pk_t <b>pk</b>, allocate a new buffer containing the
  * Base64 encoding of the DER representation of the private key as a NUL
  * terminated string, and return it via <b>priv_out</b>.  Return 0 on

src/common/crypto_rsa.h

@@ -15,13 +15,13 @@
 
 #include "orconfig.h"
 
-#include "crypto_digest.h"
 #include <stdio.h>
 #include "torint.h"
 #include "testsupport.h"
 #include "compat.h"
 #include "util.h"
 #include "torlog.h"
+#include "crypto_curve25519.h"
 
 /** Length of our public keys. */
 #define PK_BYTES (1024/8)
@@ -35,7 +35,7 @@
 /** A public key, or a public/private key-pair. */
 typedef struct crypto_pk_t crypto_pk_t;
 
-/* RSA environment setup */
+/* RSA enviroment setup */
 MOCK_DECL(crypto_pk_t *,crypto_pk_new,(void));
 void crypto_pk_free_(crypto_pk_t *env);
 #define crypto_pk_free(pk) FREE_AND_NULL(crypto_pk_t, crypto_pk_free_, (pk))
@@ -69,14 +69,6 @@ crypto_pk_t *crypto_pk_dup_key(crypto_pk_t *orig);
 crypto_pk_t *crypto_pk_copy_full(crypto_pk_t *orig);
 int crypto_pk_key_is_private(const crypto_pk_t *key);
 int crypto_pk_public_exponent_ok(crypto_pk_t *env);
-int crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env, char *to,
-                                    size_t tolen,
-                                    const char *from, size_t fromlen,
-                                    int padding, int force);
-int crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env, char *to,
-                                     size_t tolen,
-                                     const char *from, size_t fromlen,
-                                     int padding, int warnOnFailure);
 int crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen,
                              const char *from, size_t fromlen, int padding);
 int crypto_pk_private_decrypt(crypto_pk_t *env, char *to, size_t tolen,
@@ -92,13 +84,6 @@ crypto_pk_t *crypto_pk_asn1_decode(const char *str, size_t len);
 int crypto_pk_get_fingerprint(crypto_pk_t *pk, char *fp_out,int add_space);
 int crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out);
 
-MOCK_DECL(int, crypto_pk_public_checksig_digest,(crypto_pk_t *env,
-          const char *data, size_t datalen, const char *sig, size_t siglen));
-int crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen,
-                                  const char *from, size_t fromlen);
-int crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out);
-int crypto_pk_get_common_digests(crypto_pk_t *pk,
-                                 common_digests_t *digests_out);
 int crypto_pk_base64_encode(const crypto_pk_t *pk, char **priv_out);
 crypto_pk_t *crypto_pk_base64_decode(const char *str, size_t len);
 

src/common/crypto_s2k.c

@@ -12,13 +12,10 @@
 
 #define CRYPTO_S2K_PRIVATE
 
-#include "compat.h"
 #include "crypto.h"
-#include "crypto_digest.h"
-#include "crypto_rand.h"
-#include "crypto_s2k.h"
-#include "crypto_util.h"
 #include "util.h"
+#include "compat.h"
+#include "crypto_s2k.h"
 
 #include <openssl/evp.h>
 

src/common/crypto_util.c

@@ -1,107 +0,0 @@
-/* Copyright (c) 2001, Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2018, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file crypto_util.c
- *
- * \brief Common cryptographic utilities.
- **/
-
-#ifndef CRYPTO_UTIL_PRIVATE
-#define CRYPTO_UTIL_PRIVATE
-
-#include "crypto_util.h"
-
-#include <string.h>
-
-#ifdef _WIN32
-#include <winsock2.h>
-#include <windows.h>
-#include <wincrypt.h>
-#endif /* defined(_WIN32) */
-
-#include "util.h"
-
-DISABLE_GCC_WARNING(redundant-decls)
-
-#include <openssl/crypto.h>
-
-ENABLE_GCC_WARNING(redundant-decls)
-
-/**
- * Destroy the <b>sz</b> bytes of data stored at <b>mem</b>, setting them to
- * the value <b>byte</b>.
- * If <b>mem</b> is NULL or <b>sz</b> is zero, nothing happens.
- *
- * This function is preferable to memset, since many compilers will happily
- * optimize out memset() when they can convince themselves that the data being
- * cleared will never be read.
- *
- * Right now, our convention is to use this function when we are wiping data
- * that's about to become inaccessible, such as stack buffers that are about
- * to go out of scope or structures that are about to get freed.  (In
- * practice, it appears that the compilers we're currently using will optimize
- * out the memset()s for stack-allocated buffers, but not those for
- * about-to-be-freed structures. That could change, though, so we're being
- * wary.)  If there are live reads for the data, then you can just use
- * memset().
- */
-void
-memwipe(void *mem, uint8_t byte, size_t sz)
-{
-  if (sz == 0) {
-    return;
-  }
-  /* If sz is nonzero, then mem must not be NULL. */
-  tor_assert(mem != NULL);
-
-  /* Data this large is likely to be an underflow. */
-  tor_assert(sz < SIZE_T_CEILING);
-
-  /* Because whole-program-optimization exists, we may not be able to just
-   * have this function call "memset".  A smart compiler could inline it, then
-   * eliminate dead memsets, and declare itself to be clever. */
-
-#if defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY)
-  /* Here's what you do on windows. */
-  SecureZeroMemory(mem,sz);
-#elif defined(HAVE_RTLSECUREZEROMEMORY)
-  RtlSecureZeroMemory(mem,sz);
-#elif defined(HAVE_EXPLICIT_BZERO)
-  /* The BSDs provide this. */
-  explicit_bzero(mem, sz);
-#elif defined(HAVE_MEMSET_S)
-  /* This is in the C99 standard. */
-  memset_s(mem, sz, 0, sz);
-#else
-  /* This is a slow and ugly function from OpenSSL that fills 'mem' with junk
-   * based on the pointer value, then uses that junk to update a global
-   * variable.  It's an elaborate ruse to trick the compiler into not
-   * optimizing out the "wipe this memory" code.  Read it if you like zany
-   * programming tricks! In later versions of Tor, we should look for better
-   * not-optimized-out memory wiping stuff...
-   *
-   * ...or maybe not.  In practice, there are pure-asm implementations of
-   * OPENSSL_cleanse() on most platforms, which ought to do the job.
-   **/
-
-  OPENSSL_cleanse(mem, sz);
-#endif /* defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY) || ... */
-
-  /* Just in case some caller of memwipe() is relying on getting a buffer
-   * filled with a particular value, fill the buffer.
-   *
-   * If this function gets inlined, this memset might get eliminated, but
-   * that's okay: We only care about this particular memset in the case where
-   * the caller should have been using memset(), and the memset() wouldn't get
-   * eliminated.  In other words, this is here so that we won't break anything
-   * if somebody accidentally calls memwipe() instead of memset().
-   **/
-  memset(mem, byte, sz);
-}
-
-#endif /* !defined(CRYPTO_UTIL_PRIVATE) */
-

src/common/crypto_util.h

@@ -1,27 +0,0 @@
-/* Copyright (c) 2001, Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2018, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file crypto_util.h
- *
- * \brief Common functions for cryptographic routines.
- **/
-
-#ifndef TOR_CRYPTO_UTIL_H
-#define TOR_CRYPTO_UTIL_H
-
-#include "torint.h"
-
-/** OpenSSL-based utility functions. */
-void memwipe(void *mem, uint8_t byte, size_t sz);
-
-#ifdef CRYPTO_UTIL_PRIVATE
-#ifdef TOR_UNIT_TESTS
-#endif /* defined(TOR_UNIT_TESTS) */
-#endif /* defined(CRYPTO_UTIL_PRIVATE) */
-
-#endif /* !defined(TOR_CRYPTO_UTIL_H) */
-

src/common/include.am

@@ -97,7 +97,6 @@ LIBOR_A_SRC = \
   src/common/util_process.c				\
   src/common/sandbox.c					\
   src/common/storagedir.c				\
-  src/common/token_bucket.c				\
   src/common/workqueue.c				\
   $(libor_extra_source)					\
   $(threads_impl_source)				\
@@ -115,14 +114,11 @@ LIBOR_CRYPTO_A_SRC = \
   src/common/compress_zlib.c	\
   src/common/compress_zstd.c	\
   src/common/crypto.c		\
-  src/common/crypto_digest.c     \
-  src/common/crypto_format.c	\
+  src/common/crypto_rsa.c     \
   src/common/crypto_openssl_mgt.c    \
   src/common/crypto_pwbox.c     \
-  src/common/crypto_rand.c     \
-  src/common/crypto_rsa.c     \
   src/common/crypto_s2k.c	\
-  src/common/crypto_util.c	\
+  src/common/crypto_format.c	\
   src/common/tortls.c		\
   src/common/crypto_curve25519.c \
   src/common/crypto_ed25519.c
@@ -169,16 +165,13 @@ COMMONHEADERS = \
   src/common/confline.h				\
   src/common/container.h			\
   src/common/crypto.h				\
-  src/common/crypto_digest.h            \
   src/common/crypto_curve25519.h		\
   src/common/crypto_ed25519.h			\
   src/common/crypto_format.h			\
   src/common/crypto_openssl_mgt.h         \
-  src/common/crypto_pwbox.h			\
-  src/common/crypto_rand.h          \
   src/common/crypto_rsa.h           \
+  src/common/crypto_pwbox.h			\
   src/common/crypto_s2k.h			\
-  src/common/crypto_util.h			\
   src/common/di_ops.h				\
   src/common/handles.h				\
   src/common/memarea.h				\
@@ -189,7 +182,6 @@ COMMONHEADERS = \
   src/common/storagedir.h			\
   src/common/testsupport.h			\
   src/common/timers.h				\
-  src/common/token_bucket.h			\
   src/common/torint.h				\
   src/common/torlog.h				\
   src/common/tortls.h				\

src/common/log.c

@@ -52,13 +52,6 @@
 
 #define raw_assert(x) assert(x) // assert OK
 
-/** Defining compile-time constants for Tor log levels (used by the Rust
- * log wrapper at src/rust/tor_log) */
-const int LOG_WARN_ = LOG_WARN;
-const int LOG_NOTICE_ = LOG_NOTICE;
-const log_domain_mask_t LD_GENERAL_ = LD_GENERAL;
-const log_domain_mask_t LD_NET_ = LD_NET;
-
 /** Information for a single logfile; only used in log.c */
 typedef struct logfile_t {
   struct logfile_t *next; /**< Next logfile_t in the linked list. */
@@ -170,9 +163,6 @@ typedef struct pending_log_message_t {
 /** Log messages waiting to be replayed onto callback-based logs */
 static smartlist_t *pending_cb_messages = NULL;
 
-/** Callback to invoke when pending_cb_messages becomes nonempty. */
-static pending_callback_callback pending_cb_cb = NULL;
-
 /** Log messages waiting to be replayed once the logging system is initialized.
  */
 static smartlist_t *pending_startup_messages = NULL;
@@ -235,30 +225,6 @@ log_set_application_name(const char *name)
   appname = name ? tor_strdup(name) : NULL;
 }
 
-/** Return true if some of the running logs might be interested in a log
- * message of the given severity in the given domains. If this function
- * returns true, the log message might be ignored anyway, but if it returns
- * false, it is definitely_ safe not to log the message. */
-int
-log_message_is_interesting(int severity, log_domain_mask_t domain)
-{
-  (void) domain;
-  return (severity <= log_global_min_severity_);
-}
-
-/**
- * As tor_log, but takes an optional function name, and does not treat its
- * <b>string</b> as a printf format.
- *
- * For use by Rust integration.
- */
-void
-tor_log_string(int severity, log_domain_mask_t domain,
-               const char *function, const char *string)
-{
-  log_fn_(severity, domain, function, "%s", string);
-}
-
 /** Log time granularity in milliseconds. */
 static int log_time_granularity = 1;
 
@@ -541,9 +507,6 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len,
         smartlist_add(pending_cb_messages,
             pending_log_message_new(severity,domain,NULL,msg_after_prefix));
         *callbacks_deferred = 1;
-        if (smartlist_len(pending_cb_messages) == 1 && pending_cb_cb) {
-          pending_cb_cb();
-        }
       }
     } else {
       lf->callback(severity, domain, msg_after_prefix);
@@ -831,7 +794,6 @@ logs_free_all(void)
   logfiles = NULL;
   messages = pending_cb_messages;
   pending_cb_messages = NULL;
-  pending_cb_cb = NULL;
   messages2 = pending_startup_messages;
   pending_startup_messages = NULL;
   UNLOCK_LOGS();
@@ -994,24 +956,6 @@ add_temp_log(int min_severity)
   UNLOCK_LOGS();
 }
 
-/**
- * Register "cb" as the callback to call when there are new pending log
- * callbacks to be flushed with flush_pending_log_callbacks().
- *
- * Note that this callback, if present, can be invoked from any thread.
- *
- * This callback must not log.
- *
- * It is intentional that this function contains the name "callback" twice: it
- * sets a "callback" to be called on the condition that there is a "pending
- * callback".
- **/
-void
-logs_set_pending_callback_callback(pending_callback_callback cb)
-{
-  pending_cb_cb = cb;
-}
-
 /**
  * Add a log handler to send messages in <b>severity</b>
  * to the function <b>cb</b>.

src/common/procmon.c

@@ -10,6 +10,8 @@
 
 #include "util.h"
 
+#include <event2/event.h>
+
 #ifdef HAVE_SIGNAL_H
 #include <signal.h>
 #endif
@@ -42,7 +44,7 @@ typedef int pid_t;
 /* Currently we need to poll in some way on all systems. */
 
 #ifdef PROCMON_POLLS
-static void tor_process_monitor_poll_cb(periodic_timer_t *ev,
+static void tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2,
                                         void *procmon_);
 #endif
 
@@ -134,7 +136,7 @@ struct tor_process_monitor_t {
 
   /** A Libevent event structure, to either poll for the process's
    * existence or receive a notification when the process ends. */
-  periodic_timer_t *e;
+  struct event *e;
 
   /** A callback to be called when the process ends. */
   tor_procmon_callback_t cb;
@@ -157,6 +159,9 @@ tor_validate_process_specifier(const char *process_spec,
   return parse_process_specifier(process_spec, &ppspec, msg);
 }
 
+/* XXXX we should use periodic_timer_new() for this stuff */
+#define PERIODIC_TIMER_FLAGS EV_PERSIST
+
 /* DOCDOC poll_interval_tv */
 static const struct timeval poll_interval_tv = {15, 0};
 
@@ -220,9 +225,13 @@ tor_process_monitor_new(struct event_base *base,
   procmon->cb_arg = cb_arg;
 
 #ifdef PROCMON_POLLS
-  procmon->e = periodic_timer_new(base,
-                                  &poll_interval_tv,
-                                  tor_process_monitor_poll_cb, procmon);
+  procmon->e = tor_event_new(base, -1 /* no FD */, PERIODIC_TIMER_FLAGS,
+                             tor_process_monitor_poll_cb, procmon);
+  /* Note: If you port this file to plain Libevent 2, check that
+   * procmon->e is non-NULL.  We don't need to here because
+   * tor_evtimer_new never returns NULL. */
+
+  evtimer_add(procmon->e, &poll_interval_tv);
 #else /* !(defined(PROCMON_POLLS)) */
 #error OOPS?
 #endif /* defined(PROCMON_POLLS) */
@@ -237,12 +246,14 @@ tor_process_monitor_new(struct event_base *base,
 /** Libevent callback to poll for the existence of the process
  * monitored by <b>procmon_</b>. */
 static void
-tor_process_monitor_poll_cb(periodic_timer_t *event, void *procmon_)
+tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2,
+                            void *procmon_)
 {
-  (void)event;
   tor_process_monitor_t *procmon = (tor_process_monitor_t *)(procmon_);
   int its_dead_jim;
 
+  (void)unused1; (void)unused2;
+
   tor_assert(procmon != NULL);
 
 #ifdef _WIN32
@@ -325,7 +336,7 @@ tor_process_monitor_free_(tor_process_monitor_t *procmon)
 #endif
 
   if (procmon->e != NULL)
-    periodic_timer_free(procmon->e);
+    tor_event_free(procmon->e);
 
   tor_free(procmon);
 }

src/common/timers.c

@@ -37,6 +37,8 @@
 #include "torlog.h"
 #include "util.h"
 
+#include <event2/event.h>
+
 struct timeout_cb {
   timer_cb_fn_t cb;
   void *arg;
@@ -64,15 +66,10 @@ struct timeout_cb {
  * above TIMEOUT_MAX can also be super-inefficient. Choosing 5 here sets
  * timeout_max to 2^30 ticks, or 29 hours with our value for USEC_PER_TICK */
 #define WHEEL_NUM 5
-#if SIZEOF_VOID_P == 4
-/* On 32-bit platforms, we want to override wheel_bit, so that timeout.c will
- * use 32-bit math. */
-#define WHEEL_BIT 5
-#endif
 #include "src/ext/timeouts/timeout.c"
 
 static struct timeouts *global_timeouts = NULL;
-static struct mainloop_event_t *global_timer_event = NULL;
+static struct event *global_timer_event = NULL;
 
 static monotime_t start_of_time;
 
@@ -150,7 +147,7 @@ libevent_timer_reschedule(void)
   if (delay > MIN_CHECK_TICKS)
     delay = MIN_CHECK_TICKS;
   timeout_to_tv(delay, &d);
-  mainloop_event_schedule(global_timer_event, &d);
+  event_add(global_timer_event, &d);
 }
 
 /** Run the callback of every timer that has expired, based on the current
@@ -173,9 +170,10 @@ timers_run_pending(void)
  * have fired, activate their callbacks, and reschedule the libevent timer.
  */
 static void
-libevent_timer_callback(mainloop_event_t *ev, void *arg)
+libevent_timer_callback(evutil_socket_t fd, short what, void *arg)
 {
-  (void)ev;
+  (void)fd;
+  (void)what;
   (void)arg;
 
   timers_run_pending();
@@ -205,8 +203,9 @@ timers_initialize(void)
   monotime_init();
   monotime_get(&start_of_time);
 
-  mainloop_event_t *timer_event;
-  timer_event = mainloop_event_new(libevent_timer_callback, NULL);
+  struct event *timer_event;
+  timer_event = tor_event_new(tor_libevent_get_base(),
+                              -1, 0, libevent_timer_callback, NULL);
   tor_assert(timer_event);
   global_timer_event = timer_event;
 
@@ -220,7 +219,7 @@ void
 timers_shutdown(void)
 {
   if (global_timer_event) {
-    mainloop_event_free(global_timer_event);
+    tor_event_free(global_timer_event);
     global_timer_event = NULL;
   }
   if (global_timeouts) {

src/common/token_bucket.c

@@ -1,255 +0,0 @@
-/* Copyright (c) 2018, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file token_bucket.c
- * \brief Functions to use and manipulate token buckets, used for
- *    rate-limiting on connections and globally.
- *
- * Tor uses these token buckets to keep track of bandwidth usage, and
- * sometimes other things too.
- *
- * There are two layers of abstraction here: "raw" token buckets, in which all
- * the pieces are decoupled, and "read-write" token buckets, which combine all
- * the moving parts into one.
- *
- * Token buckets may become negative.
- **/
-
-#define TOKEN_BUCKET_PRIVATE
-
-#include "token_bucket.h"
-#include "util_bug.h"
-
-/**
- * Set the <b>rate</b> and <b>burst</b> value in a token_bucket_cfg.
- *
- * Note that the <b>rate</b> value is in arbitrary units, but those units will
- * determine the units of token_bucket_raw_dec(), token_bucket_raw_refill, and
- * so on.
- */
-void
-token_bucket_cfg_init(token_bucket_cfg_t *cfg,
-                      uint32_t rate,
-                      uint32_t burst)
-{
-  tor_assert_nonfatal(rate > 0);
-  tor_assert_nonfatal(burst > 0);
-  if (burst > TOKEN_BUCKET_MAX_BURST)
-    burst = TOKEN_BUCKET_MAX_BURST;
-
-  cfg->rate = rate;
-  cfg->burst = burst;
-}
-
-/**
- * Initialize a raw token bucket and its associated timestamp to the "full"
- * state, according to <b>cfg</b>.
- */
-void
-token_bucket_raw_reset(token_bucket_raw_t *bucket,
-                       const token_bucket_cfg_t *cfg)
-{
-  bucket->bucket = cfg->burst;
-}
-
-/**
- * Adust a preexisting token bucket to respect the new configuration
- * <b>cfg</b>, by decreasing its current level if needed. */
-void
-token_bucket_raw_adjust(token_bucket_raw_t *bucket,
-                        const token_bucket_cfg_t *cfg)
-{
-  bucket->bucket = MIN(bucket->bucket, cfg->burst);
-}
-
-/**
- * Given an amount of <b>elapsed</b> time units, and a bucket configuration
- * <b>cfg</b>, refill the level of <b>bucket</b> accordingly.  Note that the
- * units of time in <b>elapsed</b> must correspond to those used to set the
- * rate in <b>cfg</b>, or the result will be illogical.
- */
-int
-token_bucket_raw_refill_steps(token_bucket_raw_t *bucket,
-                              const token_bucket_cfg_t *cfg,
-                              const uint32_t elapsed)
-{
-  const int was_empty = (bucket->bucket <= 0);
-  /* The casts here prevent an underflow.
-   *
-   * Note that even if the bucket value is negative, subtracting it from
-   * "burst" will still produce a correct result.  If this result is
-   * ridiculously high, then the "elapsed > gap / rate" check below
-   * should catch it. */
-  const size_t gap = ((size_t)cfg->burst) - ((size_t)bucket->bucket);
-
-  if (elapsed > gap / cfg->rate) {
-    bucket->bucket = cfg->burst;
-  } else {
-    bucket->bucket += cfg->rate * elapsed;
-  }
-
-  return was_empty && bucket->bucket > 0;
-}
-
-/**
- * Decrement a provided bucket by <b>n</b> units.  Note that <b>n</b>
- * must be nonnegative.
- */
-int
-token_bucket_raw_dec(token_bucket_raw_t *bucket,
-                     ssize_t n)
-{
-  if (BUG(n < 0))
-    return 0;
-  const int becomes_empty = bucket->bucket > 0 && n >= bucket->bucket;
-  bucket->bucket -= n;
-  return becomes_empty;
-}
-
-/** Convert a rate in bytes per second to a rate in bytes per step */
-STATIC uint32_t
-rate_per_sec_to_rate_per_step(uint32_t rate)
-{
-  /*
-    The precise calculation we'd want to do is
-
-    (rate / 1000) * to_approximate_msec(TICKS_PER_STEP).  But to minimize
-    rounding error, we do it this way instead, and divide last.
-  */
-  uint64_t units = (uint64_t) rate * TICKS_PER_STEP;
-  uint32_t val = (uint32_t)
-    (monotime_coarse_stamp_units_to_approx_msec(units) / 1000);
-  return val ? val : 1;
-}
-
-/**
- * Initialize a token bucket in *<b>bucket</b>, set up to allow <b>rate</b>
- * bytes per second, with a maximum burst of <b>burst</b> bytes. The bucket
- * is created such that <b>now_ts</b> is the current timestamp.  The bucket
- * starts out full.
- */
-void
-token_bucket_rw_init(token_bucket_rw_t *bucket,
-                     uint32_t rate,
-                     uint32_t burst,
-                     uint32_t now_ts)
-{
-  memset(bucket, 0, sizeof(token_bucket_rw_t));
-  token_bucket_rw_adjust(bucket, rate, burst);
-  token_bucket_rw_reset(bucket, now_ts);
-}
-
-/**
- * Change the configured rate (in bytes per second) and burst (in bytes)
- * for the token bucket in *<b>bucket</b>.
- */
-void
-token_bucket_rw_adjust(token_bucket_rw_t *bucket,
-                       uint32_t rate,
-                       uint32_t burst)
-{
-  token_bucket_cfg_init(&bucket->cfg,
-                        rate_per_sec_to_rate_per_step(rate),
-                        burst);
-  token_bucket_raw_adjust(&bucket->read_bucket, &bucket->cfg);
-  token_bucket_raw_adjust(&bucket->write_bucket, &bucket->cfg);
-}
-
-/**
- * Reset <b>bucket</b> to be full, as of timestamp <b>now_ts</b>.
- */
-void
-token_bucket_rw_reset(token_bucket_rw_t *bucket,
-                      uint32_t now_ts)
-{
-  token_bucket_raw_reset(&bucket->read_bucket, &bucket->cfg);
-  token_bucket_raw_reset(&bucket->write_bucket, &bucket->cfg);
-  bucket->last_refilled_at_timestamp = now_ts;
-}
-
-/**
- * Refill <b>bucket</b> as appropriate, given that the current timestamp
- * is <b>now_ts</b>.
- *
- * Return a bitmask containing TB_READ iff read bucket was empty and became
- * nonempty, and TB_WRITE iff the write bucket was empty and became nonempty.
- */
-int
-token_bucket_rw_refill(token_bucket_rw_t *bucket,
-                       uint32_t now_ts)
-{
-  const uint32_t elapsed_ticks =
-    (now_ts - bucket->last_refilled_at_timestamp);
-  if (elapsed_ticks > UINT32_MAX-(300*1000)) {
-    /* Either about 48 days have passed since the last refill, or the
-     * monotonic clock has somehow moved backwards. (We're looking at you,
-     * Windows.).  We accept up to a 5 minute jump backwards as
-     * "unremarkable".
-     */
-    return 0;
-  }
-  const uint32_t elapsed_steps = elapsed_ticks / TICKS_PER_STEP;
-
-  if (!elapsed_steps) {
-    /* Note that if less than one whole step elapsed, we don't advance the
-     * time in last_refilled_at. That's intentional: we want to make sure
-     * that we add some bytes to it eventually. */
-    return 0;
-  }
-
-  int flags = 0;
-  if (token_bucket_raw_refill_steps(&bucket->read_bucket,
-                                    &bucket->cfg, elapsed_steps))
-    flags |= TB_READ;
-  if (token_bucket_raw_refill_steps(&bucket->write_bucket,
-                                    &bucket->cfg, elapsed_steps))
-    flags |= TB_WRITE;
-
-  bucket->last_refilled_at_timestamp = now_ts;
-  return flags;
-}
-
-/**
- * Decrement the read token bucket in <b>bucket</b> by <b>n</b> bytes.
- *
- * Return true if the bucket was nonempty and became empty; return false
- * otherwise.
- */
-int
-token_bucket_rw_dec_read(token_bucket_rw_t *bucket,
-                         ssize_t n)
-{
-  return token_bucket_raw_dec(&bucket->read_bucket, n);
-}
-
-/**
- * Decrement the write token bucket in <b>bucket</b> by <b>n</b> bytes.
- *
- * Return true if the bucket was nonempty and became empty; return false
- * otherwise.
- */
-int
-token_bucket_rw_dec_write(token_bucket_rw_t *bucket,
-                          ssize_t n)
-{
-  return token_bucket_raw_dec(&bucket->write_bucket, n);
-}
-
-/**
- * As token_bucket_rw_dec_read and token_bucket_rw_dec_write, in a single
- * operation.  Return a bitmask of TB_READ and TB_WRITE to indicate
- * which buckets became empty.
- */
-int
-token_bucket_rw_dec(token_bucket_rw_t *bucket,
-                    ssize_t n_read, ssize_t n_written)
-{
-  int flags = 0;
-  if (token_bucket_rw_dec_read(bucket, n_read))
-    flags |= TB_READ;
-  if (token_bucket_rw_dec_write(bucket, n_written))
-    flags |= TB_WRITE;
-  return flags;
-}
-

src/common/token_bucket.h

@@ -1,118 +0,0 @@
-/* Copyright (c) 2018, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file token_bucket_rw.h
- * \brief Headers for token_bucket_rw.c
- **/
-
-#ifndef TOR_TOKEN_BUCKET_H
-#define TOR_TOKEN_BUCKET_H
-
-#include "torint.h"
-#include "testsupport.h"
-
-/** Largest allowable burst value for a token buffer. */
-#define TOKEN_BUCKET_MAX_BURST INT32_MAX
-
-/** A generic token buffer configuration: determines the number of tokens
- * added to the bucket in each time unit (the "rate"), and the maximum number
- * of tokens in the bucket (the "burst") */
-typedef struct token_bucket_cfg_t {
-  uint32_t rate;
-  int32_t burst;
-} token_bucket_cfg_t;
-
-/** A raw token bucket, decoupled from its configuration and timestamp. */
-typedef struct token_bucket_raw_t {
-  int32_t bucket;
-} token_bucket_raw_t;
-
-void token_bucket_cfg_init(token_bucket_cfg_t *cfg,
-                           uint32_t rate,
-                           uint32_t burst);
-
-void token_bucket_raw_adjust(token_bucket_raw_t *bucket,
-                             const token_bucket_cfg_t *cfg);
-
-void token_bucket_raw_reset(token_bucket_raw_t *bucket,
-                            const token_bucket_cfg_t *cfg);
-
-int token_bucket_raw_dec(token_bucket_raw_t *bucket,
-                         ssize_t n);
-
-int token_bucket_raw_refill_steps(token_bucket_raw_t *bucket,
-                                  const token_bucket_cfg_t *cfg,
-                                  const uint32_t elapsed_steps);
-
-static inline size_t token_bucket_raw_get(const token_bucket_raw_t *bucket);
-/** Return the current number of bytes set in a token bucket. */
-static inline size_t
-token_bucket_raw_get(const token_bucket_raw_t *bucket)
-{
-  return bucket->bucket >= 0 ? bucket->bucket : 0;
-}
-
-/** A convenience type containing all the pieces needed for a coupled
- * read-bucket and write-bucket that have the same rate limit, and which use
- * "timestamp units" (see compat_time.h) for their time. */
-typedef struct token_bucket_rw_t {
-  token_bucket_cfg_t cfg;
-  token_bucket_raw_t read_bucket;
-  token_bucket_raw_t write_bucket;
-  uint32_t last_refilled_at_timestamp;
-} token_bucket_rw_t;
-
-void token_bucket_rw_init(token_bucket_rw_t *bucket,
-                          uint32_t rate,
-                          uint32_t burst,
-                          uint32_t now_ts);
-
-void token_bucket_rw_adjust(token_bucket_rw_t *bucket,
-                            uint32_t rate, uint32_t burst);
-
-void token_bucket_rw_reset(token_bucket_rw_t *bucket,
-                           uint32_t now_ts);
-
-#define TB_READ 1
-#define TB_WRITE 2
-
-int token_bucket_rw_refill(token_bucket_rw_t *bucket,
-                           uint32_t now_ts);
-
-int token_bucket_rw_dec_read(token_bucket_rw_t *bucket,
-                             ssize_t n);
-int token_bucket_rw_dec_write(token_bucket_rw_t *bucket,
-                              ssize_t n);
-
-int token_bucket_rw_dec(token_bucket_rw_t *bucket,
-                        ssize_t n_read, ssize_t n_written);
-
-static inline size_t token_bucket_rw_get_read(const token_bucket_rw_t *bucket);
-static inline size_t
-token_bucket_rw_get_read(const token_bucket_rw_t *bucket)
-{
-  return token_bucket_raw_get(&bucket->read_bucket);
-}
-
-static inline size_t token_bucket_rw_get_write(
-                                            const token_bucket_rw_t *bucket);
-static inline size_t
-token_bucket_rw_get_write(const token_bucket_rw_t *bucket)
-{
-  return token_bucket_raw_get(&bucket->write_bucket);
-}
-
-#ifdef TOKEN_BUCKET_PRIVATE
-
-/* To avoid making the rates too small, we consider units of "steps",
- * where a "step" is defined as this many timestamp ticks.  Keep this
- * a power of two if you can. */
-#define TICKS_PER_STEP 16
-
-STATIC uint32_t rate_per_sec_to_rate_per_step(uint32_t rate);
-
-#endif
-
-#endif /* TOR_TOKEN_BUCKET_H */
-

src/common/torint.h

@@ -40,8 +40,6 @@
 #include <inttypes.h>
 #endif
 
-#include <stdbool.h>
-
 #if (SIZEOF_INT8_T != 0)
 #define HAVE_INT8_T
 #endif

src/common/torlog.h

@@ -154,8 +154,6 @@ int add_android_log(const log_severity_list_t *severity,
                     const char *android_identity_tag);
 #endif // HAVE_ANDROID_LOG_H.
 int add_callback_log(const log_severity_list_t *severity, log_callback cb);
-typedef void (*pending_callback_callback)(void);
-void logs_set_pending_callback_callback(pending_callback_callback cb);
 void logs_set_domain_logging(int enabled);
 int get_min_log_level(void);
 void switch_logs_debug(void);
@@ -193,10 +191,6 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity,
                      const char *format, ...)
   CHECK_PRINTF(5,6);
 
-int log_message_is_interesting(int severity, log_domain_mask_t domain);
-void tor_log_string(int severity, log_domain_mask_t domain,
-                    const char *function, const char *string);
-
 #if defined(__GNUC__) && __GNUC__ <= 3
 
 /* These are the GCC varidaic macros, so that older versions of GCC don't
@@ -254,16 +248,6 @@ void tor_log_string(int severity, log_domain_mask_t domain,
                   args, ##__VA_ARGS__)
 #endif /* defined(__GNUC__) && __GNUC__ <= 3 */
 
-/** This defines log levels that are linked in the Rust log module, rather
- * than re-defining these in both Rust and C.
- *
- * C_RUST_COUPLED src/rust/tor_log LogSeverity, LogDomain
- */
-extern const int LOG_WARN_;
-extern const int LOG_NOTICE_;
-extern const log_domain_mask_t LD_NET_;
-extern const log_domain_mask_t LD_GENERAL_;
-
 #ifdef LOG_PRIVATE
 MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain,
     const char *funcname, const char *suffix, const char *format,

src/common/tortls.c

@@ -25,9 +25,6 @@
   #include <ws2tcpip.h>
 #endif
 
-#include "crypto.h"
-#include "crypto_rand.h"
-#include "crypto_util.h"
 #include "compat.h"
 
 /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in
@@ -35,6 +32,7 @@
 DISABLE_GCC_WARNING(redundant-decls)
 
 #include <openssl/opensslv.h>
+#include "crypto.h"
 
 #ifdef OPENSSL_NO_EC
 #error "We require OpenSSL with ECC support"
@@ -58,25 +56,10 @@ ENABLE_GCC_WARNING(redundant-decls)
 #include "container.h"
 #include <string.h>
 
-#ifdef OPENSSL_1_1_API
-#define X509_get_notBefore_const(cert) \
-    X509_get0_notBefore(cert)
-#define X509_get_notAfter_const(cert) \
-    X509_get0_notAfter(cert)
-#ifndef X509_get_notBefore
-#define X509_get_notBefore(cert) \
-    X509_getm_notBefore(cert)
-#endif
-#ifndef X509_get_notAfter
-#define X509_get_notAfter(cert) \
-    X509_getm_notAfter(cert)
-#endif
-#else /* ! OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */
 #define X509_get_notBefore_const(cert) \
   ((const ASN1_TIME*) X509_get_notBefore((X509 *)cert))
 #define X509_get_notAfter_const(cert) \
   ((const ASN1_TIME*) X509_get_notAfter((X509 *)cert))
-#endif
 
 /* Copied from or.h */
 #define LEGAL_NICKNAME_CHARACTERS \
@@ -372,12 +355,8 @@ tor_tls_init(void)
   check_no_tls_errors();
 
   if (!tls_library_is_initialized) {
-#ifdef OPENSSL_1_1_API
-    OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS, NULL);
-#else
     SSL_library_init();
     SSL_load_error_strings();
-#endif
 
 #if (SIZEOF_VOID_P >= 8 &&                              \
      OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,0,1))

src/common/tortls.h

@@ -11,7 +11,7 @@
  * \brief Headers for tortls.c
  **/
 
-#include "crypto_rsa.h"
+#include "crypto.h"
 #include "compat_openssl.h"
 #include "compat.h"
 #include "testsupport.h"

src/common/util.c

@@ -16,7 +16,7 @@
 #define UTIL_PRIVATE
 #include "util.h"
 #include "torlog.h"
-#include "crypto_digest.h"
+#include "crypto.h"
 #include "torint.h"
 #include "container.h"
 #include "address.h"
@@ -572,19 +572,6 @@ add_laplace_noise(int64_t signal_, double random_, double delta_f,
     return signal_ + noise;
 }
 
-/* Helper: safely add two uint32_t's, capping at UINT32_MAX rather
- * than overflow */
-uint32_t
-tor_add_u32_nowrap(uint32_t a, uint32_t b)
-{
-  /* a+b > UINT32_MAX check, without overflow */
-  if (PREDICT_UNLIKELY(a > UINT32_MAX - b)) {
-    return UINT32_MAX;
-  } else {
-    return a+b;
-  }
-}
-
 /* Helper: return greatest common divisor of a,b */
 static uint64_t
 gcd64(uint64_t a, uint64_t b)
@@ -1834,15 +1821,6 @@ format_iso_time(char *buf, time_t t)
   strftime(buf, ISO_TIME_LEN+1, "%Y-%m-%d %H:%M:%S", tor_gmtime_r(&t, &tm));
 }
 
-/** As format_local_iso_time, but use the yyyy-mm-ddThh:mm:ss format to avoid
- * embedding an internal space. */
-void
-format_local_iso_time_nospace(char *buf, time_t t)
-{
-  format_local_iso_time(buf, t);
-  buf[10] = 'T';
-}
-
 /** As format_iso_time, but use the yyyy-mm-ddThh:mm:ss format to avoid
  * embedding an internal space. */
 void
@@ -5133,6 +5111,30 @@ stream_status_to_string(enum stream_status stream_status)
   }
 }
 
+/* DOCDOC */
+static void
+log_portfw_spawn_error_message(const char *buf,
+                               const char *executable, int *child_status)
+{
+  /* Parse error message */
+  int retval, child_state, saved_errno;
+  retval = tor_sscanf(buf, SPAWN_ERROR_MESSAGE "%x/%x",
+                      &child_state, &saved_errno);
+  if (retval == 2) {
+    log_warn(LD_GENERAL,
+             "Failed to start child process \"%s\" in state %d: %s",
+             executable, child_state, strerror(saved_errno));
+    if (child_status)
+      *child_status = 1;
+  } else {
+    /* Failed to parse message from child process, log it as a
+       warning */
+    log_warn(LD_GENERAL,
+             "Unexpected message from port forwarding helper \"%s\": %s",
+             executable, buf);
+  }
+}
+
 #ifdef _WIN32
 
 /** Return a smartlist containing lines outputted from
@@ -5178,6 +5180,51 @@ tor_get_lines_from_handle, (HANDLE *handle,
   return lines;
 }
 
+/** Read from stream, and send lines to log at the specified log level.
+ * Returns -1 if there is a error reading, and 0 otherwise.
+ * If the generated stream is flushed more often than on new lines, or
+ * a read exceeds 256 bytes, lines will be truncated. This should be fixed,
+ * along with the corresponding problem on *nix (see bug #2045).
+ */
+static int
+log_from_handle(HANDLE *pipe, int severity)
+{
+  char buf[256];
+  int pos;
+  smartlist_t *lines;
+
+  pos = tor_read_all_handle(pipe, buf, sizeof(buf) - 1, NULL);
+  if (pos < 0) {
+    /* Error */
+    log_warn(LD_GENERAL, "Failed to read data from subprocess");
+    return -1;
+  }
+
+  if (0 == pos) {
+    /* There's nothing to read (process is busy or has exited) */
+    log_debug(LD_GENERAL, "Subprocess had nothing to say");
+    return 0;
+  }
+
+  /* End with a null even if there isn't a \r\n at the end */
+  /* TODO: What if this is a partial line? */
+  buf[pos] = '\0';
+  log_debug(LD_GENERAL, "Subprocess had %d bytes to say", pos);
+
+  /* Split up the buffer */
+  lines = smartlist_new();
+  tor_split_lines(lines, buf, pos);
+
+  /* Log each line */
+  SMARTLIST_FOREACH(lines, char *, line,
+  {
+    log_fn(severity, LD_GENERAL, "Port forwarding helper says: %s", line);
+  });
+  smartlist_free(lines);
+
+  return 0;
+}
+
 #else /* !(defined(_WIN32)) */
 
 /** Return a smartlist containing lines outputted from
@@ -5207,6 +5254,42 @@ tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out))
   return lines;
 }
 
+/** Read from fd, and send lines to log at the specified log level.
+ * Returns 1 if stream is closed normally, -1 if there is a error reading, and
+ * 0 otherwise. Handles lines from tor-fw-helper and
+ * tor_spawn_background() specially.
+ */
+static int
+log_from_pipe(int fd, int severity, const char *executable,
+              int *child_status)
+{
+  char buf[256];
+  enum stream_status r;
+
+  for (;;) {
+    r = get_string_from_pipe(fd, buf, sizeof(buf) - 1);
+
+    if (r == IO_STREAM_CLOSED) {
+      return 1;
+    } else if (r == IO_STREAM_EAGAIN) {
+      return 0;
+    } else if (r == IO_STREAM_TERM) {
+      return -1;
+    }
+
+    tor_assert(r == IO_STREAM_OKAY);
+
+    /* Check if buf starts with SPAWN_ERROR_MESSAGE */
+    if (strcmpstart(buf, SPAWN_ERROR_MESSAGE) == 0) {
+      log_portfw_spawn_error_message(buf, executable, child_status);
+    } else {
+      log_fn(severity, LD_GENERAL, "Port forwarding helper says: %s", buf);
+    }
+  }
+
+  /* We should never get here */
+  return -1;
+}
 #endif /* defined(_WIN32) */
 
 /** Reads from <b>fd</b> and stores input in <b>buf_out</b> making
@@ -5249,6 +5332,294 @@ get_string_from_pipe(int fd, char *buf_out, size_t count)
   return IO_STREAM_OKAY;
 }
 
+/** Parse a <b>line</b> from tor-fw-helper and issue an appropriate
+ *  log message to our user. */
+static void
+handle_fw_helper_line(const char *executable, const char *line)
+{
+  smartlist_t *tokens = smartlist_new();
+  char *message = NULL;
+  char *message_for_log = NULL;
+  const char *external_port = NULL;
+  const char *internal_port = NULL;
+  const char *result = NULL;
+  int port = 0;
+  int success = 0;
+
+  if (strcmpstart(line, SPAWN_ERROR_MESSAGE) == 0) {
+    /* We need to check for SPAWN_ERROR_MESSAGE again here, since it's
+     * possible that it got sent after we tried to read it in log_from_pipe.
+     *
+     * XXX Ideally, we should be using one of stdout/stderr for the real
+     * output, and one for the output of the startup code.  We used to do that
+     * before cd05f35d2c.
+     */
+    int child_status;
+    log_portfw_spawn_error_message(line, executable, &child_status);
+    goto done;
+  }
+
+  smartlist_split_string(tokens, line, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+
+  if (smartlist_len(tokens) < 5)
+    goto err;
+
+  if (strcmp(smartlist_get(tokens, 0), "tor-fw-helper") ||
+      strcmp(smartlist_get(tokens, 1), "tcp-forward"))
+    goto err;
+
+  external_port = smartlist_get(tokens, 2);
+  internal_port = smartlist_get(tokens, 3);
+  result = smartlist_get(tokens, 4);
+
+  if (smartlist_len(tokens) > 5) {
+    /* If there are more than 5 tokens, they are part of [<message>].
+       Let's use a second smartlist to form the whole message;
+       strncat loops suck. */
+    int i;
+    int message_words_n = smartlist_len(tokens) - 5;
+    smartlist_t *message_sl = smartlist_new();
+    for (i = 0; i < message_words_n; i++)
+      smartlist_add(message_sl, smartlist_get(tokens, 5+i));
+
+    tor_assert(smartlist_len(message_sl) > 0);
+    message = smartlist_join_strings(message_sl, " ", 0, NULL);
+
+    /* wrap the message in log-friendly wrapping */
+    tor_asprintf(&message_for_log, " ('%s')", message);
+
+    smartlist_free(message_sl);
+  }
+
+  port = atoi(external_port);
+  if (port < 1 || port > 65535)
+    goto err;
+
+  port = atoi(internal_port);
+  if (port < 1 || port > 65535)
+    goto err;
+
+  if (!strcmp(result, "SUCCESS"))
+    success = 1;
+  else if (!strcmp(result, "FAIL"))
+    success = 0;
+  else
+    goto err;
+
+  if (!success) {
+    log_warn(LD_GENERAL, "Tor was unable to forward TCP port '%s' to '%s'%s. "
+             "Please make sure that your router supports port "
+             "forwarding protocols (like NAT-PMP). Note that if '%s' is "
+             "your ORPort, your relay will be unable to receive inbound "
+             "traffic.", external_port, internal_port,
+             message_for_log ? message_for_log : "",
+             internal_port);
+  } else {
+    log_info(LD_GENERAL,
+             "Tor successfully forwarded TCP port '%s' to '%s'%s.",
+             external_port, internal_port,
+             message_for_log ? message_for_log : "");
+  }
+
+  goto done;
+
+ err:
+  log_warn(LD_GENERAL, "tor-fw-helper sent us a string we could not "
+           "parse (%s).", line);
+
+ done:
+  SMARTLIST_FOREACH(tokens, char *, cp, tor_free(cp));
+  smartlist_free(tokens);
+  tor_free(message);
+  tor_free(message_for_log);
+}
+
+/** Read what tor-fw-helper has to say in its stdout and handle it
+ *  appropriately */
+static int
+handle_fw_helper_output(const char *executable,
+                        process_handle_t *process_handle)
+{
+  smartlist_t *fw_helper_output = NULL;
+  enum stream_status stream_status = 0;
+
+  fw_helper_output =
+    tor_get_lines_from_handle(tor_process_get_stdout_pipe(process_handle),
+                              &stream_status);
+  if (!fw_helper_output) { /* didn't get any output from tor-fw-helper */
+    /* if EAGAIN we should retry in the future */
+    return (stream_status == IO_STREAM_EAGAIN) ? 0 : -1;
+  }
+
+  /* Handle the lines we got: */
+  SMARTLIST_FOREACH_BEGIN(fw_helper_output, char *, line) {
+    handle_fw_helper_line(executable, line);
+    tor_free(line);
+  } SMARTLIST_FOREACH_END(line);
+
+  smartlist_free(fw_helper_output);
+
+  return 0;
+}
+
+/** Spawn tor-fw-helper and ask it to forward the ports in
+ *  <b>ports_to_forward</b>. <b>ports_to_forward</b> contains strings
+ *  of the form "<external port>:<internal port>", which is the format
+ *  that tor-fw-helper expects. */
+void
+tor_check_port_forwarding(const char *filename,
+                          smartlist_t *ports_to_forward,
+                          time_t now)
+{
+/* When fw-helper succeeds, how long do we wait until running it again */
+#define TIME_TO_EXEC_FWHELPER_SUCCESS 300
+/* When fw-helper failed to start, how long do we wait until running it again
+ */
+#define TIME_TO_EXEC_FWHELPER_FAIL 60
+
+  /* Static variables are initialized to zero, so child_handle.status=0
+   * which corresponds to it not running on startup */
+  static process_handle_t *child_handle=NULL;
+
+  static time_t time_to_run_helper = 0;
+  int stderr_status, retval;
+  int stdout_status = 0;
+
+  tor_assert(filename);
+
+  /* Start the child, if it is not already running */
+  if ((!child_handle || child_handle->status != PROCESS_STATUS_RUNNING) &&
+      time_to_run_helper < now) {
+    /*tor-fw-helper cli looks like this: tor_fw_helper -p :5555 -p 4555:1111 */
+    const char **argv; /* cli arguments */
+    int args_n, status;
+    int argv_index = 0; /* index inside 'argv' */
+
+    tor_assert(smartlist_len(ports_to_forward) > 0);
+
+    /* check for overflow during 'argv' allocation:
+       (len(ports_to_forward)*2 + 2)*sizeof(char*) > SIZE_MAX ==
+       len(ports_to_forward) > (((SIZE_MAX/sizeof(char*)) - 2)/2) */
+    if ((size_t) smartlist_len(ports_to_forward) >
+        (((SIZE_MAX/sizeof(char*)) - 2)/2)) {
+      log_warn(LD_GENERAL,
+               "Overflow during argv allocation. This shouldn't happen.");
+      return;
+    }
+    /* check for overflow during 'argv_index' increase:
+       ((len(ports_to_forward)*2 + 2) > INT_MAX) ==
+       len(ports_to_forward) > (INT_MAX - 2)/2 */
+    if (smartlist_len(ports_to_forward) > (INT_MAX - 2)/2) {
+      log_warn(LD_GENERAL,
+               "Overflow during argv_index increase. This shouldn't happen.");
+      return;
+    }
+
+    /* Calculate number of cli arguments: one for the filename, two
+       for each smartlist element (one for "-p" and one for the
+       ports), and one for the final NULL. */
+    args_n = 1 + 2*smartlist_len(ports_to_forward) + 1;
+    argv = tor_calloc(args_n, sizeof(char *));
+
+    argv[argv_index++] = filename;
+    SMARTLIST_FOREACH_BEGIN(ports_to_forward, const char *, port) {
+      argv[argv_index++] = "-p";
+      argv[argv_index++] = port;
+    } SMARTLIST_FOREACH_END(port);
+    argv[argv_index] = NULL;
+
+    /* Assume tor-fw-helper will succeed, start it later*/
+    time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_SUCCESS;
+
+    if (child_handle) {
+      tor_process_handle_destroy(child_handle, 1);
+      child_handle = NULL;
+    }
+
+#ifdef _WIN32
+    /* Passing NULL as lpApplicationName makes Windows search for the .exe */
+    status = tor_spawn_background(NULL, argv, NULL, &child_handle);
+#else
+    status = tor_spawn_background(filename, argv, NULL, &child_handle);
+#endif /* defined(_WIN32) */
+
+    tor_free_((void*)argv);
+    argv=NULL;
+
+    if (PROCESS_STATUS_ERROR == status) {
+      log_warn(LD_GENERAL, "Failed to start port forwarding helper %s",
+              filename);
+      time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_FAIL;
+      return;
+    }
+
+    log_info(LD_GENERAL,
+             "Started port forwarding helper (%s) with pid '%d'",
+             filename, tor_process_get_pid(child_handle));
+  }
+
+  /* If child is running, read from its stdout and stderr) */
+  if (child_handle && PROCESS_STATUS_RUNNING == child_handle->status) {
+    /* Read from stdout/stderr and log result */
+    retval = 0;
+#ifdef _WIN32
+    stderr_status = log_from_handle(child_handle->stderr_pipe, LOG_INFO);
+#else
+    stderr_status = log_from_pipe(child_handle->stderr_pipe,
+                                  LOG_INFO, filename, &retval);
+#endif /* defined(_WIN32) */
+    if (handle_fw_helper_output(filename, child_handle) < 0) {
+      log_warn(LD_GENERAL, "Failed to handle fw helper output.");
+      stdout_status = -1;
+      retval = -1;
+    }
+
+    if (retval) {
+      /* There was a problem in the child process */
+      time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_FAIL;
+    }
+
+    /* Combine the two statuses in order of severity */
+    if (-1 == stdout_status || -1 == stderr_status)
+      /* There was a failure */
+      retval = -1;
+#ifdef _WIN32
+    else if (!child_handle || tor_get_exit_code(child_handle, 0, NULL) !=
+             PROCESS_EXIT_RUNNING) {
+      /* process has exited or there was an error */
+      /* TODO: Do something with the process return value */
+      /* TODO: What if the process output something since
+       * between log_from_handle and tor_get_exit_code? */
+      retval = 1;
+    }
+#else /* !(defined(_WIN32)) */
+    else if (1 == stdout_status || 1 == stderr_status)
+      /* stdout or stderr was closed, the process probably
+       * exited. It will be reaped by waitpid() in main.c */
+      /* TODO: Do something with the process return value */
+      retval = 1;
+#endif /* defined(_WIN32) */
+    else
+      /* Both are fine */
+      retval = 0;
+
+    /* If either pipe indicates a failure, act on it */
+    if (0 != retval) {
+      if (1 == retval) {
+        log_info(LD_GENERAL, "Port forwarding helper terminated");
+        child_handle->status = PROCESS_STATUS_NOTRUNNING;
+      } else {
+        log_warn(LD_GENERAL, "Failed to read from port forwarding helper");
+        child_handle->status = PROCESS_STATUS_ERROR;
+      }
+
+      /* TODO: The child might not actually be finished (maybe it failed or
+         closed stdout/stderr), so maybe we shouldn't start another? */
+    }
+  }
+}
+
 /** Initialize the insecure RNG <b>rng</b> from a seed value <b>seed</b>. */
 void
 tor_init_weak_random(tor_weak_rng_t *rng, unsigned seed)

src/common/util.h

@@ -73,9 +73,9 @@ extern int dmalloc_free(const char *file, const int line, void *pnt,
     }                                               \
   STMT_END
 #else /* !(defined(USE_DMALLOC)) */
-/** Release memory allocated by tor_malloc, tor_realloc, tor_strdup,
- * etc.  Unlike the free() function, the tor_free() macro sets the
- * pointer value to NULL after freeing it.
+/** Release memory allocated by tor_malloc, tor_realloc, tor_strdup, etc.
+ * Unlike the free() function, tor_free() will still work on NULL pointers,
+ * and it sets the pointer value to NULL after freeing it.
  *
  * This is a macro.  If you need a function pointer to release memory from
  * tor_malloc(), use tor_free_().
@@ -88,13 +88,17 @@ extern int dmalloc_free(const char *file, const int line, void *pnt,
 #ifdef __GNUC__
 #define tor_free(p) STMT_BEGIN                                 \
     typeof(&(p)) tor_free__tmpvar = &(p);                      \
-    raw_free(*tor_free__tmpvar);                               \
-    *tor_free__tmpvar=NULL;                                    \
+    if (PREDICT_LIKELY((*tor_free__tmpvar)!=NULL)) {           \
+      raw_free(*tor_free__tmpvar);                             \
+      *tor_free__tmpvar=NULL;                                  \
+    }                                                          \
   STMT_END
 #else
 #define tor_free(p) STMT_BEGIN                                 \
-  raw_free(p);                                                 \
-  (p)=NULL;                                                    \
+    if (PREDICT_LIKELY((p)!=NULL)) {                           \
+      raw_free(p);                                             \
+      (p)=NULL;                                                \
+    }                                                          \
   STMT_END
 #endif
 #endif /* defined(USE_DMALLOC) */
@@ -176,8 +180,6 @@ int n_bits_set_u8(uint8_t v);
 int64_t clamp_double_to_int64(double number);
 void simplify_fraction64(uint64_t *numer, uint64_t *denom);
 
-uint32_t tor_add_u32_nowrap(uint32_t a, uint32_t b);
-
 /* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b>
  * and positive <b>b</b>.  Works on integer types only. Not defined if a+(b-1)
  * can overflow. */
@@ -271,7 +273,6 @@ int parse_rfc1123_time(const char *buf, time_t *t);
 #define ISO_TIME_USEC_LEN (ISO_TIME_LEN+7)
 void format_local_iso_time(char *buf, time_t t);
 void format_iso_time(char *buf, time_t t);
-void format_local_iso_time_nospace(char *buf, time_t t);
 void format_iso_time_nospace(char *buf, time_t t);
 void format_iso_time_nospace_usec(char *buf, const struct timeval *tv);
 int parse_iso_time_(const char *cp, time_t *t, int strict, int nospace);
@@ -417,6 +418,11 @@ void start_daemon(void);
 void finish_daemon(const char *desired_cwd);
 int write_pidfile(const char *filename);
 
+/* Port forwarding */
+void tor_check_port_forwarding(const char *filename,
+                               struct smartlist_t *ports_to_forward,
+                               time_t now);
+
 void tor_disable_spawning_background_processes(void);
 
 typedef struct process_handle_t process_handle_t;
@@ -455,7 +461,9 @@ void set_environment_variable_in_smartlist(struct smartlist_t *env_vars,
                                            void (*free_old)(void*),
                                            int free_p);
 
-/* Values of process_handle_t.status. */
+/* Values of process_handle_t.status. PROCESS_STATUS_NOTRUNNING must be
+ * 0 because tor_check_port_forwarding depends on this being the initial
+ * statue of the static instance of process_handle_t */
 #define PROCESS_STATUS_NOTRUNNING 0
 #define PROCESS_STATUS_RUNNING 1
 #define PROCESS_STATUS_ERROR -1

src/common/workqueue.c

@@ -1,4 +1,3 @@
-
 /* copyright (c) 2013-2015, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
@@ -25,16 +24,13 @@
 
 #include "orconfig.h"
 #include "compat.h"
-#include "compat_libevent.h"
 #include "compat_threads.h"
-#include "crypto_rand.h"
+#include "crypto.h"
 #include "util.h"
 #include "workqueue.h"
 #include "tor_queue.h"
 #include "torlog.h"
 
-#include <event2/event.h>
-
 #define WORKQUEUE_PRIORITY_FIRST WQ_PRI_HIGH
 #define WORKQUEUE_PRIORITY_LAST WQ_PRI_LOW
 #define WORKQUEUE_N_PRIORITIES (((int) WORKQUEUE_PRIORITY_LAST)+1)
@@ -67,9 +63,6 @@ struct threadpool_s {
   void (*free_update_arg_fn)(void *);
   /** Array of n_threads update arguments. */
   void **update_args;
-  /** Event to notice when another thread has sent a reply. */
-  struct event *reply_event;
-  void (*reply_cb)(threadpool_t *);
 
   /** Number of elements in threads. */
   int n_threads;
@@ -604,41 +597,15 @@ replyqueue_new(uint32_t alertsocks_flags)
   return rq;
 }
 
-/** Internal: Run from the libevent mainloop when there is work to handle in
- * the reply queue handler. */
-static void
-reply_event_cb(evutil_socket_t sock, short events, void *arg)
-{
-  threadpool_t *tp = arg;
-  (void) sock;
-  (void) events;
-  replyqueue_process(tp->reply_queue);
-  if (tp->reply_cb)
-    tp->reply_cb(tp);
-}
-
-/** Register the threadpool <b>tp</b>'s reply queue with the libevent
- * mainloop of <b>base</b>. If <b>tp</b> is provided, it is run after
- * each time there is work to process from the reply queue. Return 0 on
- * success, -1 on failure.
+/**
+ * Return the "read socket" for a given reply queue.  The main thread should
+ * listen for read events on this socket, and call replyqueue_process() every
+ * time it triggers.
  */
-int
-threadpool_register_reply_event(threadpool_t *tp,
-                                void (*cb)(threadpool_t *tp))
+tor_socket_t
+replyqueue_get_socket(replyqueue_t *rq)
 {
-  struct event_base *base = tor_libevent_get_base();
-
-  if (tp->reply_event) {
-    tor_event_free(tp->reply_event);
-  }
-  tp->reply_event = tor_event_new(base,
-                                  tp->reply_queue->alert.read_fd,
-                                  EV_READ|EV_PERSIST,
-                                  reply_event_cb,
-                                  tp);
-  tor_assert(tp->reply_event);
-  tp->reply_cb = cb;
-  return event_add(tp->reply_event, NULL);
+  return rq->alert.read_fd;
 }
 
 /**

src/common/workqueue.h

@@ -56,11 +56,8 @@ threadpool_t *threadpool_new(int n_threads,
 replyqueue_t *threadpool_get_replyqueue(threadpool_t *tp);
 
 replyqueue_t *replyqueue_new(uint32_t alertsocks_flags);
+tor_socket_t replyqueue_get_socket(replyqueue_t *rq);
 void replyqueue_process(replyqueue_t *queue);
 
-struct event_base;
-int threadpool_register_reply_event(threadpool_t *tp,
-                                    void (*cb)(threadpool_t *tp));
-
 #endif /* !defined(TOR_WORKQUEUE_H) */
 

src/config/torrc.minimal.in-staging

@@ -100,7 +100,7 @@
 ## A handle for your relay, so people don't have to refer to it by key.
 ## Nicknames must be between 1 and 19 characters inclusive, and must
 ## contain only the alphanumeric characters (a-z, A-Z, 0-9). No unicode,
-## no emoji. If not set, "Unnamed" will be used.
+## no emoji.
 #Nickname ididnteditheconfig
 
 ## Define these to limit how much relayed traffic you will allow. Your

src/config/torrc.sample.in

@@ -106,7 +106,6 @@
 ## A handle for your relay, so people don't have to refer to it by key.
 ## Nicknames must be between 1 and 19 characters inclusive, and must
 ## contain only the characters [a-zA-Z0-9].
-## If not set, "Unnamed" will be used.
 #Nickname ididnteditheconfig
 
 ## Define these to limit how much relayed traffic you will allow. Your

src/ext/ed25519/donna/ed25519-hash-custom.h

@@ -9,7 +9,7 @@
 	void ed25519_hash(uint8_t *hash, const uint8_t *in, size_t inlen);
 */
 
-#include "crypto_digest.h"
+#include "crypto.h"
 
 typedef struct ed25519_hash_context {
   crypto_digest_t *ctx;

src/ext/ed25519/donna/ed25519-randombytes-custom.h

@@ -8,7 +8,7 @@
 */
 
 /* Tor: Instead of calling OpenSSL's CSPRNG directly, call the wrapper. */
-#include "crypto_rand.h"
+#include "crypto.h"
 
 static void
 ED25519_FN(ed25519_randombytes_unsafe) (void *p, size_t len)

src/ext/ed25519/donna/ed25519_tor.c

@@ -40,8 +40,6 @@
 #include "ed25519-randombytes.h"
 #include "ed25519-hash.h"
 
-#include "crypto_util.h"
-
 typedef unsigned char ed25519_signature[64];
 typedef unsigned char ed25519_public_key[32];
 typedef unsigned char ed25519_secret_key[32];

src/ext/ed25519/ref10/blinding.c

@@ -7,7 +7,7 @@
 #include "ed25519_ref10.h"
 
 #include <string.h>
-#include "crypto_util.h"
+#include "crypto.h"
 
 static void
 ed25519_ref10_gettweak(unsigned char *out, const unsigned char *param)

src/ext/ed25519/ref10/crypto_hash_sha512.h

@@ -1,5 +1,5 @@
 /* Added for Tor. */
-#include "crypto_digest.h"
+#include "crypto.h"
 
 /* Set 'out' to the 512-bit SHA512 hash of the 'len'-byte string in 'inp' */
 #define crypto_hash_sha512(out, inp, len) \

src/ext/ed25519/ref10/keypair.c

@@ -6,9 +6,6 @@
 #include "crypto_hash_sha512.h"
 #include "ge.h"
 
-#include "crypto_rand.h"
-#include "crypto_util.h"
-
 int
 crypto_sign_seckey(unsigned char *sk)
 {

src/ext/ed25519/ref10/randombytes.h

@@ -1,4 +1,4 @@
 /* Added for Tor. */
-#include "crypto_rand.h"
+#include "crypto.h"
 #define randombytes(b, n) \
   (crypto_strongest_rand((b), (n)), 0)

src/ext/keccak-tiny/keccak-tiny-unrolled.c

@@ -9,7 +9,7 @@
 #include "keccak-tiny.h"
 
 #include <string.h>
-#include "crypto_util.h"
+#include "crypto.h"
 #include "byteorder.h"
 
 /******** Endianness conversion helpers ********/

src/ext/rust

@@ -1 +1 @@
-Subproject commit aa37fb84fb829902e83ca11a7244bbc6b86b809b
+Subproject commit fbc0c25785696a25b9cbc09ed645cc8d404ee0f6

src/ext/timeouts/timeout.c

@@ -150,7 +150,7 @@
 #else
 #define ctz(n) ctz32(n)
 #define clz(n) clz32(n)
-#define fls(n) ((int)(32 - clz32((uint32_t)n)))
+#define fls(n) ((int)(32 - clz32(n)))
 #endif
 
 #if WHEEL_BIT == 6
@@ -432,7 +432,7 @@ TIMEOUT_PUBLIC void timeouts_update(struct timeouts *T, abstime_t curtime) {
 			 * or can be replaced with a simpler operation.
 			 */
 			oslot = WHEEL_MASK & (T->curtime >> (wheel * WHEEL_BIT));
-			pending = rotl(((WHEEL_C(1) << _elapsed) - 1), oslot);
+			pending = rotl(((UINT64_C(1) << _elapsed) - 1), oslot);
 
 			nslot = WHEEL_MASK & (curtime >> (wheel * WHEEL_BIT));
 			pending |= rotr(rotl(((WHEEL_C(1) << _elapsed) - 1), nslot), (int)_elapsed);

src/or/addressmap.c

@@ -21,10 +21,9 @@
 #include "config.h"
 #include "connection_edge.h"
 #include "control.h"
-#include "crypto_rand.h"
 #include "dns.h"
-#include "nodelist.h"
 #include "routerset.h"
+#include "nodelist.h"
 
 /** A client-side struct to remember requests to rewrite addresses
  * to new addresses. These structs are stored in the hash table
@@ -960,11 +959,9 @@ addressmap_get_virtual_address(int type)
         char tmp[TOR_ADDR_BUF_LEN];
         tor_addr_to_str(tmp, &addr, sizeof(tmp), 0);
         if (strmap_get(addressmap, tmp)) {
-          // LCOV_EXCL_START
           log_warn(LD_BUG, "%s wasn't in the addressmap, but %s was.",
                    buf, tmp);
           continue;
-          // LCOV_EXCL_STOP
         }
 
         return tor_strdup(buf);
@@ -973,10 +970,8 @@ addressmap_get_virtual_address(int type)
     log_warn(LD_CONFIG, "Ran out of virtual addresses!");
     return NULL;
   } else {
-    // LCOV_EXCL_START
     log_warn(LD_BUG, "Called with unsupported address type (%d)", type);
     return NULL;
-    // LCOV_EXCL_STOP
   }
 }
 

src/or/bridges.c

@@ -11,8 +11,6 @@
  * Bridges are fixed entry nodes, used for censorship circumvention.
  **/
 
-#define TOR_BRIDGES_PRIVATE
-
 #include "or.h"
 #include "bridges.h"
 #include "circuitbuild.h"
@@ -95,7 +93,7 @@ sweep_bridge_list(void)
 }
 
 /** Initialize the bridge list to empty, creating it if needed. */
-STATIC void
+static void
 clear_bridge_list(void)
 {
   if (!bridge_list)
@@ -158,7 +156,7 @@ bridge_get_addr_port(const bridge_info_t *bridge)
  * bridge with no known digest whose address matches any of the
  * tor_addr_port_t's in <b>orports</b>, return that bridge.  Else return
  * NULL. */
-STATIC bridge_info_t *
+static bridge_info_t *
 get_configured_bridge_by_orports_digest(const char *digest,
                                         const smartlist_t *orports)
 {
@@ -352,7 +350,7 @@ bridge_has_digest(const bridge_info_t *bridge, const char *digest)
  * existing bridge with the same address and port, and warn the user as
  * appropriate.
  */
-STATIC void
+static void
 bridge_resolve_conflicts(const tor_addr_t *addr, uint16_t port,
                          const char *digest, const char *transport_name)
 {
@@ -473,7 +471,7 @@ bridge_add_from_config(bridge_line_t *bridge_line)
 }
 
 /** If <b>digest</b> is one of our known bridges, return it. */
-STATIC bridge_info_t *
+bridge_info_t *
 find_bridge_by_digest(const char *digest)
 {
   if (! bridge_list)

src/or/bridges.h

@@ -20,6 +20,7 @@ typedef struct bridge_info_t bridge_info_t;
 void mark_bridge_list(void);
 void sweep_bridge_list(void);
 const smartlist_t *bridge_list_get(void);
+bridge_info_t *find_bridge_by_digest(const char *digest);
 const uint8_t *bridge_get_rsa_id_digest(const bridge_info_t *bridge);
 const tor_addr_port_t * bridge_get_addr_port(const bridge_info_t *bridge);
 bridge_info_t *get_configured_bridge_by_addr_port_digest(
@@ -64,17 +65,5 @@ MOCK_DECL(download_status_t *, get_bridge_dl_status_by_id,
 
 void bridges_free_all(void);
 
-#ifdef TOR_BRIDGES_PRIVATE
-STATIC void clear_bridge_list(void);
-STATIC bridge_info_t *find_bridge_by_digest(const char *digest);
-STATIC bridge_info_t *get_configured_bridge_by_orports_digest(
-                                                  const char *digest,
-                                                  const smartlist_t *orports);
-STATIC void bridge_resolve_conflicts(const tor_addr_t *addr,
-                                     uint16_t port,
-                                     const char *digest,
-                                     const char *transport_name);
-#endif /* defined(TOR_BRIDGES_PRIVATE) */
-
 #endif /* !defined(TOR_BRIDGES_H) */
 

src/or/channel.c

@@ -69,7 +69,6 @@
 #include "circuitmux.h"
 #include "entrynodes.h"
 #include "geoip.h"
-#include "main.h"
 #include "nodelist.h"
 #include "relay.h"
 #include "rephist.h"
@@ -405,7 +404,6 @@ channel_register(channel_t *chan)
     /* Put it in the finished list, creating it if necessary */
     if (!finished_channels) finished_channels = smartlist_new();
     smartlist_add(finished_channels, chan);
-    mainloop_schedule_postloop_cleanup();
   } else {
     /* Put it in the active list, creating it if necessary */
     if (!active_channels) active_channels = smartlist_new();
@@ -1550,7 +1548,6 @@ channel_change_state_(channel_t *chan, channel_state_t to_state)
       if (active_channels) smartlist_remove(active_channels, chan);
       if (!finished_channels) finished_channels = smartlist_new();
       smartlist_add(finished_channels, chan);
-      mainloop_schedule_postloop_cleanup();
     }
     /* Need to put on active list? */
     else if (!was_active && is_active) {
@@ -1669,7 +1666,6 @@ channel_listener_change_state(channel_listener_t *chan_l,
       if (active_listeners) smartlist_remove(active_listeners, chan_l);
       if (!finished_listeners) finished_listeners = smartlist_new();
       smartlist_add(finished_listeners, chan_l);
-      mainloop_schedule_postloop_cleanup();
     }
     /* Need to put on active list? */
     else if (!was_active && is_active) {
@@ -2112,6 +2108,21 @@ channel_listener_dumpstats(int severity)
   }
 }
 
+/**
+ * Set the cmux policy on all active channels.
+ */
+void
+channel_set_cmux_policy_everywhere(circuitmux_policy_t *pol)
+{
+  if (!active_channels) return;
+
+  SMARTLIST_FOREACH_BEGIN(active_channels, channel_t *, curr) {
+    if (curr->cmux) {
+      circuitmux_set_policy(curr->cmux, pol);
+    }
+  } SMARTLIST_FOREACH_END(curr);
+}
+
 /**
  * Clean up channels.
  *
@@ -2391,7 +2402,7 @@ channel_get_for_extend(const char *rsa_id_digest,
 {
   channel_t *chan, *best = NULL;
   int n_inprogress_goodaddr = 0, n_old = 0;
-  int n_noncanonical = 0;
+  int n_noncanonical = 0, n_possible = 0;
 
   tor_assert(msg_out);
   tor_assert(launch_out);
@@ -2454,6 +2465,8 @@ channel_get_for_extend(const char *rsa_id_digest,
       continue;
     }
 
+    ++n_possible;
+
     if (!best) {
       best = chan; /* If we have no 'best' so far, this one is good enough. */
       continue;

src/or/channel.h

@@ -422,6 +422,9 @@ void channel_free_all(void);
 void channel_dumpstats(int severity);
 void channel_listener_dumpstats(int severity);
 
+/* Set the cmux policy on all active channels */
+void channel_set_cmux_policy_everywhere(circuitmux_policy_t *pol);
+
 #ifdef TOR_CHANNEL_INTERNAL_
 
 #ifdef CHANNEL_PRIVATE_

src/or/channelpadding.c

@@ -16,11 +16,11 @@
 #include "networkstatus.h"
 #include "connection.h"
 #include "connection_or.h"
-#include "crypto_rand.h"
 #include "main.h"
 #include "rephist.h"
 #include "router.h"
 #include "compat_time.h"
+#include <event2/event.h>
 #include "rendservice.h"
 
 STATIC int32_t channelpadding_get_netflow_inactive_timeout_ms(

src/or/channeltls.c

@@ -160,8 +160,9 @@ channel_tls_common_init(channel_tls_t *tlschan)
   chan->write_var_cell = channel_tls_write_var_cell_method;
 
   chan->cmux = circuitmux_alloc();
-  /* We only have one policy for now so always set it to EWMA. */
-  circuitmux_set_policy(chan->cmux, &ewma_policy);
+  if (cell_ewma_enabled()) {
+    circuitmux_set_policy(chan->cmux, &ewma_policy);
+  }
 }
 
 /**

src/or/circpathbias.c

@@ -30,7 +30,6 @@
 #include "circuitstats.h"
 #include "connection_edge.h"
 #include "config.h"
-#include "crypto_rand.h"
 #include "entrynodes.h"
 #include "networkstatus.h"
 #include "relay.h"

src/or/circuitbuild.c

@@ -43,7 +43,7 @@
 #include "connection_edge.h"
 #include "connection_or.h"
 #include "control.h"
-#include "crypto_rand.h"
+#include "crypto.h"
 #include "directory.h"
 #include "entrynodes.h"
 #include "hs_ntor.h"
@@ -56,7 +56,6 @@
 #include "onion_fast.h"
 #include "policies.h"
 #include "relay.h"
-#include "relay_crypto.h"
 #include "rendcommon.h"
 #include "rephist.h"
 #include "router.h"
@@ -72,7 +71,10 @@ static channel_t * channel_connect_for_circuit(const tor_addr_t *addr,
 static int circuit_deliver_create_cell(circuit_t *circ,
                                        const create_cell_t *create_cell,
                                        int relayed);
+static int onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit,
+                                 int is_hs_v3_rp_circuit);
 static crypt_path_t *onion_next_hop_in_cpath(crypt_path_t *cpath);
+static int onion_extend_cpath(origin_circuit_t *circ);
 STATIC int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
 static int circuit_send_first_onion_skin(origin_circuit_t *circ);
 static int circuit_build_no_more_hops(origin_circuit_t *circ);
@@ -1053,7 +1055,7 @@ circuit_build_no_more_hops(origin_circuit_t *circ)
     clear_broken_connection_map(1);
     if (server_mode(options) && !check_whether_orport_reachable(options)) {
       inform_testing_reachability();
-      router_do_reachability_checks(1, 1);
+      consider_testing_reachability(1, 1);
     }
   }
 
@@ -1130,29 +1132,19 @@ circuit_send_intermediate_onion_skin(origin_circuit_t *circ,
   return 0;
 }
 
-/** Our clock just jumped by <b>seconds_elapsed</b>. If <b>was_idle</b> is
- * true, then the monotonic time matches; otherwise it doesn't. Assume
- * something has also gone wrong with our network: notify the user, and
- * abandon all not-yet-used circuits. */
+/** Our clock just jumped by <b>seconds_elapsed</b>. Assume
+ * something has also gone wrong with our network: notify the user,
+ * and abandon all not-yet-used circuits. */
 void
-circuit_note_clock_jumped(int64_t seconds_elapsed, bool was_idle)
+circuit_note_clock_jumped(int seconds_elapsed)
 {
   int severity = server_mode(get_options()) ? LOG_WARN : LOG_NOTICE;
-  if (was_idle) {
-    tor_log(severity, LD_GENERAL, "Tor has been idle for "I64_FORMAT
-            " seconds; assuming established circuits no longer work.",
-            I64_PRINTF_ARG(seconds_elapsed));
-  } else {
-    tor_log(severity, LD_GENERAL,
-            "Your system clock just jumped "I64_FORMAT" seconds %s; "
-            "assuming established circuits no longer work.",
-            I64_PRINTF_ARG(
-                 seconds_elapsed >=0 ? seconds_elapsed : -seconds_elapsed),
-            seconds_elapsed >=0 ? "forward" : "backward");
-  }
-  control_event_general_status(LOG_WARN, "CLOCK_JUMPED TIME="I64_FORMAT
-                               " IDLE=%d",
-                               I64_PRINTF_ARG(seconds_elapsed), was_idle?1:0);
+  tor_log(severity, LD_GENERAL, "Your system clock just jumped %d seconds %s; "
+      "assuming established circuits no longer work.",
+      seconds_elapsed >=0 ? seconds_elapsed : -seconds_elapsed,
+      seconds_elapsed >=0 ? "forward" : "backward");
+  control_event_general_status(LOG_WARN, "CLOCK_JUMPED TIME=%d",
+                               seconds_elapsed);
   /* so we log when it works again */
   note_that_we_maybe_cant_complete_circuits();
   control_event_client_status(severity, "CIRCUIT_NOT_ESTABLISHED REASON=%s",
@@ -1344,10 +1336,69 @@ circuit_init_cpath_crypto(crypt_path_t *cpath,
                           const char *key_data, size_t key_data_len,
                           int reverse, int is_hs_v3)
 {
+  crypto_digest_t *tmp_digest;
+  crypto_cipher_t *tmp_crypto;
+  size_t digest_len = 0;
+  size_t cipher_key_len = 0;
 
   tor_assert(cpath);
-  return relay_crypto_init(&cpath->crypto, key_data, key_data_len, reverse,
-                           is_hs_v3);
+  tor_assert(key_data);
+  tor_assert(!(cpath->f_crypto || cpath->b_crypto ||
+             cpath->f_digest || cpath->b_digest));
+
+  /* Basic key size validation */
+  if (is_hs_v3 && BUG(key_data_len != HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN)) {
+    return -1;
+  } else if (!is_hs_v3 && BUG(key_data_len != CPATH_KEY_MATERIAL_LEN)) {
+    return -1;
+  }
+
+  /* If we are using this cpath for next gen onion services use SHA3-256,
+     otherwise use good ol' SHA1 */
+  if (is_hs_v3) {
+    digest_len = DIGEST256_LEN;
+    cipher_key_len = CIPHER256_KEY_LEN;
+    cpath->f_digest = crypto_digest256_new(DIGEST_SHA3_256);
+    cpath->b_digest = crypto_digest256_new(DIGEST_SHA3_256);
+  } else {
+    digest_len = DIGEST_LEN;
+    cipher_key_len = CIPHER_KEY_LEN;
+    cpath->f_digest = crypto_digest_new();
+    cpath->b_digest = crypto_digest_new();
+  }
+
+  tor_assert(digest_len != 0);
+  tor_assert(cipher_key_len != 0);
+  const int cipher_key_bits = (int) cipher_key_len * 8;
+
+  crypto_digest_add_bytes(cpath->f_digest, key_data, digest_len);
+  crypto_digest_add_bytes(cpath->b_digest, key_data+digest_len, digest_len);
+
+  cpath->f_crypto = crypto_cipher_new_with_bits(key_data+(2*digest_len),
+                                                cipher_key_bits);
+  if (!cpath->f_crypto) {
+    log_warn(LD_BUG,"Forward cipher initialization failed.");
+    return -1;
+  }
+
+  cpath->b_crypto = crypto_cipher_new_with_bits(
+                                        key_data+(2*digest_len)+cipher_key_len,
+                                        cipher_key_bits);
+  if (!cpath->b_crypto) {
+    log_warn(LD_BUG,"Backward cipher initialization failed.");
+    return -1;
+  }
+
+  if (reverse) {
+    tmp_digest = cpath->f_digest;
+    cpath->f_digest = cpath->b_digest;
+    cpath->b_digest = tmp_digest;
+    tmp_crypto = cpath->f_crypto;
+    cpath->f_crypto = cpath->b_crypto;
+    cpath->b_crypto = tmp_crypto;
+  }
+
+  return 0;
 }
 
 /** A "created" cell <b>reply</b> came back to us on circuit <b>circ</b>.
@@ -1470,6 +1521,7 @@ onionskin_answer(or_circuit_t *circ,
                  const uint8_t *rend_circ_nonce)
 {
   cell_t cell;
+  crypt_path_t *tmp_cpath;
 
   tor_assert(keys_len == CPATH_KEY_MATERIAL_LEN);
 
@@ -1480,15 +1532,25 @@ onionskin_answer(or_circuit_t *circ,
   }
   cell.circ_id = circ->p_circ_id;
 
+  tmp_cpath = tor_malloc_zero(sizeof(crypt_path_t));
+  tmp_cpath->magic = CRYPT_PATH_MAGIC;
+
   circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN);
 
   log_debug(LD_CIRC,"init digest forward 0x%.8x, backward 0x%.8x.",
             (unsigned int)get_uint32(keys),
             (unsigned int)get_uint32(keys+20));
-  if (relay_crypto_init(&circ->crypto, keys, keys_len, 0, 0)<0) {
+  if (circuit_init_cpath_crypto(tmp_cpath, keys, keys_len, 0, 0)<0) {
     log_warn(LD_BUG,"Circuit initialization failed");
+    tor_free(tmp_cpath);
     return -1;
   }
+  circ->n_digest = tmp_cpath->f_digest;
+  circ->n_crypto = tmp_cpath->f_crypto;
+  circ->p_digest = tmp_cpath->b_digest;
+  circ->p_crypto = tmp_cpath->b_crypto;
+  tmp_cpath->magic = 0;
+  tor_free(tmp_cpath);
 
   memcpy(circ->rend_circ_nonce, rend_circ_nonce, DIGEST_LEN);
 
@@ -1549,7 +1611,7 @@ onionskin_answer(or_circuit_t *circ,
  *     rend_service_launch_establish_intro())
  *
  *   - We are a router testing its own reachabiity
- *     (CIRCUIT_PURPOSE_TESTING, via router_do_reachability_checks())
+ *     (CIRCUIT_PURPOSE_TESTING, via consider_testing_reachability())
  *
  * onion_pick_cpath_exit() bypasses us (by not calling
  * new_route_len()) in the one-hop tunnel case, so we don't need to
@@ -2290,7 +2352,7 @@ warn_if_last_router_excluded(origin_circuit_t *circ,
  * be used as an HS v3 rendezvous point.
  *
  * Return 0 if ok, -1 if circuit should be closed. */
-STATIC int
+static int
 onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei,
                       int is_hs_v3_rp_circuit)
 {
@@ -2460,72 +2522,13 @@ cpath_get_n_hops(crypt_path_t **head_ptr)
 
 #endif /* defined(TOR_UNIT_TESTS) */
 
-/**
- * Build the exclude list for vanguard circuits.
- *
- * For vanguard circuits we exclude all the already chosen nodes (including the
- * exit) from being middle hops to prevent the creation of A - B - A subpaths.
- * We also allow the 4th hop to be the same as the guard node so as to not leak
- * guard information to RP/IP/HSDirs.
- *
- * For vanguard circuits, we don't apply any subnet or family restrictions.
- * This is to avoid impossible-to-build circuit paths, or just situations where
- * our earlier guards prevent us from using most of our later ones.
- *
- * The alternative is building the circuit in reverse. Reverse calls to
- * onion_extend_cpath() (ie: select outer hops first) would then have the
- * property that you don't gain information about inner hops by observing
- * outer ones. See https://trac.torproject.org/projects/tor/ticket/24487
- * for this.
- *
- * (Note further that we still exclude the exit to prevent A - B - A
- * at the end of the path. */
-static smartlist_t *
-build_vanguard_middle_exclude_list(uint8_t purpose,
-                                   cpath_build_state_t *state,
-                                   crypt_path_t *head,
-                                   int cur_len)
-{
-  smartlist_t *excluded;
-  const node_t *r;
-  crypt_path_t *cpath;
-  int i;
-
-  (void) purpose;
-
-  excluded = smartlist_new();
-
-  /* Add the exit to the exclude list (note that the exit/last hop is always
-   * chosen first in circuit_establish_circuit()). */
-  if ((r = build_state_get_exit_node(state))) {
-    smartlist_add(excluded, (node_t*)r);
-  }
-
-  /* If we are picking the 4th hop, allow that node to be the guard too.
-   * This prevents us from avoiding the Guard for those hops, which
-   * gives the adversary information about our guard if they control
-   * the RP, IP, or HSDIR. We don't do this check based on purpose
-   * because we also want to allow HS_VANGUARDS pre-build circuits
-   * to use the guard for that last hop.
-   */
-  if (cur_len == DEFAULT_ROUTE_LEN+1) {
-    /* Skip the first hop for the exclude list below */
-    head = head->next;
-    cur_len--;
-  }
-
-  for (i = 0, cpath = head; cpath && i < cur_len; ++i, cpath=cpath->next) {
-    if ((r = node_get_by_id(cpath->extend_info->identity_digest))) {
-      smartlist_add(excluded, (node_t*)r);
-    }
-  }
-
-  return excluded;
-}
-
 /**
  * Build a list of nodes to exclude from the choice of this middle
  * hop, based on already chosen nodes.
+ *
+ * XXX: At present, this function does not exclude any nodes from
+ * the vanguard circuits. See
+ * https://trac.torproject.org/projects/tor/ticket/24487
  */
 static smartlist_t *
 build_middle_exclude_list(uint8_t purpose,
@@ -2538,21 +2541,32 @@ build_middle_exclude_list(uint8_t purpose,
   crypt_path_t *cpath;
   int i;
 
-  /** Vanguard circuits have their own path selection rules */
-  if (circuit_should_use_vanguards(purpose)) {
-    return build_vanguard_middle_exclude_list(purpose, state, head, cur_len);
-  }
-
   excluded = smartlist_new();
 
-  /* For non-vanguard circuits, add the exit and its family to the exclude list
-   * (note that the exit/last hop is always chosen first in
-   * circuit_establish_circuit()). */
+  /* Add the exit to the exclude list (note that the exit/last hop is always
+   * chosen first in circuit_establish_circuit()). */
   if ((r = build_state_get_exit_node(state))) {
     nodelist_add_node_and_family(excluded, r);
   }
 
-  /* also exclude all other already chosen nodes and their family */
+  /* XXX: We don't apply any other previously selected node restrictions for
+   * vanguards, and allow nodes to be reused for those hop positions in the
+   * same circuit. This is because after many rotations, you get to learn
+   * inner guard nodes through the nodes that are not selected for outer
+   * hops.
+   *
+   * The alternative is building the circuit in reverse. Reverse calls to
+   * onion_extend_cpath() (ie: select outer hops first) would then have the
+   * property that you don't gain information about inner hops by observing
+   * outer ones. See https://trac.torproject.org/projects/tor/ticket/24487
+   * for this.
+   *
+   * (Note further that we can and do still exclude the exit in the block
+   * above, because it is chosen first in circuit_establish_circuit()..) */
+  if (circuit_should_use_vanguards(purpose)) {
+    return excluded;
+  }
+
   for (i = 0, cpath = head; cpath && i < cur_len; ++i, cpath=cpath->next) {
     if ((r = node_get_by_id(cpath->extend_info->identity_digest))) {
       nodelist_add_node_and_family(excluded, r);
@@ -2652,9 +2666,7 @@ choose_good_middle_server(uint8_t purpose,
   /** If a hidden service circuit wants a specific middle node, pin it. */
   if (middle_node_must_be_vanguard(options, purpose, cur_len)) {
     log_debug(LD_GENERAL, "Picking a sticky node (cur_len = %d)", cur_len);
-    choice = pick_vanguard_middle_node(options, flags, cur_len, excluded);
-    smartlist_free(excluded);
-    return choice;
+    return pick_vanguard_middle_node(options, flags, cur_len, excluded);
   }
 
   choice = router_choose_random_node(excluded, options->ExcludeNodes, flags);
@@ -2694,7 +2706,7 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state,
     /* This request is for an entry server to use for a regular circuit,
      * and we use entry guard nodes.  Just return one of the guard nodes.  */
     tor_assert(guard_state_out);
-    return guards_choose_guard(state, purpose, guard_state_out);
+    return guards_choose_guard(state, guard_state_out);
   }
 
   excluded = smartlist_new();
@@ -2737,7 +2749,7 @@ onion_next_hop_in_cpath(crypt_path_t *cpath)
  * Return 1 if the path is complete, 0 if we successfully added a hop,
  * and -1 on error.
  */
-STATIC int
+static int
 onion_extend_cpath(origin_circuit_t *circ)
 {
   uint8_t purpose = circ->base_.purpose;
@@ -2859,13 +2871,14 @@ extend_info_from_node(const node_t *node, int for_direct_connect)
     return NULL;
   }
 
-  /* Choose a preferred address first, but fall back to an allowed address. */
+  /* Choose a preferred address first, but fall back to an allowed address.
+   * choose_address returns 1 on success, but get_prim_orport returns 0. */
   if (for_direct_connect)
-    fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0, &ap);
-  else {
-    node_get_prim_orport(node, &ap);
-  }
-  valid_addr = tor_addr_port_is_valid_ap(&ap, 0);
+    valid_addr = fascist_firewall_choose_address_node(node,
+                                                      FIREWALL_OR_CONNECTION,
+                                                      0, &ap);
+  else
+    valid_addr = !node_get_prim_orport(node, &ap);
 
   if (valid_addr)
     log_debug(LD_CIRC, "using %s for %s",

src/or/circuitbuild.h

@@ -29,7 +29,7 @@ void circuit_n_chan_done(channel_t *chan, int status,
 int inform_testing_reachability(void);
 int circuit_timeout_want_to_count_circ(const origin_circuit_t *circ);
 int circuit_send_next_onion_skin(origin_circuit_t *circ);
-void circuit_note_clock_jumped(int64_t seconds_elapsed, bool was_idle);
+void circuit_note_clock_jumped(int seconds_elapsed);
 int circuit_extend(cell_t *cell, circuit_t *circ);
 int circuit_init_cpath_crypto(crypt_path_t *cpath,
                               const char *key_data, size_t key_data_len,
@@ -83,13 +83,6 @@ STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan);
 STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei,
                          smartlist_t *nodes);
 MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes));
-
-STATIC int onion_extend_cpath(origin_circuit_t *circ);
-
-STATIC int
-onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei,
-                      int is_hs_v3_rp_circuit);
-
 #if defined(ENABLE_TOR2WEB_MODE) || defined(TOR_UNIT_TESTS)
 STATIC const node_t *pick_tor2web_rendezvous_node(router_crn_flags_t flags,
                                                   const or_options_t *options);

src/or/circuitlist.c

@@ -65,8 +65,6 @@
 #include "connection_edge.h"
 #include "connection_or.h"
 #include "control.h"
-#include "crypto_rand.h"
-#include "crypto_util.h"
 #include "entrynodes.h"
 #include "main.h"
 #include "hs_circuit.h"
@@ -78,7 +76,6 @@
 #include "onion_fast.h"
 #include "policies.h"
 #include "relay.h"
-#include "relay_crypto.h"
 #include "rendclient.h"
 #include "rendcommon.h"
 #include "rephist.h"
@@ -409,6 +406,9 @@ circuit_set_p_circid_chan(or_circuit_t *or_circ, circid_t id,
   circuit_set_circid_chan_helper(circ, CELL_DIRECTION_IN, id, chan);
 
   if (chan) {
+    tor_assert(bool_eq(or_circ->p_chan_cells.n,
+                       or_circ->next_active_on_p_chan));
+
     chan->timestamp_last_had_circuits = approx_time();
   }
 
@@ -431,6 +431,8 @@ circuit_set_n_circid_chan(circuit_t *circ, circid_t id,
   circuit_set_circid_chan_helper(circ, CELL_DIRECTION_OUT, id, chan);
 
   if (chan) {
+    tor_assert(bool_eq(circ->n_chan_cells.n, circ->next_active_on_n_chan));
+
     chan->timestamp_last_had_circuits = approx_time();
   }
 
@@ -1085,7 +1087,10 @@ circuit_free_(circuit_t *circ)
 
     should_free = (ocirc->workqueue_entry == NULL);
 
-    relay_crypto_clear(&ocirc->crypto);
+    crypto_cipher_free(ocirc->p_crypto);
+    crypto_digest_free(ocirc->p_digest);
+    crypto_cipher_free(ocirc->n_crypto);
+    crypto_digest_free(ocirc->n_digest);
 
     if (ocirc->rend_splice) {
       or_circuit_t *other = ocirc->rend_splice;
@@ -1225,7 +1230,10 @@ circuit_free_cpath_node(crypt_path_t *victim)
   if (!victim)
     return;
 
-  relay_crypto_clear(&victim->crypto);
+  crypto_cipher_free(victim->f_crypto);
+  crypto_cipher_free(victim->b_crypto);
+  crypto_digest_free(victim->f_digest);
+  crypto_digest_free(victim->b_digest);
   onion_handshake_state_release(&victim->handshake_state);
   crypto_dh_free(victim->rend_dh_handshake_state);
   extend_info_free(victim->extend_info);
@@ -2069,7 +2077,6 @@ circuit_mark_for_close_, (circuit_t *circ, int reason, int line,
     circuits_pending_close = smartlist_new();
 
   smartlist_add(circuits_pending_close, circ);
-  mainloop_schedule_postloop_cleanup();
 
   log_info(LD_GENERAL, "Circuit %u (id: %" PRIu32 ") marked for close at "
                        "%s:%d (orig reason: %d, new reason: %d)",
@@ -2589,7 +2596,8 @@ assert_cpath_layer_ok(const crypt_path_t *cp)
   switch (cp->state)
     {
     case CPATH_STATE_OPEN:
-      relay_crypto_assert_ok(&cp->crypto);
+      tor_assert(cp->f_crypto);
+      tor_assert(cp->b_crypto);
       /* fall through */
     case CPATH_STATE_CLOSED:
       /*XXXX Assert that there's no handshake_state either. */
@@ -2679,7 +2687,10 @@ assert_circuit_ok,(const circuit_t *c))
       c->state == CIRCUIT_STATE_GUARD_WAIT) {
     tor_assert(!c->n_chan_create_cell);
     if (or_circ) {
-      relay_crypto_assert_ok(&or_circ->crypto);
+      tor_assert(or_circ->n_crypto);
+      tor_assert(or_circ->p_crypto);
+      tor_assert(or_circ->n_digest);
+      tor_assert(or_circ->p_digest);
     }
   }
   if (c->state == CIRCUIT_STATE_CHAN_WAIT && !c->marked_for_close) {

src/or/circuitmux.c

@@ -114,6 +114,13 @@ struct circuitmux_s {
    */
   chanid_circid_muxinfo_map_t *chanid_circid_map;
 
+  /*
+   * Double-linked ring of circuits with queued cells waiting for room to
+   * free up on this connection's outbuf.  Every time we pull cells from
+   * a circuit, we advance this pointer to the next circuit in the ring.
+   */
+  struct circuit_t *active_circuits_head, *active_circuits_tail;
+
   /** List of queued destroy cells */
   destroy_cell_queue_t destroy_cell_queue;
   /** Boolean: True iff the last cell to circuitmux_get_first_active_circuit
@@ -169,6 +176,17 @@ struct chanid_circid_muxinfo_t {
   circuit_muxinfo_t muxinfo;
 };
 
+/*
+ * Internal-use #defines
+ */
+
+#ifdef CMUX_PARANOIA
+#define circuitmux_assert_okay_paranoid(cmux) \
+  circuitmux_assert_okay(cmux)
+#else
+#define circuitmux_assert_okay_paranoid(cmux)
+#endif /* defined(CMUX_PARANOIA) */
+
 /*
  * Static function declarations
  */
@@ -181,9 +199,21 @@ chanid_circid_entry_hash(chanid_circid_muxinfo_t *a);
 static chanid_circid_muxinfo_t *
 circuitmux_find_map_entry(circuitmux_t *cmux, circuit_t *circ);
 static void
-circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ);
+circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ,
+                               cell_direction_t direction);
 static void
-circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ);
+circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ,
+                                 cell_direction_t direction);
+static inline void
+circuitmux_move_active_circ_to_tail(circuitmux_t *cmux, circuit_t *circ,
+                                    cell_direction_t direction);
+static inline circuit_t **
+circuitmux_next_active_circ_p(circuitmux_t *cmux, circuit_t *circ);
+static inline circuit_t **
+circuitmux_prev_active_circ_p(circuitmux_t *cmux, circuit_t *circ);
+static void circuitmux_assert_okay_pass_one(circuitmux_t *cmux);
+static void circuitmux_assert_okay_pass_two(circuitmux_t *cmux);
+static void circuitmux_assert_okay_pass_three(circuitmux_t *cmux);
 
 /* Static global variables */
 
@@ -192,6 +222,119 @@ static int64_t global_destroy_ctr = 0;
 
 /* Function definitions */
 
+/**
+ * Linked list helpers
+ */
+
+/**
+ * Move an active circuit to the tail of the cmux's active circuits list;
+ * used by circuitmux_notify_xmit_cells().
+ */
+
+static inline void
+circuitmux_move_active_circ_to_tail(circuitmux_t *cmux, circuit_t *circ,
+                                    cell_direction_t direction)
+{
+  circuit_t **next_p = NULL, **prev_p = NULL;
+  circuit_t **next_prev = NULL, **prev_next = NULL;
+  circuit_t **tail_next = NULL;
+  or_circuit_t *or_circ = NULL;
+
+  tor_assert(cmux);
+  tor_assert(circ);
+
+  circuitmux_assert_okay_paranoid(cmux);
+
+  /* Figure out our next_p and prev_p for this cmux/direction */
+  if (direction) {
+    if (direction == CELL_DIRECTION_OUT) {
+      tor_assert(circ->n_mux == cmux);
+      next_p = &(circ->next_active_on_n_chan);
+      prev_p = &(circ->prev_active_on_n_chan);
+    } else {
+      or_circ = TO_OR_CIRCUIT(circ);
+      tor_assert(or_circ->p_mux == cmux);
+      next_p = &(or_circ->next_active_on_p_chan);
+      prev_p = &(or_circ->prev_active_on_p_chan);
+    }
+  } else {
+    if (circ->n_mux == cmux) {
+      next_p = &(circ->next_active_on_n_chan);
+      prev_p = &(circ->prev_active_on_n_chan);
+    } else {
+      or_circ = TO_OR_CIRCUIT(circ);
+      tor_assert(or_circ->p_mux == cmux);
+      next_p = &(or_circ->next_active_on_p_chan);
+      prev_p = &(or_circ->prev_active_on_p_chan);
+    }
+  }
+  tor_assert(next_p);
+  tor_assert(prev_p);
+
+  /* Check if this really is an active circuit */
+  if ((*next_p == NULL && *prev_p == NULL) &&
+      !(circ == cmux->active_circuits_head ||
+        circ == cmux->active_circuits_tail)) {
+    /* Not active, no-op */
+    return;
+  }
+
+  /* Check if this is already the tail */
+  if (circ == cmux->active_circuits_tail) return;
+
+  /* Okay, we have to move it; figure out next_prev and prev_next */
+  if (*next_p) next_prev = circuitmux_prev_active_circ_p(cmux, *next_p);
+  if (*prev_p) prev_next = circuitmux_next_active_circ_p(cmux, *prev_p);
+  /* Adjust the previous node's next pointer, if any */
+  if (prev_next) *prev_next = *next_p;
+  /* Otherwise, we were the head */
+  else cmux->active_circuits_head = *next_p;
+  /* Adjust the next node's previous pointer, if any */
+  if (next_prev) *next_prev = *prev_p;
+  /* We're out of the list; now re-attach at the tail */
+  /* Adjust our next and prev pointers */
+  *next_p = NULL;
+  *prev_p = cmux->active_circuits_tail;
+  /* Set the next pointer of the tail, or the head if none */
+  if (cmux->active_circuits_tail) {
+    tail_next = circuitmux_next_active_circ_p(cmux,
+                                              cmux->active_circuits_tail);
+    *tail_next = circ;
+  } else {
+    cmux->active_circuits_head = circ;
+  }
+  /* Set the tail to this circuit */
+  cmux->active_circuits_tail = circ;
+
+  circuitmux_assert_okay_paranoid(cmux);
+}
+
+static inline circuit_t **
+circuitmux_next_active_circ_p(circuitmux_t *cmux, circuit_t *circ)
+{
+  tor_assert(cmux);
+  tor_assert(circ);
+
+  if (circ->n_mux == cmux) return &(circ->next_active_on_n_chan);
+  else {
+    tor_assert(TO_OR_CIRCUIT(circ)->p_mux == cmux);
+    return &(TO_OR_CIRCUIT(circ)->next_active_on_p_chan);
+  }
+}
+
+static inline circuit_t **
+circuitmux_prev_active_circ_p(circuitmux_t *cmux, circuit_t *circ)
+{
+  tor_assert(cmux);
+  tor_assert(circ);
+
+  if (circ->n_mux == cmux) return &(circ->prev_active_on_n_chan);
+  else {
+    tor_assert(TO_OR_CIRCUIT(circ)->p_mux == cmux);
+    return &(TO_OR_CIRCUIT(circ)->prev_active_on_p_chan);
+  }
+}
+
 /**
  * Helper for chanid_circid_cell_count_map_t hash table: compare the channel
  * ID and circuit ID for a and b, and return less than, equal to, or greater
@@ -263,6 +406,11 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
   circuit_t *circ = NULL;
 
   tor_assert(cmux);
+  /*
+   * Don't circuitmux_assert_okay_paranoid() here; this gets called when
+   * channels are being freed and have already been unregistered, so
+   * the channel ID lookups it does will fail.
+   */
 
   i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map);
   while (i) {
@@ -287,7 +435,7 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
              */
 
             if (to_remove->muxinfo.cell_count > 0) {
-              circuitmux_make_circuit_inactive(cmux, circ);
+              circuitmux_make_circuit_inactive(cmux, circ, CELL_DIRECTION_OUT);
             }
 
             /* Clear n_mux */
@@ -302,7 +450,7 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
              */
 
             if (to_remove->muxinfo.cell_count > 0) {
-              circuitmux_make_circuit_inactive(cmux, circ);
+              circuitmux_make_circuit_inactive(cmux, circ, CELL_DIRECTION_IN);
             }
 
             /*
@@ -458,7 +606,9 @@ circuitmux_clear_policy(circuitmux_t *cmux)
   tor_assert(cmux);
 
   /* Internally, this is just setting policy to NULL */
-  circuitmux_set_policy(cmux, NULL);
+  if (cmux->policy) {
+    circuitmux_set_policy(cmux, NULL);
+  }
 }
 
 /**
@@ -794,6 +944,7 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
   tor_assert(circ);
   tor_assert(direction == CELL_DIRECTION_IN ||
              direction == CELL_DIRECTION_OUT);
+  circuitmux_assert_okay_paranoid(cmux);
 
   /*
    * Figure out which channel we're using, and get the circuit's current
@@ -851,10 +1002,10 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
      */
     if (hashent->muxinfo.cell_count > 0 && cell_count == 0) {
       --(cmux->n_active_circuits);
-      circuitmux_make_circuit_inactive(cmux, circ);
+      circuitmux_make_circuit_inactive(cmux, circ, direction);
     } else if (hashent->muxinfo.cell_count == 0 && cell_count > 0) {
       ++(cmux->n_active_circuits);
-      circuitmux_make_circuit_active(cmux, circ);
+      circuitmux_make_circuit_active(cmux, circ, direction);
     }
     cmux->n_cells -= hashent->muxinfo.cell_count;
     cmux->n_cells += cell_count;
@@ -882,7 +1033,7 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
     hashent->muxinfo.cell_count = cell_count;
     hashent->muxinfo.direction = direction;
     /* Allocate policy specific circuit data if we need it */
-    if (cmux->policy->alloc_circ_data) {
+    if (cmux->policy && cmux->policy->alloc_circ_data) {
       /* Assert that we have the means to free policy-specific data */
       tor_assert(cmux->policy->free_circ_data);
       /* Allocate it */
@@ -902,14 +1053,25 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
     if (direction == CELL_DIRECTION_OUT) circ->n_mux = cmux;
     else TO_OR_CIRCUIT(circ)->p_mux = cmux;
 
+    /* Make sure the next/prev pointers are NULL */
+    if (direction == CELL_DIRECTION_OUT) {
+      circ->next_active_on_n_chan = NULL;
+      circ->prev_active_on_n_chan = NULL;
+    } else {
+      TO_OR_CIRCUIT(circ)->next_active_on_p_chan = NULL;
+      TO_OR_CIRCUIT(circ)->prev_active_on_p_chan = NULL;
+    }
+
     /* Update counters */
     ++(cmux->n_circuits);
     if (cell_count > 0) {
       ++(cmux->n_active_circuits);
-      circuitmux_make_circuit_active(cmux, circ);
+      circuitmux_make_circuit_active(cmux, circ, direction);
     }
     cmux->n_cells += cell_count;
   }
+
+  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /**
@@ -933,6 +1095,7 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ))
   tor_assert(cmux);
   tor_assert(cmux->chanid_circid_map);
   tor_assert(circ);
+  circuitmux_assert_okay_paranoid(cmux);
 
   /* See if we have it for n_chan/n_circ_id */
   if (circ->n_chan) {
@@ -970,7 +1133,7 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ))
     if (hashent->muxinfo.cell_count > 0) {
       --(cmux->n_active_circuits);
       /* This does policy notifies, so comes before freeing policy data */
-      circuitmux_make_circuit_inactive(cmux, circ);
+      circuitmux_make_circuit_inactive(cmux, circ, last_searched_direction);
     }
     cmux->n_cells -= hashent->muxinfo.cell_count;
 
@@ -999,6 +1162,8 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ))
     /* Free the hash entry */
     tor_free(hashent);
   }
+
+  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /**
@@ -1007,22 +1172,94 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ))
  */
 
 static void
-circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ)
+circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ,
+                               cell_direction_t direction)
 {
+  circuit_t **next_active = NULL, **prev_active = NULL, **next_prev = NULL;
+  circuitmux_t *circuit_cmux = NULL;
+  chanid_circid_muxinfo_t *hashent = NULL;
+  channel_t *chan = NULL;
+  circid_t circ_id;
+  int already_active;
+
   tor_assert(cmux);
-  tor_assert(cmux->policy);
   tor_assert(circ);
+  tor_assert(direction == CELL_DIRECTION_OUT ||
+             direction == CELL_DIRECTION_IN);
+  /*
+   * Don't circuitmux_assert_okay_paranoid(cmux) here because the cell count
+   * already got changed and we have to update the list for it to be consistent
+   * again.
+   */
+
+  /* Get the right set of active list links for this direction */
+  if (direction == CELL_DIRECTION_OUT) {
+    next_active = &(circ->next_active_on_n_chan);
+    prev_active = &(circ->prev_active_on_n_chan);
+    circuit_cmux = circ->n_mux;
+    chan = circ->n_chan;
+    circ_id = circ->n_circ_id;
+  } else {
+    next_active = &(TO_OR_CIRCUIT(circ)->next_active_on_p_chan);
+    prev_active = &(TO_OR_CIRCUIT(circ)->prev_active_on_p_chan);
+    circuit_cmux = TO_OR_CIRCUIT(circ)->p_mux;
+    chan = TO_OR_CIRCUIT(circ)->p_chan;
+    circ_id = TO_OR_CIRCUIT(circ)->p_circ_id;
+  }
+
+  /* Assert that it is attached to this mux and a channel */
+  tor_assert(cmux == circuit_cmux);
+  tor_assert(chan != NULL);
+
+  /*
+   * Check if the circuit really was inactive; if it's active, at least one
+   * of the next_active and prev_active pointers will not be NULL, or this
+   * circuit will be either the head or tail of the list for this cmux.
+   */
+  already_active = (*prev_active != NULL || *next_active != NULL ||
+                    cmux->active_circuits_head == circ ||
+                    cmux->active_circuits_tail == circ);
+
+  /* If we're already active, log a warning and finish */
+  if (already_active) {
+    log_warn(LD_CIRC,
+             "Circuit %u on channel " U64_FORMAT " was already active",
+             (unsigned)circ_id, U64_PRINTF_ARG(chan->global_identifier));
+    return;
+  }
+
+  /*
+   * This is going at the head of the list; if the old head is not NULL,
+   * then its prev pointer should point to this.
+   */
+  *next_active = cmux->active_circuits_head; /* Next is old head */
+  *prev_active = NULL; /* Prev is NULL (this will be the head) */
+  if (cmux->active_circuits_head) {
+    /* The list had an old head; update its prev pointer */
+    next_prev =
+      circuitmux_prev_active_circ_p(cmux, cmux->active_circuits_head);
+    tor_assert(next_prev);
+    *next_prev = circ;
+  } else {
+    /* The list was empty; this becomes the tail as well */
+    cmux->active_circuits_tail = circ;
+  }
+  /* This becomes the new head of the list */
+  cmux->active_circuits_head = circ;
 
   /* Policy-specific notification */
-  if (cmux->policy->notify_circ_active) {
+  if (cmux->policy &&
+      cmux->policy->notify_circ_active) {
     /* Okay, we need to check the circuit for policy data now */
-    chanid_circid_muxinfo_t *hashent = circuitmux_find_map_entry(cmux, circ);
+    hashent = circuitmux_find_map_entry(cmux, circ);
     /* We should have found something */
     tor_assert(hashent);
     /* Notify */
     cmux->policy->notify_circ_active(cmux, cmux->policy_data,
                                      circ, hashent->muxinfo.policy_data);
   }
+
+  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /**
@@ -1031,22 +1268,112 @@ circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ)
  */
 
 static void
-circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ)
+circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ,
+                                 cell_direction_t direction)
 {
+  circuit_t **next_active = NULL, **prev_active = NULL;
+  circuit_t **next_prev = NULL, **prev_next = NULL;
+  circuitmux_t *circuit_cmux = NULL;
+  chanid_circid_muxinfo_t *hashent = NULL;
+  channel_t *chan = NULL;
+  circid_t circ_id;
+  int already_inactive;
+
   tor_assert(cmux);
-  tor_assert(cmux->policy);
   tor_assert(circ);
+  tor_assert(direction == CELL_DIRECTION_OUT ||
+             direction == CELL_DIRECTION_IN);
+  /*
+   * Don't circuitmux_assert_okay_paranoid(cmux) here because the cell count
+   * already got changed and we have to update the list for it to be consistent
+   * again.
+   */
+
+  /* Get the right set of active list links for this direction */
+  if (direction == CELL_DIRECTION_OUT) {
+    next_active = &(circ->next_active_on_n_chan);
+    prev_active = &(circ->prev_active_on_n_chan);
+    circuit_cmux = circ->n_mux;
+    chan = circ->n_chan;
+    circ_id = circ->n_circ_id;
+  } else {
+    next_active = &(TO_OR_CIRCUIT(circ)->next_active_on_p_chan);
+    prev_active = &(TO_OR_CIRCUIT(circ)->prev_active_on_p_chan);
+    circuit_cmux = TO_OR_CIRCUIT(circ)->p_mux;
+    chan = TO_OR_CIRCUIT(circ)->p_chan;
+    circ_id = TO_OR_CIRCUIT(circ)->p_circ_id;
+  }
+
+  /* Assert that it is attached to this mux and a channel */
+  tor_assert(cmux == circuit_cmux);
+  tor_assert(chan != NULL);
+
+  /*
+   * Check if the circuit really was active; if it's inactive, the
+   * next_active and prev_active pointers will be NULL and this circuit
+   * will not be the head or tail of the list for this cmux.
+   */
+  already_inactive = (*prev_active == NULL && *next_active == NULL &&
+                      cmux->active_circuits_head != circ &&
+                      cmux->active_circuits_tail != circ);
+
+  /* If we're already inactive, log a warning and finish */
+  if (already_inactive) {
+    log_warn(LD_CIRC,
+             "Circuit %d on channel " U64_FORMAT " was already inactive",
+             (unsigned)circ_id, U64_PRINTF_ARG(chan->global_identifier));
+    return;
+  }
+
+  /* Remove from the list; first get next_prev and prev_next */
+  if (*next_active) {
+    /*
+     * If there's a next circuit, its previous circuit becomes this
+     * circuit's previous circuit.
+     */
+    next_prev = circuitmux_prev_active_circ_p(cmux, *next_active);
+  } else {
+    /* Else, the tail becomes this circuit's previous circuit */
+    next_prev = &(cmux->active_circuits_tail);
+  }
+
+  /* Got next_prev, now prev_next */
+  if (*prev_active) {
+    /*
+     * If there's a previous circuit, its next circuit becomes this circuit's
+     * next circuit.
+     */
+    prev_next = circuitmux_next_active_circ_p(cmux, *prev_active);
+  } else {
+    /* Else, the head becomes this circuit's next circuit */
+    prev_next = &(cmux->active_circuits_head);
+  }
+
+  /* Assert that we got sensible values for the next/prev pointers */
+  tor_assert(next_prev != NULL);
+  tor_assert(prev_next != NULL);
+
+  /* Update the next/prev pointers - this removes circ from the list */
+  *next_prev = *prev_active;
+  *prev_next = *next_active;
+
+  /* Now null out prev_active/next_active */
+  *prev_active = NULL;
+  *next_active = NULL;
 
   /* Policy-specific notification */
-  if (cmux->policy->notify_circ_inactive) {
+  if (cmux->policy &&
+      cmux->policy->notify_circ_inactive) {
     /* Okay, we need to check the circuit for policy data now */
-    chanid_circid_muxinfo_t *hashent = circuitmux_find_map_entry(cmux, circ);
+    hashent = circuitmux_find_map_entry(cmux, circ);
     /* We should have found something */
     tor_assert(hashent);
     /* Notify */
     cmux->policy->notify_circ_inactive(cmux, cmux->policy_data,
                                        circ, hashent->muxinfo.policy_data);
   }
+
+  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /**
@@ -1073,6 +1400,8 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ,
   tor_assert(cmux);
   tor_assert(circ);
 
+  circuitmux_assert_okay_paranoid(cmux);
+
   /* Search for this circuit's entry */
   hashent = circuitmux_find_map_entry(cmux, circ);
   /* Assert that we found one */
@@ -1083,7 +1412,7 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ,
   cmux->n_cells += n_cells;
 
   /* Do we need to notify a cmux policy? */
-  if (cmux->policy->notify_set_n_cells) {
+  if (cmux->policy && cmux->policy->notify_set_n_cells) {
     /* Call notify_set_n_cells */
     cmux->policy->notify_set_n_cells(cmux,
                                      cmux->policy_data,
@@ -1099,15 +1428,21 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ,
   if (hashent->muxinfo.cell_count > 0 && n_cells == 0) {
     --(cmux->n_active_circuits);
     hashent->muxinfo.cell_count = n_cells;
-    circuitmux_make_circuit_inactive(cmux, circ);
+    circuitmux_make_circuit_inactive(cmux, circ, hashent->muxinfo.direction);
   /* Is the old cell count == 0 and the new cell count > 0 ? */
   } else if (hashent->muxinfo.cell_count == 0 && n_cells > 0) {
     ++(cmux->n_active_circuits);
     hashent->muxinfo.cell_count = n_cells;
-    circuitmux_make_circuit_active(cmux, circ);
+    circuitmux_make_circuit_active(cmux, circ, hashent->muxinfo.direction);
   } else {
+    /*
+     * Update the entry cell count like this so we can put a
+     * circuitmux_assert_okay_paranoid inside make_circuit_(in)active() too.
+     */
     hashent->muxinfo.cell_count = n_cells;
   }
+
+  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /*
@@ -1133,9 +1468,6 @@ circuitmux_get_first_active_circuit(circuitmux_t *cmux,
   circuit_t *circ = NULL;
 
   tor_assert(cmux);
-  tor_assert(cmux->policy);
-  /* This callback is mandatory. */
-  tor_assert(cmux->policy->pick_active_circuit);
   tor_assert(destroy_queue_out);
 
   *destroy_queue_out = NULL;
@@ -1154,7 +1486,14 @@ circuitmux_get_first_active_circuit(circuitmux_t *cmux,
     /* We also must have a cell available for this to be the case */
     tor_assert(cmux->n_cells > 0);
     /* Do we have a policy-provided circuit selector? */
-    circ = cmux->policy->pick_active_circuit(cmux, cmux->policy_data);
+    if (cmux->policy && cmux->policy->pick_active_circuit) {
+      circ = cmux->policy->pick_active_circuit(cmux, cmux->policy_data);
+    }
+    /* Fall back on the head of the active circuits list */
+    if (!circ) {
+      tor_assert(cmux->active_circuits_head);
+      circ = cmux->active_circuits_head;
+    }
     cmux->last_cell_was_destroy = 0;
   } else {
     tor_assert(cmux->n_cells == 0);
@@ -1178,6 +1517,7 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ,
 
   tor_assert(cmux);
   tor_assert(circ);
+  circuitmux_assert_okay_paranoid(cmux);
 
   if (n_cells == 0) return;
 
@@ -1204,11 +1544,17 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ,
   /* Adjust the mux cell counter */
   cmux->n_cells -= n_cells;
 
+  /* If we aren't making it inactive later, move it to the tail of the list */
+  if (!becomes_inactive) {
+    circuitmux_move_active_circ_to_tail(cmux, circ,
+                                        hashent->muxinfo.direction);
+  }
+
   /*
    * We call notify_xmit_cells() before making the circuit inactive if needed,
    * so the policy can always count on this coming in on an active circuit.
    */
-  if (cmux->policy->notify_xmit_cells) {
+  if (cmux->policy && cmux->policy->notify_xmit_cells) {
     cmux->policy->notify_xmit_cells(cmux, cmux->policy_data, circ,
                                     hashent->muxinfo.policy_data,
                                     n_cells);
@@ -1220,8 +1566,10 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ,
    */
   if (becomes_inactive) {
     --(cmux->n_active_circuits);
-    circuitmux_make_circuit_inactive(cmux, circ);
+    circuitmux_make_circuit_inactive(cmux, circ, hashent->muxinfo.direction);
   }
+
+  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /**
@@ -1244,6 +1592,282 @@ circuitmux_notify_xmit_destroy(circuitmux_t *cmux)
             I64_PRINTF_ARG(global_destroy_ctr));
 }
 
+/*
+ * Circuitmux consistency checking assertions
+ */
+
+/**
+ * Check that circuitmux data structures are consistent and fail with an
+ * assert if not.
+ */
+
+void
+circuitmux_assert_okay(circuitmux_t *cmux)
+{
+  tor_assert(cmux);
+
+  /*
+   * Pass 1: iterate the hash table; for each entry:
+   *  a) Check that the circuit has this cmux for n_mux or p_mux
+   *  b) If the cell_count is > 0, set the mark bit; otherwise clear it
+   *  c) Also check activeness (cell_count > 0 should be active)
+   *  d) Count the number of circuits, active circuits and queued cells
+   *     and at the end check that they match the counters in the cmux.
+   *
+   * Pass 2: iterate the active circuits list; for each entry,
+   *         make sure the circuit is attached to this mux and appears
+   *         in the hash table.  Make sure the mark bit is 1, and clear
+   *         it in the hash table entry.  Consistency-check the linked
+   *         list pointers.
+   *
+   * Pass 3: iterate the hash table again; assert if any active circuits
+   *         (mark bit set to 1) are discovered that weren't cleared in pass 2
+   *         (don't appear in the linked list).
+   */
+
+  circuitmux_assert_okay_pass_one(cmux);
+  circuitmux_assert_okay_pass_two(cmux);
+  circuitmux_assert_okay_pass_three(cmux);
+}
+
+/**
+ * Do the first pass of circuitmux_assert_okay(); see the comment in that
+ * function.
+ */
+
+static void
+circuitmux_assert_okay_pass_one(circuitmux_t *cmux)
+{
+  chanid_circid_muxinfo_t **i = NULL;
+  uint64_t chan_id;
+  channel_t *chan;
+  circid_t circ_id;
+  circuit_t *circ;
+  or_circuit_t *or_circ;
+  circuit_t **next_p, **prev_p;
+  unsigned int n_circuits, n_active_circuits, n_cells;
+
+  tor_assert(cmux);
+  tor_assert(cmux->chanid_circid_map);
+
+  /* Reset the counters */
+  n_circuits = n_active_circuits = n_cells = 0;
+  /* Start iterating the hash table */
+  i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map);
+  while (i) {
+    /* Assert that the hash table entry isn't null */
+    tor_assert(*i);
+
+    /* Get the channel and circuit id */
+    chan_id = (*i)->chan_id;
+    circ_id = (*i)->circ_id;
+
+    /* Find the channel and circuit, assert that they exist */
+    chan = channel_find_by_global_id(chan_id);
+    tor_assert(chan);
+    circ = circuit_get_by_circid_channel_even_if_marked(circ_id, chan);
+    tor_assert(circ);
+
+    /* Assert that we know which direction this is going */
+    tor_assert((*i)->muxinfo.direction == CELL_DIRECTION_OUT ||
+               (*i)->muxinfo.direction == CELL_DIRECTION_IN);
+
+    if ((*i)->muxinfo.direction == CELL_DIRECTION_OUT) {
+      /* We should be n_mux on this circuit */
+      tor_assert(cmux == circ->n_mux);
+      tor_assert(chan == circ->n_chan);
+      /* Get next and prev for next test */
+      next_p = &(circ->next_active_on_n_chan);
+      prev_p = &(circ->prev_active_on_n_chan);
+    } else {
+      /* This should be an or_circuit_t and we should be p_mux */
+      or_circ = TO_OR_CIRCUIT(circ);
+      tor_assert(cmux == or_circ->p_mux);
+      tor_assert(chan == or_circ->p_chan);
+      /* Get next and prev for next test */
+      next_p = &(or_circ->next_active_on_p_chan);
+      prev_p = &(or_circ->prev_active_on_p_chan);
+    }
+
+    /*
+     * Should this circuit be active?  I.e., does the mux know about > 0
+     * cells on it?
+     */
+    const int circ_is_active = ((*i)->muxinfo.cell_count > 0);
+
+    /* It should be in the linked list iff it's active */
+    if (circ_is_active) {
+      /* Either we have a next link or we are the tail */
+      tor_assert(*next_p || (circ == cmux->active_circuits_tail));
+      /* Either we have a prev link or we are the head */
+      tor_assert(*prev_p || (circ == cmux->active_circuits_head));
+      /* Increment the active circuits counter */
+      ++n_active_circuits;
+    } else {
+      /* Shouldn't be in list, so no next or prev link */
+      tor_assert(!(*next_p));
+      tor_assert(!(*prev_p));
+      /* And can't be head or tail */
+      tor_assert(circ != cmux->active_circuits_head);
+      tor_assert(circ != cmux->active_circuits_tail);
+    }
+
+    /* Increment the circuits counter */
+    ++n_circuits;
+    /* Adjust the cell counter */
+    n_cells += (*i)->muxinfo.cell_count;
+
+    /* Set the mark bit to circ_is_active */
+    (*i)->muxinfo.mark = circ_is_active;
+
+    /* Advance to the next entry */
+    i = HT_NEXT(chanid_circid_muxinfo_map, cmux->chanid_circid_map, i);
+  }
+
+  /* Now check the counters */
+  tor_assert(n_cells == cmux->n_cells);
+  tor_assert(n_circuits == cmux->n_circuits);
+  tor_assert(n_active_circuits == cmux->n_active_circuits);
+}
+
+/**
+ * Do the second pass of circuitmux_assert_okay(); see the comment in that
+ * function.
+ */
+
+static void
+circuitmux_assert_okay_pass_two(circuitmux_t *cmux)
+{
+  circuit_t *curr_circ, *prev_circ = NULL, *next_circ;
+  or_circuit_t *curr_or_circ;
+  uint64_t curr_chan_id;
+  circid_t curr_circ_id;
+  circuit_t **next_p, **prev_p;
+  channel_t *chan;
+  unsigned int n_active_circuits = 0;
+  chanid_circid_muxinfo_t search, *hashent = NULL;
+
+  tor_assert(cmux);
+  tor_assert(cmux->chanid_circid_map);
+
+  /*
+   * Walk the linked list of active circuits in cmux; keep track of the
+   * previous circuit seen for consistency checking purposes.  Count them
+   * to make sure the number in the linked list matches
+   * cmux->n_active_circuits.
+   */
+  curr_circ = cmux->active_circuits_head;
+  while (curr_circ) {
+    /* Reset some things */
+    chan = NULL;
+    curr_or_circ = NULL;
+    next_circ = NULL;
+    next_p = prev_p = NULL;
+    cell_direction_t direction;
+
+    /* Figure out if this is n_mux or p_mux */
+    if (cmux == curr_circ->n_mux) {
+      /* Get next_p and prev_p */
+      next_p = &(curr_circ->next_active_on_n_chan);
+      prev_p = &(curr_circ->prev_active_on_n_chan);
+      /* Get the channel */
+      chan = curr_circ->n_chan;
+      /* Get the circuit id */
+      curr_circ_id = curr_circ->n_circ_id;
+      /* Remember the direction */
+      direction = CELL_DIRECTION_OUT;
+    } else {
+      /* We must be p_mux and this must be an or_circuit_t */
+      curr_or_circ = TO_OR_CIRCUIT(curr_circ);
+      tor_assert(cmux == curr_or_circ->p_mux);
+      /* Get next_p and prev_p */
+      next_p = &(curr_or_circ->next_active_on_p_chan);
+      prev_p = &(curr_or_circ->prev_active_on_p_chan);
+      /* Get the channel */
+      chan = curr_or_circ->p_chan;
+      /* Get the circuit id */
+      curr_circ_id = curr_or_circ->p_circ_id;
+      /* Remember the direction */
+      direction = CELL_DIRECTION_IN;
+    }
+
+    /* Assert that we got a channel and get the channel ID */
+    tor_assert(chan);
+    curr_chan_id = chan->global_identifier;
+
+    /* Assert that prev_p points to last circuit we saw */
+    tor_assert(*prev_p == prev_circ);
+    /* If that's NULL, assert that we are the head */
+    if (!(*prev_p)) tor_assert(curr_circ == cmux->active_circuits_head);
+
+    /* Get the next circuit */
+    next_circ = *next_p;
+    /* If it's NULL, assert that we are the tail */
+    if (!(*next_p)) tor_assert(curr_circ == cmux->active_circuits_tail);
+
+    /* Now find the hash table entry for this circuit */
+    search.chan_id = curr_chan_id;
+    search.circ_id = curr_circ_id;
+    hashent = HT_FIND(chanid_circid_muxinfo_map, cmux->chanid_circid_map,
+                      &search);
+
+    /* Assert that we have one */
+    tor_assert(hashent);
+
+    /* Assert that the direction matches */
+    tor_assert(direction == hashent->muxinfo.direction);
+
+    /* Assert that the hash entry got marked in pass one */
+    tor_assert(hashent->muxinfo.mark);
+
+    /* Clear the mark */
+    hashent->muxinfo.mark = 0;
+
+    /* Increment the counter */
+    ++n_active_circuits;
+
+    /* Advance to the next active circuit and update prev_circ */
+    prev_circ = curr_circ;
+    curr_circ = next_circ;
+  }
+
+  /* Assert that the counter matches the cmux */
+  tor_assert(n_active_circuits == cmux->n_active_circuits);
+}
+
+/**
+ * Do the third pass of circuitmux_assert_okay(); see the comment in that
+ * function.
+ */
+
+static void
+circuitmux_assert_okay_pass_three(circuitmux_t *cmux)
+{
+  chanid_circid_muxinfo_t **i = NULL;
+
+  tor_assert(cmux);
+  tor_assert(cmux->chanid_circid_map);
+
+  /* Start iterating the hash table */
+  i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map);
+
+  /* Advance through each entry */
+  while (i) {
+    /* Assert that it isn't null */
+    tor_assert(*i);
+
+    /*
+     * Assert that this entry is not marked - i.e., that either we didn't
+     * think it should be active in pass one or we saw it in the active
+     * circuits linked list.
+     */
+    tor_assert(!((*i)->muxinfo.mark));
+
+    /* Advance to the next entry */
+    i = HT_NEXT(chanid_circid_muxinfo_map, cmux->chanid_circid_map, i);
+  }
+}
+
 /*DOCDOC */
 void
 circuitmux_append_destroy_cell(channel_t *chan,

src/or/circuitmux_ewma.c

@@ -28,7 +28,7 @@
  *
  **/
 
-#define CIRCUITMUX_EWMA_PRIVATE
+#define TOR_CIRCUITMUX_EWMA_C_
 
 #include "orconfig.h"
 
@@ -37,7 +37,6 @@
 #include "or.h"
 #include "circuitmux.h"
 #include "circuitmux_ewma.h"
-#include "crypto_rand.h"
 #include "networkstatus.h"
 
 /*** EWMA parameter #defines ***/
@@ -170,6 +169,8 @@ TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *pol)
 
 static void add_cell_ewma(ewma_policy_data_t *pol, cell_ewma_t *ewma);
 static int compare_cell_ewma_counts(const void *p1, const void *p2);
+static unsigned cell_ewma_tick_from_timeval(const struct timeval *now,
+                                            double *remainder_out);
 static circuit_t * cell_ewma_to_circuit(cell_ewma_t *ewma);
 static inline double get_scale_factor(unsigned from_tick, unsigned to_tick);
 static cell_ewma_t * pop_first_cell_ewma(ewma_policy_data_t *pol);
@@ -222,6 +223,8 @@ ewma_cmp_cmux(circuitmux_t *cmux_1, circuitmux_policy_data_t *pol_data_1,
  * has value ewma_scale_factor ** N.)
  */
 static double ewma_scale_factor = 0.1;
+/* DOCDOC ewma_enabled */
+static int ewma_enabled = 0;
 
 /*** EWMA circuitmux_policy_t method table ***/
 
@@ -238,26 +241,8 @@ circuitmux_policy_t ewma_policy = {
   /*.cmp_cmux =*/ ewma_cmp_cmux
 };
 
-/** Have we initialized the ewma tick-counting logic? */
-static int ewma_ticks_initialized = 0;
-/** At what monotime_coarse_t did the current tick begin? */
-static monotime_coarse_t start_of_current_tick;
-/** What is the number of the current tick? */
-static unsigned current_tick_num;
-
 /*** EWMA method implementations using the below EWMA helper functions ***/
 
-/** Compute and return the current cell_ewma tick. */
-static inline unsigned int
-cell_ewma_get_tick(void)
-{
-  monotime_coarse_t now;
-  monotime_coarse_get(&now);
-  int32_t msec_diff = monotime_coarse_diff_msec32(&start_of_current_tick,
-                                                  &now);
-  return current_tick_num + msec_diff / (1000*EWMA_TICK_LEN);
-}
-
 /**
  * Allocate an ewma_policy_data_t and upcast it to a circuitmux_policy_data_t;
  * this is called when setting the policy on a circuitmux_t to ewma_policy.
@@ -431,6 +416,8 @@ ewma_notify_xmit_cells(circuitmux_t *cmux,
   ewma_policy_circ_data_t *cdata = NULL;
   unsigned int tick;
   double fractional_tick, ewma_increment;
+  /* The current (hi-res) time */
+  struct timeval now_hires;
   cell_ewma_t *cell_ewma, *tmp;
 
   tor_assert(cmux);
@@ -443,7 +430,8 @@ ewma_notify_xmit_cells(circuitmux_t *cmux,
   cdata = TO_EWMA_POL_CIRC_DATA(pol_circ_data);
 
   /* Rescale the EWMAs if needed */
-  tick = cell_ewma_get_current_tick_and_fraction(&fractional_tick);
+  tor_gettimeofday_cached(&now_hires);
+  tick = cell_ewma_tick_from_timeval(&now_hires, &fractional_tick);
 
   if (tick != pol->active_circuit_pqueue_last_recalibrated) {
     scale_active_circuits(pol, tick);
@@ -604,122 +592,79 @@ cell_ewma_to_circuit(cell_ewma_t *ewma)
    rescale.
  */
 
-/**
- * Initialize the system that tells which ewma tick we are in.
- */
-STATIC void
-cell_ewma_initialize_ticks(void)
-{
-  if (ewma_ticks_initialized)
-    return;
-  monotime_coarse_get(&start_of_current_tick);
-  crypto_rand((char*)&current_tick_num, sizeof(current_tick_num));
-  ewma_ticks_initialized = 1;
-}
-
-/** Compute the current cell_ewma tick and the fraction of the tick that has
- * elapsed between the start of the tick and the current time.  Return the
- * former and store the latter in *<b>remainder_out</b>.
+/** Given a timeval <b>now</b>, compute the cell_ewma tick in which it occurs
+ * and the fraction of the tick that has elapsed between the start of the tick
+ * and <b>now</b>.  Return the former and store the latter in
+ * *<b>remainder_out</b>.
  *
  * These tick values are not meant to be shared between Tor instances, or used
  * for other purposes. */
-STATIC unsigned
-cell_ewma_get_current_tick_and_fraction(double *remainder_out)
+
+static unsigned
+cell_ewma_tick_from_timeval(const struct timeval *now,
+                            double *remainder_out)
 {
-  if (BUG(!ewma_ticks_initialized)) {
-    cell_ewma_initialize_ticks(); // LCOV_EXCL_LINE
-  }
-  monotime_coarse_t now;
-  monotime_coarse_get(&now);
-  int32_t msec_diff = monotime_coarse_diff_msec32(&start_of_current_tick,
-                                                  &now);
-  if (msec_diff > (1000*EWMA_TICK_LEN)) {
-    unsigned ticks_difference = msec_diff / (1000*EWMA_TICK_LEN);
-    monotime_coarse_add_msec(&start_of_current_tick,
-                             &start_of_current_tick,
-                             ticks_difference * 1000 * EWMA_TICK_LEN);
-    current_tick_num += ticks_difference;
-    msec_diff %= 1000*EWMA_TICK_LEN;
-  }
-  *remainder_out = ((double)msec_diff) / (1.0e3 * EWMA_TICK_LEN);
-  return current_tick_num;
+  unsigned res = (unsigned) (now->tv_sec / EWMA_TICK_LEN);
+  /* rem */
+  double rem = (now->tv_sec % EWMA_TICK_LEN) +
+    ((double)(now->tv_usec)) / 1.0e6;
+  *remainder_out = rem / EWMA_TICK_LEN;
+  return res;
 }
 
-/* Default value for the CircuitPriorityHalflifeMsec consensus parameter in
- * msec. */
-#define CMUX_PRIORITY_HALFLIFE_MSEC_DEFAULT 30000
-/* Minimum and maximum value for the CircuitPriorityHalflifeMsec consensus
- * parameter. */
-#define CMUX_PRIORITY_HALFLIFE_MSEC_MIN 1
-#define CMUX_PRIORITY_HALFLIFE_MSEC_MAX INT32_MAX
-
-/* Return the value of the circuit priority halflife from the options if
- * available or else from the consensus (in that order). If none can be found,
- * a default value is returned.
- *
- * The source_msg points to a string describing from where the value was
- * picked so it can be used for logging. */
-static double
-get_circuit_priority_halflife(const or_options_t *options,
-                              const networkstatus_t *consensus,
-                              const char **source_msg)
+/** Tell the caller whether ewma_enabled is set */
+int
+cell_ewma_enabled(void)
 {
-  int32_t halflife_ms;
-  double halflife;
-  /* Compute the default value now. We might need it. */
-  double halflife_default =
-    ((double) CMUX_PRIORITY_HALFLIFE_MSEC_DEFAULT) / 1000.0;
+  return ewma_enabled;
+}
 
-  /* Try to get it from configuration file first. */
-  if (options && options->CircuitPriorityHalflife >= -EPSILON) {
-    halflife = options->CircuitPriorityHalflife;
-    *source_msg = "CircuitPriorityHalflife in configuration";
-    goto end;
-  }
-
-  /* Try to get the msec value from the consensus. */
-  halflife_ms = networkstatus_get_param(consensus,
-                                        "CircuitPriorityHalflifeMsec",
-                                        CMUX_PRIORITY_HALFLIFE_MSEC_DEFAULT,
-                                        CMUX_PRIORITY_HALFLIFE_MSEC_MIN,
-                                        CMUX_PRIORITY_HALFLIFE_MSEC_MAX);
-  halflife = ((double) halflife_ms) / 1000.0;
-  *source_msg = "CircuitPriorityHalflifeMsec in consensus";
-
- end:
-  /* We should never go below the EPSILON else we would consider it disabled
-   * and we can't have that. */
-  if (halflife < EPSILON) {
-    log_warn(LD_CONFIG, "CircuitPriorityHalflife is too small (%f). "
-                        "Adjusting to the smallest value allowed: %f.",
-             halflife, halflife_default);
-    halflife = halflife_default;
-  }
-  return halflife;
+/** Compute and return the current cell_ewma tick. */
+unsigned int
+cell_ewma_get_tick(void)
+{
+  return ((unsigned)approx_time() / EWMA_TICK_LEN);
 }
 
 /** Adjust the global cell scale factor based on <b>options</b> */
 void
-cmux_ewma_set_options(const or_options_t *options,
-                      const networkstatus_t *consensus)
+cell_ewma_set_scale_factor(const or_options_t *options,
+                           const networkstatus_t *consensus)
 {
+  int32_t halflife_ms;
   double halflife;
   const char *source;
+  if (options && options->CircuitPriorityHalflife >= -EPSILON) {
+    halflife = options->CircuitPriorityHalflife;
+    source = "CircuitPriorityHalflife in configuration";
+  } else if (consensus && (halflife_ms = networkstatus_get_param(
+                 consensus, "CircuitPriorityHalflifeMsec",
+                 -1, -1, INT32_MAX)) >= 0) {
+    halflife = ((double)halflife_ms)/1000.0;
+    source = "CircuitPriorityHalflifeMsec in consensus";
+  } else {
+    halflife = EWMA_DEFAULT_HALFLIFE;
+    source = "Default value";
+  }
 
-  cell_ewma_initialize_ticks();
-
-  /* Both options and consensus can be NULL. This assures us to either get a
-   * valid configured value or the default one. */
-  halflife = get_circuit_priority_halflife(options, consensus, &source);
-
-  /* convert halflife into halflife-per-tick. */
-  halflife /= EWMA_TICK_LEN;
-  /* compute per-tick scale factor. */
-  ewma_scale_factor = exp( LOG_ONEHALF / halflife );
-  log_info(LD_OR,
-           "Enabled cell_ewma algorithm because of value in %s; "
-           "scale factor is %f per %d seconds",
-           source, ewma_scale_factor, EWMA_TICK_LEN);
+  if (halflife <= EPSILON) {
+    /* The cell EWMA algorithm is disabled. */
+    ewma_scale_factor = 0.1;
+    ewma_enabled = 0;
+    log_info(LD_OR,
+             "Disabled cell_ewma algorithm because of value in %s",
+             source);
+  } else {
+    /* convert halflife into halflife-per-tick. */
+    halflife /= EWMA_TICK_LEN;
+    /* compute per-tick scale factor. */
+    ewma_scale_factor = exp( LOG_ONEHALF / halflife );
+    ewma_enabled = 1;
+    log_info(LD_OR,
+             "Enabled cell_ewma algorithm because of value in %s; "
+             "scale factor is %f per %d seconds",
+             source, ewma_scale_factor, EWMA_TICK_LEN);
+  }
 }
 
 /** Return the multiplier necessary to convert the value of a cell sent in
@@ -818,12 +763,3 @@ pop_first_cell_ewma(ewma_policy_data_t *pol)
                               offsetof(cell_ewma_t, heap_index));
 }
 
-/**
- * Drop all resources held by circuitmux_ewma.c, and deinitialize the
- * module. */
-void
-circuitmux_ewma_free_all(void)
-{
-  ewma_ticks_initialized = 0;
-}
-

src/or/circuitmux_ewma.h

@@ -12,19 +12,13 @@
 #include "or.h"
 #include "circuitmux.h"
 
-/* The public EWMA policy callbacks object. */
 extern circuitmux_policy_t ewma_policy;
 
 /* Externally visible EWMA functions */
-void cmux_ewma_set_options(const or_options_t *options,
-                           const networkstatus_t *consensus);
-
-void circuitmux_ewma_free_all(void);
-
-#ifdef CIRCUITMUX_EWMA_PRIVATE
-STATIC unsigned cell_ewma_get_current_tick_and_fraction(double *remainder_out);
-STATIC void cell_ewma_initialize_ticks(void);
-#endif
+int cell_ewma_enabled(void);
+unsigned int cell_ewma_get_tick(void);
+void cell_ewma_set_scale_factor(const or_options_t *options,
+                                const networkstatus_t *consensus);
 
 #endif /* !defined(TOR_CIRCUITMUX_EWMA_H) */
 

src/or/circuitstats.c

@@ -31,12 +31,10 @@
 #include "config.h"
 #include "confparse.h"
 #include "control.h"
-#include "crypto_rand.h"
 #include "main.h"
 #include "networkstatus.h"
 #include "rendclient.h"
 #include "rendservice.h"
-#include "router.h"
 #include "statefile.h"
 #include "circuitlist.h"
 #include "circuituse.h"
@@ -127,7 +125,7 @@ circuit_build_times_disabled_(const or_options_t *options,
       ignore_consensus ? 0 : networkstatus_get_param(NULL, "cbtdisabled",
                                                      0, 0, 1);
     int config_disabled = !options->LearnCircuitBuildTimeout;
-    int dirauth_disabled = authdir_mode(options);
+    int dirauth_disabled = options->AuthoritativeDir;
     int state_disabled = did_last_state_file_write_fail() ? 1 : 0;
     /* LearnCircuitBuildTimeout and Tor2web/Single Onion Services are
      * incompatible in two ways:

src/or/circuituse.c

@@ -55,6 +55,7 @@
 #include "rephist.h"
 #include "router.h"
 #include "routerlist.h"
+#include "config.h"
 
 static void circuit_expire_old_circuits_clientside(void);
 static void circuit_increment_failure_count(void);
@@ -1631,7 +1632,7 @@ circuit_testing_opened(origin_circuit_t *circ)
     router_perform_bandwidth_test(NUM_PARALLEL_TESTING_CIRCS, time(NULL));
     have_performed_bandwidth_test = 1;
   } else
-    router_do_reachability_checks(1, 0);
+    consider_testing_reachability(1, 0);
 }
 
 /** A testing circuit has failed to build. Take whatever stats we want. */
@@ -1756,39 +1757,6 @@ circuit_build_failed(origin_circuit_t *circ)
    * the last hop or an earlier hop. then use this info below.
    */
   int failed_at_last_hop = 0;
-
-  /* First, check to see if this was a path failure, rather than build
-   * failure.
-   *
-   * Note that we deliberately use circuit_get_cpath_len() (and not
-   * circuit_get_cpath_opened_len()) because we only want to ensure
-   * that a full path is *chosen*. This is different than a full path
-   * being *built*. We only want to count *build* failures below.
-   *
-   * Path selection failures can happen spuriously for a number
-   * of reasons (such as aggressive/invalid user-specified path
-   * restrictions in the torrc, insufficient microdescriptors, and
-   * non-user reasons like exitpolicy issues), and so should not be
-   * counted as failures below.
-   */
-  if (circuit_get_cpath_len(circ) < circ->build_state->desired_path_len) {
-    static ratelim_t pathfail_limit = RATELIM_INIT(3600);
-    log_fn_ratelim(&pathfail_limit, LOG_NOTICE, LD_CIRC,
-             "Our circuit %u (id: %" PRIu32 ") died due to an invalid "
-             "selected path, purpose %s. This may be a torrc "
-             "configuration issue, or a bug.",
-              TO_CIRCUIT(circ)->n_circ_id, circ->global_identifier,
-              circuit_purpose_to_string(TO_CIRCUIT(circ)->purpose));
-
-    /* If the path failed on an RP, retry it. */
-    if (TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND)
-      hs_circ_retry_service_rendezvous_point(circ);
-
-    /* In all other cases, just bail. The rest is just failure accounting
-     * that we don't want to do */
-    return;
-  }
-
   /* If the last hop isn't open, and the second-to-last is, we failed
    * at the last hop. */
   if (circ->cpath &&
@@ -1838,8 +1806,18 @@ circuit_build_failed(origin_circuit_t *circ)
        * If we have guard state (new guard API) and our path selection
        * code actually chose a full path, then blame the failure of this
        * circuit on the guard.
+       *
+       * Note that we deliberately use circuit_get_cpath_len() (and not
+       * circuit_get_cpath_opened_len()) because we only want to ensure
+       * that a full path is *chosen*. This is different than a full path
+       * being *built*. We only want to blame *build* failures on this
+       * guard. Path selection failures can happen spuriously for a number
+       * of reasons (such as aggressive/invalid user-specified path
+       * restrictions in the torrc, as well as non-user reasons like
+       * exitpolicy issues), and so should not be counted here.
        */
-      if (circ->guard_state)
+      if (circ->guard_state &&
+          circuit_get_cpath_len(circ) >= circ->build_state->desired_path_len)
         entry_guard_failed(&circ->guard_state);
       /* if there are any one-hop streams waiting on this circuit, fail
        * them now so they can retry elsewhere. */
@@ -2606,7 +2584,7 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ,
   log_debug(LD_APP|LD_CIRC, "attaching new conn to circ. n_circ_id %u.",
             (unsigned)circ->base_.n_circ_id);
   /* reset it, so we can measure circ timeouts */
-  ENTRY_TO_CONN(apconn)->timestamp_last_read_allowed = time(NULL);
+  ENTRY_TO_CONN(apconn)->timestamp_lastread = time(NULL);
   ENTRY_TO_EDGE_CONN(apconn)->next_stream = circ->p_streams;
   ENTRY_TO_EDGE_CONN(apconn)->on_circuit = TO_CIRCUIT(circ);
   /* assert_connection_ok(conn, time(NULL)); */
@@ -3106,41 +3084,3 @@ mark_circuit_unusable_for_new_conns(origin_circuit_t *circ)
   circ->unusable_for_new_conns = 1;
 }
 
-/**
- * Add relay_body_len and RELAY_PAYLOAD_SIZE-relay_body_len to
- * the valid delivered written fields and the overhead field,
- * respectively.
- */
-void
-circuit_sent_valid_data(origin_circuit_t *circ, uint16_t relay_body_len)
-{
-  if (!circ) return;
-
-  tor_assert_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE);
-
-  circ->n_delivered_written_circ_bw =
-      tor_add_u32_nowrap(circ->n_delivered_written_circ_bw, relay_body_len);
-  circ->n_overhead_written_circ_bw =
-      tor_add_u32_nowrap(circ->n_overhead_written_circ_bw,
-                         RELAY_PAYLOAD_SIZE-relay_body_len);
-}
-
-/**
- * Add relay_body_len and RELAY_PAYLOAD_SIZE-relay_body_len to
- * the valid delivered read field and the overhead field,
- * respectively.
- */
-void
-circuit_read_valid_data(origin_circuit_t *circ, uint16_t relay_body_len)
-{
-  if (!circ) return;
-
-  tor_assert_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE);
-
-  circ->n_delivered_read_circ_bw =
-      tor_add_u32_nowrap(circ->n_delivered_read_circ_bw, relay_body_len);
-  circ->n_overhead_read_circ_bw =
-      tor_add_u32_nowrap(circ->n_overhead_read_circ_bw,
-                         RELAY_PAYLOAD_SIZE-relay_body_len);
-}
-