Merge branch 'maint-0.2.8' into release-0.2.8

also reorder a stanza to put the more exciting bug first

.appveyor.yml

@@ -1,62 +0,0 @@
-version: 1.0.{build}
-
-clone_depth: 50
-
-environment:
-  compiler: mingw
-
-  matrix:
-  - target: i686-w64-mingw32
-    compiler_path: mingw32
-    openssl_path: /c/OpenSSL-Win32
-  - target: x86_64-w64-mingw32
-    compiler_path: mingw64
-    openssl_path: /c/OpenSSL-Win64
-
-install:
-- ps: >-
-    Function Execute-Command ($commandPath)
-    {
-        & $commandPath $args 2>&1
-        if ( $LastExitCode -ne 0 ) {
-            $host.SetShouldExit( $LastExitCode )
-        }
-    }
-    Function Execute-Bash ()
-    {
-        Execute-Command 'c:\msys64\usr\bin\bash' '-e' '-c' $args
-    }
-    Execute-Command "C:\msys64\usr\bin\pacman" -Sy --noconfirm openssl-devel openssl libevent-devel libevent mingw-w64-i686-libevent mingw-w64-x86_64-libevent mingw-w64-i686-openssl mingw-w64-x86_64-openssl mingw-w64-i686-zstd mingw-w64-x86_64-zstd
-
-build_script:
-- ps: >-
-    if ($env:compiler -eq "mingw") {
-            $oldpath = ${env:Path} -split ';'
-            $buildpath = @("C:\msys64\${env:compiler_path}\bin", "C:\msys64\usr\bin") + $oldpath
-            $env:Path = @($buildpath) -join ';'
-            $env:build = @("${env:APPVEYOR_BUILD_FOLDER}", $env:target) -join '\'
-            Set-Location "${env:APPVEYOR_BUILD_FOLDER}"
-            Execute-Bash 'autoreconf -i'
-            mkdir "${env:build}"
-            Set-Location "${env:build}"
-            Execute-Bash "../configure --prefix=/${env:compiler_path} --build=${env:target} --host=${env:target} --disable-asciidoc --enable-fatal-warnings --with-openssl-dir=${env:openssl_path}"
-            Execute-Bash "V=1 make -j2"
-            Execute-Bash "V=1 make -j2 install"
-     }
-
-test_script:
-- ps: >-
-    if ($env:compiler -eq "mingw") {
-            $oldpath = ${env:Path} -split ';'
-            $buildpath = @("C:\msys64\${env:compiler_path}\bin") + $oldpath
-            $env:Path = $buildpath -join ';'
-            Set-Location "${env:build}"
-            Execute-Bash "VERBOSE=1 make -j2 check"
-    }
-
-on_success:
-- cmd: C:\Python27\python.exe %APPVEYOR_BUILD_FOLDER%\scripts\test\appveyor-irc-notify.py irc.oftc.net:6697 tor-ci success
-
-on_failure:
-- cmd: C:\Python27\python.exe %APPVEYOR_BUILD_FOLDER%\scripts\test\appveyor-irc-notify.py irc.oftc.net:6697 tor-ci failure
-

.gitignore

@@ -3,7 +3,6 @@
 .#*
 *~
 *.swp
-*.swo
 # C stuff
 *.o
 *.obj
@@ -19,8 +18,6 @@
 .dirstamp
 *.trs
 *.log
-# Calltool stuff
-.*.graph
 # Stuff made by our makefiles
 *.bak
 # Python droppings
@@ -41,7 +38,6 @@ uptime-*.json
 /Makefile
 /Makefile.in
 /aclocal.m4
-/ar-lib
 /autom4te.cache
 /build-stamp
 /compile
@@ -60,7 +56,6 @@ uptime-*.json
 /stamp-h
 /stamp-h.in
 /stamp-h1
-/TAGS
 /test-driver
 /tor.sh
 /tor.spec
@@ -71,7 +66,6 @@ uptime-*.json
 /Tor*Bundle.dmg
 /tor-*-win32.exe
 /coverage_html/
-/callgraph/
 
 # /contrib/
 /contrib/dist/tor.sh
@@ -99,6 +93,11 @@ uptime-*.json
 /doc/tor.html
 /doc/tor.html.in
 /doc/tor.1.xml
+/doc/tor-fw-helper.1
+/doc/tor-fw-helper.1.in
+/doc/tor-fw-helper.html
+/doc/tor-fw-helper.html.in
+/doc/tor-fw-helper.1.xml
 /doc/tor-gencert.1
 /doc/tor-gencert.1.in
 /doc/tor-gencert.html
@@ -127,18 +126,12 @@ uptime-*.json
 /src/Makefile
 /src/Makefile.in
 
-# /src/trace
-/src/trace/libor-trace.a
-
 # /src/common/
 /src/common/Makefile
 /src/common/Makefile.in
 /src/common/libor.a
 /src/common/libor-testing.a
 /src/common/libor.lib
-/src/common/libor-ctime.a
-/src/common/libor-ctime-testing.a
-/src/common/libor-ctime.lib
 /src/common/libor-crypto.a
 /src/common/libor-crypto-testing.a
 /src/common/libor-crypto.lib
@@ -175,12 +168,6 @@ uptime-*.json
 /src/or/libtor-testing.a
 /src/or/libtor.lib
 
-# /src/rust
-/src/rust/.cargo/config
-/src/rust/.cargo/registry
-/src/rust/target
-/src/rust/registry
-
 # /src/test
 /src/test/Makefile
 /src/test/Makefile.in
@@ -192,27 +179,18 @@ uptime-*.json
 /src/test/test-child
 /src/test/test-memwipe
 /src/test/test-ntor-cl
-/src/test/test-hs-ntor-cl
 /src/test/test-switch-id
-/src/test/test-timers
 /src/test/test_workqueue
 /src/test/test.exe
 /src/test/test-slow.exe
 /src/test/test-bt-cl.exe
 /src/test/test-child.exe
 /src/test/test-ntor-cl.exe
-/src/test/test-hs-ntor-cl.exe
 /src/test/test-memwipe.exe
 /src/test/test-switch-id.exe
-/src/test/test-timers.exe
 /src/test/test_workqueue.exe
 
-# /src/test/fuzz
-/src/test/fuzz/fuzz-*
-/src/test/fuzz/lf-fuzz-*
-
 # /src/tools/
-/src/tools/libtorrunner.a
 /src/tools/tor-checkkey
 /src/tools/tor-resolve
 /src/tools/tor-cov-resolve
@@ -230,6 +208,12 @@ uptime-*.json
 /src/trunnel/libor-trunnel-testing.a
 /src/trunnel/libor-trunnel.a
 
+# /src/tools/tor-fw-helper/
+/src/tools/tor-fw-helper/tor-fw-helper
+/src/tools/tor-fw-helper/tor-fw-helper.exe
+/src/tools/tor-fw-helper/Makefile
+/src/tools/tor-fw-helper/Makefile.in
+
 # /src/win32/
 /src/win32/Makefile
 /src/win32/Makefile.in

.gitlab-ci.yml

@@ -1,45 +0,0 @@
-before_script:
-    - apt-get update -qq
-    - apt-get upgrade -qy
-
-build:
-  script:
-    - apt-get install -qy --fix-missing automake build-essential
-      libevent-dev libssl-dev zlib1g-dev
-      libseccomp-dev liblzma-dev libscrypt-dev
-    - ./autogen.sh
-    - ./configure --disable-asciidoc --enable-fatal-warnings
-      --disable-silent-rules
-    - make check || (e=$?; cat test-suite.log; exit $e)
-    - make install
-
-update:
-  only:
-    - schedules
-  script: 
-    - "apt-get install -y --fix-missing git openssh-client"
-    
-    # Run ssh-agent (inside the build environment)
-    - eval $(ssh-agent -s)
-
-    # Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
-    - ssh-add <(echo "$DEPLOY_KEY")
-
-    # For Docker builds disable host key checking. Be aware that by adding that
-    # you are susceptible to man-in-the-middle attacks.
-    # WARNING: Use this only with the Docker executor, if you use it with shell
-    # you will overwrite your user's SSH config.
-    - mkdir -p ~/.ssh
-    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
-    # In order to properly check the server's host key, assuming you created the
-    # SSH_SERVER_HOSTKEYS variable previously, uncomment the following two lines
-    # instead.
-    - mkdir -p ~/.ssh
-    - '[[ -f /.dockerenv ]] && echo "$SSH_SERVER_HOSTKEYS" > ~/.ssh/known_hosts'
-    - echo "merging from torgit"
-    - git config --global user.email "labadmin@oniongit.eu"
-    - git config --global user.name "gitadmin"
-    - "mkdir tor"
-    - "cd tor" 
-    - git clone --bare https://git.torproject.org/tor.git
-    - git push --mirror git@oniongit.eu:network/tor.git

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "src/ext/rust"]
-	path = src/ext/rust
-	url = https://git.torproject.org/tor-rust-dependencies

.travis.yml

@@ -1,10 +1,8 @@
 language: c
 
-## Comment out the compiler list for now to allow an explicit build
-## matrix.
-# compiler:
-#   - gcc
-#   - clang
+compiler:
+  - gcc
+  - clang
 
 notifications:
   irc:
@@ -30,10 +28,6 @@ dist: trusty
 
 ## We don't need sudo. (The "apt:" stanza after this allows us to not need sudo;
 ## otherwise, we would need it for getting dependencies.)
-##
-## We override this in the explicit build matrix to work around a
-## Travis CI environment regression
-## https://github.com/travis-ci/travis-ci/issues/9033
 sudo: false
 
 ## (Linux only) Download our dependencies
@@ -60,76 +54,18 @@ env:
   global:
     ## The Travis CI environment allows us two cores, so let's use both.
     - MAKEFLAGS="-j 2"
-  matrix:
-    ## Leave at least one entry here or Travis seems to generate a
-    ## matrix entry with empty matrix environment variables.  Leaving
-    ## more than one entry causes unwanted matrix entries with
-    ## unspecified compilers.
-    - RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
-    # - RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true
-    # - RUST_OPTIONS=""
 
 matrix:
-  ## Uncomment to allow the build to report success (with non-required
-  ## sub-builds continuing to run) if all required sub-builds have
-  ## succeeded.  This is somewhat buggy currently: it can cause
-  ## duplicate notifications and prematurely report success if a
-  ## single sub-build has succeeded.  See
-  ## https://github.com/travis-ci/travis-ci/issues/1696
-  # fast_finish: true
-
-  ## Uncomment the appropriate lines below to allow the build to
-  ## report success even if some less-critical sub-builds fail and it
-  ## seems likely to take a while for someone to fix it.  Currently
-  ## Travis CI doesn't distinguish "all builds succeeded" from "some
-  ## non-required sub-builds failed" except on the individual build's
-  ## page, which makes it somewhat annoying to detect from the
-  ## branches and build history pages.  See
-  ## https://github.com/travis-ci/travis-ci/issues/8716
-  allow_failures:
-    # - env: RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true
-    # - env: RUST_OPTIONS="--enable-rust --enable-cargo-online-mode
-    # - compiler: clang
-
-  ## Create explicit matrix entries to work around a Travis CI
-  ## environment issue.  Missing keys inherit from the first list
-  ## entry under that key outside the "include" clause.
-  include:
-    - compiler: gcc
-    - compiler: gcc
-      env: RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true
-    - compiler: gcc
-      env: RUST_OPTIONS=""
-    - compiler: gcc
-      env: COVERAGE_OPTIONS="--enable-coverage"
-    - compiler: gcc
-      env: DISTCHECK="yes" RUST_OPTIONS=""
-    - compiler: gcc
-      env: DISTCHECK="yes" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
-    - compiler: gcc
-      env: MODULES_OPTIONS="--disable-module-dirauth"
-    ## The "sudo: required" forces non-containerized builds, working
-    ## around a Travis CI environment issue: clang LeakAnalyzer fails
-    ## because it requires ptrace and the containerized environment no
-    ## longer allows ptrace.
-    - compiler: clang
-      sudo: required
-    - compiler: clang
-      sudo: required
-      env: RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true
-    - compiler: clang
-      sudo: required
-      env: RUST_OPTIONS=""
-    - compiler: clang
-      sudo: required
-      env: MODULES_OPTIONS="--disable-module-dirauth"
+  ## If one build in the matrix fails (e.g. if building withour Rust and Clang
+  ## fails, but building with Rust and GCC is still going), then cancel the
+  ## entire job early and call the whole thing a failure.
+  fast_finish: true
 
 before_install:
   ## If we're on OSX, homebrew usually needs to updated first
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update ; fi
   ## Download rustup
-  - if [[ "$RUST_OPTIONS" != "" ]]; then curl -Ssf -o rustup.sh https://sh.rustup.rs; fi
-  - if [[ "$COVERAGE_OPTIONS" != "" ]]; then pip install --user cpp-coveralls; fi
+  - curl -Ssf -o rustup.sh https://sh.rustup.rs
 
 install:
   ## If we're on OSX use brew to install required dependencies (for Linux, see the "apt:" section above)
@@ -140,30 +76,13 @@ install:
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then { brew outdated xz         || brew upgrade xz;         }; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then { brew outdated libscrypt  || brew upgrade libscrypt;  }; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then { brew outdated zstd       || brew upgrade zstd;       }; fi
-  ## Install the stable channels of rustc and cargo and setup our toolchain environment
-  - if [[ "$RUST_OPTIONS" != "" ]]; then sh rustup.sh -y --default-toolchain stable; fi
-  - if [[ "$RUST_OPTIONS" != "" ]]; then source $HOME/.cargo/env; fi
-  ## Get some info about rustc and cargo
-  - if [[ "$RUST_OPTIONS" != "" ]]; then which rustc; fi
-  - if [[ "$RUST_OPTIONS" != "" ]]; then which cargo; fi
-  - if [[ "$RUST_OPTIONS" != "" ]]; then rustc --version; fi
-  - if [[ "$RUST_OPTIONS" != "" ]]; then cargo --version; fi
-  ## If we're testing rust builds in offline-mode, then set up our vendored dependencies
-  - if [[ "$TOR_RUST_DEPENDENCIES" == "true" ]]; then export TOR_RUST_DEPENDENCIES=$PWD/src/ext/rust/crates; fi
 
 script:
   - ./autogen.sh
-  - ./configure $RUST_OPTIONS $COVERAGE_OPTIONS $MODULES_OPTIONS --disable-asciidoc --enable-fatal-warnings --disable-silent-rules --enable-fragile-hardening
+  - ./configure $RUST_OPTIONS --disable-asciidoc --enable-gcc-warnings --disable-silent-rules --enable-fragile-hardening
   ## We run `make check` because that's what https://jenkins.torproject.org does.
-  - if [[ "$DISTCHECK" == "" ]]; then make check; fi
-  - if [[ "$DISTCHECK" != "" ]]; then make distcheck DISTCHECK_CONFIGURE_FLAGS="$RUST_OPTIONS $COVERAGE_OPTIONS --disable-asciidoc --enable-fatal-warnings --disable-silent-rules --enable-fragile-hardening"; fi
+  - make check
 
 after_failure:
   ## `make check` will leave a log file with more details of test failures.
-  - if [[ "$DISTCHECK" == "" ]]; then cat test-suite.log; fi
-  ## `make distcheck` puts it somewhere different.
-  - if [[ "$DISTCHECK" != "" ]]; then make show-distdir-testlog; fi
-
-after_success:
-  ## If this build was one that produced coverage, upload it.
-  - if [[ "$COVERAGE_OPTIONS" != "" ]]; then coveralls -b . --exclude src/test --exclude src/trunnel --gcov-options '\-p'; fi
+  - cat test-suite.log

CONTRIBUTING

@@ -1,39 +0,0 @@
-Contributing to Tor
--------------------
-
-### Getting started
-
-Welcome!
-
-We have a bunch of documentation about how to develop Tor in the
-doc/HACKING/ directory.  We recommend that you start with
-doc/HACKING/README.1st.md , and then go from there.  It will tell
-you how to find your way around the source code, how to get
-involved with the Tor community, how to write patches, and much
-more!
-
-You don't have to be a C developer to help with Tor: have a look
-at https://www.torproject.org/getinvolved/volunteer !
-
-The Tor Project is committed to fostering a inclusive community
-where people feel safe to engage, share their points of view, and
-participate. For the latest version of our Code of Conduct, please
-see
-
-https://gitweb.torproject.org/community/policies.git/plain/code_of_conduct.txt
-
-
-
-### License issues
-
-Tor is distributed under the license terms in the LICENSE -- in
-brief, the "3-clause BSD license".  If you send us code to
-distribute with Tor, it needs to be code that we can distribute
-under those terms.  Please don't send us patches unless you agree
-to allow this.
-
-Some compatible licenses include:
-
-  - 3-clause BSD
-  - 2-clause BSD
-  - CC0 Public Domain Dedication

Doxyfile.in

@@ -446,6 +446,12 @@ MAX_INITIALIZER_LINES  = 30
 
 SHOW_USED_FILES        = YES
 
+# If the sources in your project are distributed over multiple directories 
+# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy 
+# in the documentation. The default is NO.
+
+SHOW_DIRECTORIES       = NO
+
 # Set the SHOW_FILES tag to NO to disable the generation of the Files page.
 # This will remove the Files entry from the Quick Index and from the 
 # Folder Tree View (if specified). The default is YES.
@@ -754,6 +760,12 @@ HTML_FOOTER            =
 
 HTML_STYLESHEET        = 
 
+# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, 
+# files or namespaces will be aligned in HTML using tables. If set to 
+# NO a bullet list will be used.
+
+HTML_ALIGN_MEMBERS     = YES
+
 # If the GENERATE_HTMLHELP tag is set to YES, additional index files 
 # will be generated that can be used as input for tools like the 
 # Microsoft HTML help workshop to generate a compiled HTML help file (.chm) 
@@ -1035,6 +1047,18 @@ GENERATE_XML           = NO
 
 XML_OUTPUT             = xml
 
+# The XML_SCHEMA tag can be used to specify an XML schema, 
+# which can be used by a validating XML parser to check the 
+# syntax of the XML files.
+
+XML_SCHEMA             = 
+
+# The XML_DTD tag can be used to specify an XML DTD, 
+# which can be used by a validating XML parser to check the 
+# syntax of the XML files.
+
+XML_DTD                = 
+
 # If the XML_PROGRAMLISTING tag is set to YES Doxygen will 
 # dump the program listings (including syntax highlighting 
 # and cross-referencing information) to the XML output. Note that 
@@ -1240,7 +1264,7 @@ HAVE_DOT               = NO
 # DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory 
 # containing the font.
 
-DOT_FONTNAME           =
+DOT_FONTNAME           = FreeSans
 
 # By default doxygen will tell dot to use the output directory to look for the 
 # FreeSans.ttf font (which doxygen will put there itself). If you specify a 

LICENSE

@@ -13,7 +13,7 @@ Tor is distributed under this license:
 
 Copyright (c) 2001-2004, Roger Dingledine
 Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
-Copyright (c) 2007-2017, The Tor Project, Inc.
+Copyright (c) 2007-2016, The Tor Project, Inc.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
@@ -270,110 +270,6 @@ src/ext/readpassphrase.[ch] are distributed under this license:
   Agency (DARPA) and Air Force Research Laboratory, Air Force
   Materiel Command, USAF, under agreement number F39502-99-1-0512.
 
-===============================================================================
-src/ext/mulodi4.c is distributed under this license:
-
-     =========================================================================
-     compiler_rt License
-     =========================================================================
-
-     The compiler_rt library is dual licensed under both the
-     University of Illinois "BSD-Like" license and the MIT license.
-     As a user of this code you may choose to use it under either
-     license.  As a contributor, you agree to allow your code to be
-     used under both.
-
-     Full text of the relevant licenses is included below.
-
-     =========================================================================
-
-     University of Illinois/NCSA
-     Open Source License
-
-     Copyright (c) 2009-2016 by the contributors listed in CREDITS.TXT
-
-     All rights reserved.
-
-     Developed by:
-
-         LLVM Team
-
-         University of Illinois at Urbana-Champaign
-
-         http://llvm.org
-
-     Permission is hereby granted, free of charge, to any person
-     obtaining a copy of this software and associated documentation
-     files (the "Software"), to deal with the Software without
-     restriction, including without limitation the rights to use,
-     copy, modify, merge, publish, distribute, sublicense, and/or sell
-     copies of the Software, and to permit persons to whom the
-     Software is furnished to do so, subject to the following
-     conditions:
-
-         * Redistributions of source code must retain the above
-           copyright notice, this list of conditions and the following
-           disclaimers.
-
-         * Redistributions in binary form must reproduce the above
-           copyright notice, this list of conditions and the following
-           disclaimers in the documentation and/or other materials
-           provided with the distribution.
-
-         * Neither the names of the LLVM Team, University of Illinois
-           at Urbana-Champaign, nor the names of its contributors may
-           be used to endorse or promote products derived from this
-           Software without specific prior written permission.
-
-     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-     NONINFRINGEMENT.  IN NO EVENT SHALL THE CONTRIBUTORS OR COPYRIGHT
-     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-     OTHER DEALINGS WITH THE SOFTWARE.
-
-     =========================================================================
-
-     Copyright (c) 2009-2015 by the contributors listed in CREDITS.TXT
-
-     Permission is hereby granted, free of charge, to any person
-     obtaining a copy of this software and associated documentation
-     files (the "Software"), to deal in the Software without
-     restriction, including without limitation the rights to use,
-     copy, modify, merge, publish, distribute, sublicense, and/or sell
-     copies of the Software, and to permit persons to whom the
-     Software is furnished to do so, subject to the following
-     conditions:
-
-     The above copyright notice and this permission notice shall be
-     included in all copies or substantial portions of the Software.
-
-     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-     OTHER DEALINGS IN THE SOFTWARE.
-
-     =========================================================================
-     Copyrights and Licenses for Third Party Software Distributed with LLVM:
-     =========================================================================
-
-     The LLVM software contains code written by third parties.  Such
-     software will have its own individual LICENSE.TXT file in the
-     directory in which it appears.  This file will describe the
-     copyrights, license, and restrictions which apply to that code.
-
-     The disclaimer of warranty in the University of Illinois Open
-     Source License applies to all code in the LLVM Distribution, and
-     nothing in any of the other licenses gives permission to use the
-     names of the LLVM Team or the University of Illinois to endorse
-     or promote products derived from this Software.
-
 ===============================================================================
 If you got Tor as a static binary with OpenSSL included, then you should know:
  "This product includes software developed by the OpenSSL Project

Makefile.am

@@ -1,6 +1,6 @@
 # Copyright (c) 2001-2004, Roger Dingledine
 # Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
-# Copyright (c) 2007-2017, The Tor Project, Inc.
+# Copyright (c) 2007-2015, The Tor Project, Inc.
 # See LICENSE for licensing information
 
 ACLOCAL_AMFLAGS = -I m4
@@ -9,27 +9,19 @@ noinst_LIBRARIES=
 EXTRA_DIST=
 noinst_HEADERS=
 bin_PROGRAMS=
-EXTRA_PROGRAMS=
 CLEANFILES=
 TESTS=
 noinst_PROGRAMS=
 DISTCLEANFILES=
 bin_SCRIPTS=
 AM_CPPFLAGS=
-AM_CFLAGS=@TOR_SYSTEMD_CFLAGS@ @CFLAGS_BUGTRAP@ @TOR_LZMA_CFLAGS@ @TOR_ZSTD_CFLAGS@
-SHELL=@SHELL@
+AM_CFLAGS = @TOR_SYSTEMD_CFLAGS@
+SHELL = @SHELL@
 
 if COVERAGE_ENABLED
-TESTING_TOR_BINARY=$(top_builddir)/src/or/tor-cov$(EXEEXT)
+TESTING_TOR_BINARY="$(top_builddir)/src/or/tor-cov"
 else
-TESTING_TOR_BINARY=$(top_builddir)/src/or/tor$(EXEEXT)
-endif
-
-if USE_RUST
-rust_ldadd=$(top_builddir)/src/rust/target/release/@TOR_RUST_STATIC_NAME@ \
-	@TOR_RUST_EXTRA_LIBS@
-else
-rust_ldadd=
+TESTING_TOR_BINARY="$(top_builddir)/src/or/tor"
 endif
 
 include src/include.am
@@ -38,36 +30,25 @@ include contrib/include.am
 
 EXTRA_DIST+= \
 	ChangeLog					\
-	CONTRIBUTING					\
 	INSTALL						\
 	LICENSE						\
 	Makefile.nmake					\
 	README						\
-	ReleaseNotes					\
-	scripts/maint/checkSpace.pl
-
-## This tells etags how to find mockable function definitions.
-AM_ETAGSFLAGS=--regex='{c}/MOCK_IMPL([^,]+,\W*\([a-zA-Z0-9_]+\)\W*,/\1/s'
+	ReleaseNotes
 
 if COVERAGE_ENABLED
 TEST_CFLAGS=-fno-inline -fprofile-arcs -ftest-coverage
 if DISABLE_ASSERTS_IN_UNIT_TESTS
-TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE -DDISABLE_ASSERTS_IN_UNIT_TESTS @TOR_MODULES_ALL_ENABLED@
+TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE -DDISABLE_ASSERTS_IN_UNIT_TESTS
 else
-TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE @TOR_MODULES_ALL_ENABLED@
+TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE
 endif
 TEST_NETWORK_FLAGS=--coverage --hs-multi-client 1
 else
 TEST_CFLAGS=
-TEST_CPPFLAGS=-DTOR_UNIT_TESTS @TOR_MODULES_ALL_ENABLED@
+TEST_CPPFLAGS=-DTOR_UNIT_TESTS
 TEST_NETWORK_FLAGS=--hs-multi-client 1
 endif
-TEST_NETWORK_WARNING_FLAGS=--quiet --only-warnings
-
-if LIBFUZZER_ENABLED
-TEST_CFLAGS += -fsanitize-coverage=trace-pc-guard,trace-cmp,trace-div
-# not "edge"
-endif
 
 TEST_NETWORK_ALL_LOG_DIR=$(top_builddir)/test_network_log
 TEST_NETWORK_ALL_DRIVER_FLAGS=--color-tests yes
@@ -98,8 +79,6 @@ doxygen:
 test: all
 	$(top_builddir)/src/test/test
 
-check-local: check-spaces check-changes
-
 need-chutney-path:
 	@if test ! -d "$$CHUTNEY_PATH"; then \
 		echo '$$CHUTNEY_PATH was not set.'; \
@@ -114,24 +93,22 @@ need-chutney-path:
 
 # Note that test-network requires a copy of Chutney in $CHUTNEY_PATH.
 # Chutney can be cloned from https://git.torproject.org/chutney.git .
-test-network: need-chutney-path $(TESTING_TOR_BINARY) src/tools/tor-gencert
+test-network: need-chutney-path all
 	$(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_FLAGS)
 
 # Run all available tests using automake's test-driver
 # only run IPv6 tests if we can ping6 ::1 (localhost)
-# only run IPv6 tests if we can ping ::1 (localhost)
 # some IPv6 tests will fail without an IPv6 DNS server (see #16971 and #17011)
 # only run mixed tests if we have a tor-stable binary
-# Try the syntax for BSD ping6, Linux ping6, and Linux ping -6,
-# because they're incompatible
-test-network-all: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/tor-gencert
+# see #17015 for autodetection of different tor versions
+test-network-all: need-chutney-path all test-driver
 	mkdir -p $(TEST_NETWORK_ALL_LOG_DIR)
 	@flavors="$(TEST_CHUTNEY_FLAVORS)"; \
-	if ping6 -q -c 1 -o ::1 >/dev/null 2>&1 || ping6 -q -c 1 -W 1 ::1 >/dev/null 2>&1 || ping -6 -c 1 -W 1 ::1 >/dev/null 2>&1; then \
-		echo "ping6 ::1 or ping ::1 succeeded, running IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
+	if ping6 -q -c 1 -o ::1 >/dev/null 2>&1; then \
+		echo "ping6 ::1 succeeded, running IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
 		flavors="$$flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \
 	else \
-		echo "ping6 ::1 and ping ::1 failed, skipping IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
+		echo "ping6 ::1 failed, skipping IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
 		skip_flavors="$$skip_flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \
 	fi; \
 	if command -v tor-stable >/dev/null 2>&1; then \
@@ -145,8 +122,7 @@ test-network-all: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/
 		echo "SKIP: $$f"; \
 	done; \
 	for f in $$flavors; do \
-		$(SHELL) $(top_srcdir)/test-driver --test-name $$f --log-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.log --trs-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.trs $(TEST_NETWORK_ALL_DRIVER_FLAGS) $(top_srcdir)/src/test/test-network.sh --flavor $$f $(TEST_NETWORK_FLAGS); \
-		$(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_WARNING_FLAGS); \
+		./test-driver --test-name $$f --log-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.log --trs-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.trs $(TEST_NETWORK_ALL_DRIVER_FLAGS) $(top_srcdir)/src/test/test-network.sh --flavor $$f $(TEST_NETWORK_FLAGS); \
 	done; \
 	echo "Log and result files are available in $(TEST_NETWORK_ALL_LOG_DIR)."; \
 	! grep -q FAIL test_network_log/*.trs
@@ -158,11 +134,11 @@ need-stem-path:
 		exit 1; \
 	fi
 
-test-stem: need-stem-path $(TESTING_TOR_BINARY)
-	@$(PYTHON) "$$STEM_SOURCE_DIR"/run_tests.py --tor "$(TESTING_TOR_BINARY)" --all --log notice --target RUN_ALL;
+test-stem: need-stem-path all
+	@$(PYTHON) "$$STEM_SOURCE_DIR"/run_tests.py --tor $(TESTING_TOR_BINARY) --all --log notice --target RUN_ALL;
 
-test-stem-full: need-stem-path $(TESTING_TOR_BINARY)
-	@$(PYTHON) "$$STEM_SOURCE_DIR"/run_tests.py --tor "$(TESTING_TOR_BINARY)" --all --log notice --target RUN_ALL,ONLINE -v;
+test-stem-full: need-stem-path all
+	@$(PYTHON) "$$STEM_SOURCE_DIR"/run_tests.py --tor $(TESTING_TOR_BINARY) --all --log notice --target RUN_ALL,ONLINE -v;
 
 test-full: need-stem-path need-chutney-path check test-network test-stem
 
@@ -200,16 +176,13 @@ coverage-html-full: all
 	genhtml --branch-coverage -o "$(HTML_COVER_DIR)" "$(HTML_COVER_DIR)/lcov.info"
 
 # Avoid strlcpy.c, strlcat.c, aes.c, OpenBSD_malloc_Linux.c, sha256.c,
-# tinytest*.[ch]
+# eventdns.[hc], tinytest*.[ch]
 check-spaces:
-if USE_PERL
-	$(PERL) $(top_srcdir)/scripts/maint/checkSpace.pl -C \
+	$(top_srcdir)/scripts/maint/checkSpace.pl -C \
 		$(top_srcdir)/src/common/*.[ch] \
 		$(top_srcdir)/src/or/*.[ch] \
 		$(top_srcdir)/src/test/*.[ch] \
-		$(top_srcdir)/src/test/*/*.[ch] \
 		$(top_srcdir)/src/tools/*.[ch]
-endif
 
 check-docs: all
 	$(PERL) $(top_builddir)/scripts/maint/checkOptionDocs.pl
@@ -218,42 +191,16 @@ check-logs:
 	$(top_srcdir)/scripts/maint/checkLogs.pl \
 		$(top_srcdir)/src/*/*.[ch] | sort -n
 
-.PHONY: check-typos
-check-typos:
-	@if test -x "`which misspell 2>&1;true`"; then \
-		echo "Checking for Typos ..."; \
-		(misspell \
-			$(top_srcdir)/src/[^e]*/*.[ch] \
-			$(top_srcdir)/doc \
-			$(top_srcdir)/contrib \
-			$(top_srcdir)/scripts \
-			$(top_srcdir)/README \
-			$(top_srcdir)/ChangeLog \
-			$(top_srcdir)/INSTALL \
-			$(top_srcdir)/ReleaseNotes \
-			$(top_srcdir)/LICENSE); \
-	else \
-		echo "Tor can use misspell to check for typos."; \
-		echo "It seems that you don't have misspell installed."; \
-		echo "You can install the latest version of misspell here: https://github.com/client9/misspell#install"; \
-	fi
-
 .PHONY: check-changes
 check-changes:
-if USEPYTHON
 	@if test -d "$(top_srcdir)/changes"; then \
-		$(PYTHON) $(top_srcdir)/scripts/maint/lintChanges.py $(top_srcdir)/changes; \
+		$(PYTHON) $(top_srcdir)/scripts/maint/lintChanges.py $(top_srcdir)/changes/*; \
 		fi
-endif
 
 .PHONY: update-versions
 update-versions:
 	$(PERL) $(top_builddir)/scripts/maint/updateVersions.pl
 
-.PHONY: callgraph
-callgraph:
-	$(top_builddir)/scripts/maint/run_calltool.sh
-
 version:
 	@echo "Tor @VERSION@"
 	@if test -d "$(top_srcdir)/.git" && test -x "`which git 2>&1;true`"; then \
@@ -266,20 +213,3 @@ mostlyclean-local:
 	rm -rf $(HTML_COVER_DIR)
 	rm -rf $(top_builddir)/doc/doxygen
 	rm -rf $(TEST_NETWORK_ALL_LOG_DIR)
-
-clean-local:
-	rm -rf $(top_builddir)/src/rust/target
-	rm -rf $(top_builddir)/src/rust/.cargo/registry
-
-if USE_RUST
-distclean-local: distclean-rust
-endif
-
-# This relies on some internal details of how automake implements
-# distcheck.  We check two directories because automake-1.15 changed
-# from $(distdir)/_build to $(distdir)/_build/sub.
-show-distdir-testlog:
-	@if test -d "$(distdir)/_build/sub"; then \
-	  cat $(distdir)/_build/sub/$(TEST_SUITE_LOG); \
-	else \
-	  cat $(distdir)/_build/$(TEST_SUITE_LOG); fi

README

@@ -27,6 +27,3 @@ Frequently Asked Questions:
 
 To get started working on Tor development:
         See the doc/HACKING directory.
-
-Release timeline:
-         https://trac.torproject.org/projects/tor/wiki/org/teams/NetworkTeam/CoreTorReleases

acinclude.m4

@@ -2,7 +2,7 @@ dnl Helper macros for Tor configure.ac
 dnl Copyright (c) 2001-2004, Roger Dingledine
 dnl Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
 dnl Copyright (c) 2007-2008, Roger Dingledine, Nick Mathewson
-dnl Copyright (c) 2007-2017, The Tor Project, Inc.
+dnl Copyright (c) 2007-2015, The Tor Project, Inc.
 dnl See LICENSE for licensing information
 
 AC_DEFUN([TOR_EXTEND_CODEPATH],
@@ -42,21 +42,20 @@ AC_DEFUN([TOR_DEFINE_CODEPATH],
   AC_SUBST(TOR_LDFLAGS_$2)
 ])
 
-dnl 1: flags
-dnl 2: try to link too if this is nonempty.
-dnl 3: what to do on success compiling
-dnl 4: what to do on failure compiling
-AC_DEFUN([TOR_TRY_COMPILE_WITH_CFLAGS], [
+dnl 1:flags
+dnl 2:also try to link (yes: non-empty string)
+dnl   will set yes or no in $tor_can_link_$1 (as modified by AS_VAR_PUSHDEF)
+AC_DEFUN([TOR_CHECK_CFLAGS], [
   AS_VAR_PUSHDEF([VAR],[tor_cv_cflags_$1])
   AC_CACHE_CHECK([whether the compiler accepts $1], VAR, [
     tor_saved_CFLAGS="$CFLAGS"
     CFLAGS="$CFLAGS -pedantic -Werror $1"
-    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[]])],
+    AC_TRY_COMPILE([], [return 0;],
                    [AS_VAR_SET(VAR,yes)],
                    [AS_VAR_SET(VAR,no)])
     if test x$2 != x; then
       AS_VAR_PUSHDEF([can_link],[tor_can_link_$1])
-      AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])],
+      AC_TRY_LINK([], [return 0;],
                   [AS_VAR_SET(can_link,yes)],
                   [AS_VAR_SET(can_link,no)])
       AS_VAR_POPDEF([can_link])
@@ -64,20 +63,11 @@ AC_DEFUN([TOR_TRY_COMPILE_WITH_CFLAGS], [
     CFLAGS="$tor_saved_CFLAGS"
   ])
   if test x$VAR = xyes; then
-     $3
-  else
-     $4
+    CFLAGS="$CFLAGS $1"
   fi
   AS_VAR_POPDEF([VAR])
 ])
 
-dnl 1:flags
-dnl 2:also try to link (yes: non-empty string)
-dnl   will set yes or no in $tor_can_link_$1 (as modified by AS_VAR_PUSHDEF)
-AC_DEFUN([TOR_CHECK_CFLAGS], [
-  TOR_TRY_COMPILE_WITH_CFLAGS($1, $2, CFLAGS="$CFLAGS $1", true)
-])
-
 dnl 1:flags
 dnl 2:extra ldflags
 dnl 3:extra libraries
@@ -93,7 +83,7 @@ AC_DEFUN([TOR_CHECK_LDFLAGS], [
     AC_RUN_IFELSE([AC_LANG_PROGRAM([#include <stdio.h>], [fputs("", stdout)])],
                   [AS_VAR_SET(VAR,yes)],
                   [AS_VAR_SET(VAR,no)],
-                  [AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])],
+	          [AC_TRY_LINK([], [return 0;],
                                    [AS_VAR_SET(VAR,yes)],
                                    [AS_VAR_SET(VAR,no)])])
     CFLAGS="$tor_saved_CFLAGS"
@@ -113,21 +103,21 @@ if test x$2 = xdevpkg; then
   h=" headers for"
 fi
 if test -f /etc/debian_version && test x"$tor_$1_$2_debian" != x; then
-  AC_MSG_WARN([On Debian, you can install$h $1 using "apt-get install $tor_$1_$2_debian"])
+  AC_WARN([On Debian, you can install$h $1 using "apt-get install $tor_$1_$2_debian"])
   if test x"$tor_$1_$2_debian" != x"$tor_$1_devpkg_debian"; then 
-    AC_MSG_WARN([   You will probably need $tor_$1_devpkg_debian too.])
+    AC_WARN([   You will probably need $tor_$1_devpkg_debian too.])
   fi 
 fi
 if test -f /etc/fedora-release && test x"$tor_$1_$2_redhat" != x; then
-  AC_MSG_WARN([On Fedora, you can install$h $1 using "dnf install $tor_$1_$2_redhat"])
+  AC_WARN([On Fedora, you can install$h $1 using "dnf install $tor_$1_$2_redhat"])
   if test x"$tor_$1_$2_redhat" != x"$tor_$1_devpkg_redhat"; then 
-    AC_MSG_WARN([   You will probably need to install $tor_$1_devpkg_redhat too.])
+    AC_WARN([   You will probably need to install $tor_$1_devpkg_redhat too.])
   fi 
 else
   if test -f /etc/redhat-release && test x"$tor_$1_$2_redhat" != x; then
-    AC_MSG_WARN([On most Redhat-based systems, you can get$h $1 by installing the $tor_$1_$2_redhat RPM package])
+    AC_WARN([On most Redhat-based systems, you can get$h $1 by installing the $tor_$1_$2_redhat RPM package])
     if test x"$tor_$1_$2_redhat" != x"$tor_$1_devpkg_redhat"; then 
-      AC_MSG_WARN([   You will probably need to install $tor_$1_devpkg_redhat too.])
+      AC_WARN([   You will probably need to install $tor_$1_devpkg_redhat too.])
     fi 
   fi
 fi
@@ -245,10 +235,7 @@ if test "$cross_compiling" != yes; then
        LDFLAGS="$tor_tryextra $orig_LDFLAGS"
      fi
      AC_RUN_IFELSE([AC_LANG_PROGRAM([$5], [$6])],
-                   [runnable=yes], [runnable=no],
-                   [AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])],
-                                   [runnable=yes],
-                                   [runnable=no])])
+                   [runnable=yes], [runnable=no])
      if test "$runnable" = yes; then
         tor_cv_library_$1_linker_option=$tor_tryextra
         break

autogen.sh

@@ -1,12 +1,12 @@
 #!/bin/sh
 
 if [ -x "`which autoreconf 2>/dev/null`" ] ; then
-  opt="-i -f -W all,error"
+  opt="-if"
 
   for i in $@; do
     case "$i" in
       -v)
-        opt="${opt} -v"
+        opt=$opt"v"
         ;;
     esac
   done

changes/TROVE-2018-005

@@ -1,6 +0,0 @@
-  o Major bugfixes (security, directory authority, denial-of-service):
-    - Fix a bug that could have allowed an attacker to force a
-      directory authority to use up all its RAM by passing it a
-      maliciously crafted protocol versions string. Fixes bug 25517;
-      bugfix on 0.2.9.4-alpha.  This issue is also tracked as
-      TROVE-2018-005.

changes/bug24167

@@ -0,0 +1,7 @@
+  o Minor bugfixes (network layer):
+    - When closing a connection via close_connection_immediately(), we
+      mark it as "not blocked on bandwidth", to prevent later calls
+      from trying to unblock it, and give it permission to read. This
+      fixes a backtrace warning that can happen on relays under various
+      circumstances. Fixes bug 24167; bugfix on 0.1.0.1-rc.
+

changes/bug24480

@@ -0,0 +1,3 @@
+  o Minor bugfixes (compilation):
+    - Fix a signed/unsigned comparison warning introduced by our
+      fix to TROVE-2017-009. Fixes bug 24480; bugfix on 0.2.5.16.

changes/bug24666

@@ -0,0 +1,7 @@
+  o Minor bugfixes (memory usage):
+
+    - When queuing DESTROY cells on a channel, only queue the
+      circuit-id and reason fields: not the entire 514-byte
+      cell. This fix should help mitigate any bugs or attacks that
+      fill up these queues, and free more RAM for other uses. Fixes
+      bug 24666; bugfix on 0.2.5.1-alpha.

changes/bug25939

@@ -1,3 +0,0 @@
-  o Minor bugfixes (onion services):
-    - Fix a bug that blocked the creation of ephemeral v3 onion services. Fixes
-      bug 25939; bugfix on 0.3.4.1-alpha.

changes/bug26101_26102

@@ -1,5 +0,0 @@
-  o Minor bugfixes (test coverage tools):
-    - Update our "cov-diff" script to handle output from the latest
-      version of gcov, and to remove extraneous timestamp information
-      from its output. Fixes bugs 26101 and 26102; bugfix on
-      0.2.5.1-alpha.

changes/bug26116

@@ -1,7 +0,0 @@
-  o Minor bugfixes (compatibility, openssl):
-    - Work around a change in OpenSSL 1.1.1 where
-      return values that would previously indicate "no password" now
-      indicate an empty password. Without this workaround, Tor instances
-      running with OpenSSL 1.1.1 would accept descriptors that other Tor
-      instances would reject. Fixes bug 26116; bugfix on 0.2.5.16.
-      

changes/bug26121

@@ -1,6 +0,0 @@
-  o Minor bugfixes (controller):
-    - Improve accuracy of the BUILDTIMEOUT_SET control port event's
-      TIMEOUT_RATE and CLOSE_RATE fields. (We were previously miscounting
-      the total number of circuits for these field values.) Fixes bug
-      26121; bugfix on 0.3.3.1-alpha.
-

changes/bug26156

@@ -1,3 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Fix compilation when building with OpenSSL 1.1.0 with the
-      "no-deprecated" flag enabled. Fixes bug 26156; bugfix on 0.3.4.1-alpha.

changes/bug26196

@@ -1,4 +0,0 @@
-  o Minor bugfixes (hardening):
-    - Prevent a possible out-of-bounds smartlist read in
-      protover_compute_vote(). Fixes bug 26196; bugfix on
-      0.2.9.4-alpha.

changes/bug26259

@@ -1,4 +0,0 @@
-  o Minor bugfixes (control port):
-    - Do not count 0-length RELAY_COMMAND_DATA cells as valid data in CIRC_BW
-      events. Previously, such cells were counted entirely in the OVERHEAD
-      field. Now they are not. Fixes bug 26259; bugfix on 0.3.4.1-alpha.

changes/doc25237

@@ -1,4 +0,0 @@
-  o Documentation:
-    - In code comment, point the reader to the exact section
-      in Tor specification that specifies circuit close error
-      code values. Resolves ticket 25237.

changes/geoip-2017-12-06

@@ -0,0 +1,4 @@
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the December 6 2017 Maxmind GeoLite2
+      Country database.
+

changes/ticket23856

@@ -0,0 +1,4 @@
+  o Minor feature (relay statistics): 
+    - Change relay bandwidth reporting stats interval from 4 hours to 24 hours
+      in order to reduce the efficiency of guard discovery attacks. Fixes
+      ticket 23856.

changes/ticket25549

@@ -1,4 +0,0 @@
-  o Minor features (continuous integration):
-    - Add the necessary configuration files for continuous integration
-      testing on Windows, via the Appveyor platform. Closes ticket 25549.
-      Patches from Marcin Cieślak and Isis Lovecruft.

contrib/clang/sanitize_blacklist.txt

@@ -2,17 +2,6 @@
 # syntax specified in http://clang.llvm.org/docs/SanitizerSpecialCaseList.html
 # for more info see http://clang.llvm.org/docs/AddressSanitizer.html
 
-#
-# Tor notes: This file is obsolete!
-#
-# It was necessary in order to apply the sanitizers to all of tor.  But
-# we don't believe that's a good idea: some parts of tor need constant-time
-# behavior that is hard to guarantee with these sanitizers.
-#
-# If you need this behavior, then please consider --enable-expensive-hardening,
-# and report bugs as needed.
-#
-
 # usage:
 # 1. configure tor build:
 #    ./configure \

contrib/operator-tools/linux-tor-prio.sh

@@ -87,7 +87,7 @@ RATE_UP=5000
 # machine does any other network activity. That is not very fun.
 RATE_UP_TOR=1500
 
-# RATE_UP_TOR_CEIL is the maximum rate allowed for all Tor traffic in
+# RATE_UP_TOR_CEIL is the maximum rate allowed for all Tor trafic in
 # kbits/sec.
 RATE_UP_TOR_CEIL=5000
 

contrib/win32build/tor-mingw.nsi.in

@@ -8,7 +8,7 @@
 !include "LogicLib.nsh"
 !include "FileFunc.nsh"
 !insertmacro GetParameters
-!define VERSION "0.3.4.1-alpha-dev"
+!define VERSION "0.2.8.17-dev"
 !define INSTALLER "tor-${VERSION}-win32.exe"
 !define WEBSITE "https://www.torproject.org/"
 !define LICENSE "LICENSE"

doc/HACKING/CodingStandards.md

@@ -3,11 +3,10 @@ Coding conventions for Tor
 
 tl;dr:
 
-   - Run configure with `--enable-fatal-warnings`
+   - Run configure with `--enable-gcc-warnings`
+   - Run `make check-spaces` to catch whitespace errors
    - Document your functions
    - Write unit tests
-   - Run `make check` before submitting a patch
-   - Run `make distcheck` if you have made changes to build system components
    - Add a file in `changes` for your branch.
 
 Patch checklist
@@ -22,43 +21,14 @@ preference)
 
 Did you remember...
 
-   - To build your code while configured with `--enable-fatal-warnings`?
+   - To build your code while configured with `--enable-gcc-warnings`?
+   - To run `make check-spaces` on your code?
    - To run `make check-docs` to see whether all new options are on
      the manpage?
    - To write unit tests, as possible?
-   - To run `make test-full` to test against all unit and integration tests (or
-     `make test-full-online` if you have a working connection to the internet)?
-   - To test that the distribution will actually work via `make distcheck`?
    - To base your code on the appropriate branch?
    - To include a file in the `changes` directory as appropriate?
 
-If you are submitting a major patch or new feature, or want to in the future...
-
-   - Set up Chutney and Stem, see HACKING/WritingTests.md
-   - Run `make test-full` to test against all unit and integration tests.
-
-If you have changed build system components:
-   - Please run `make distcheck`
-   - For example, if you have changed Makefiles, autoconf files, or anything
-     else that affects the build system.
-
-License issues
-==============
-
-Tor is distributed under the license terms in the LICENSE -- in
-brief, the "3-clause BSD license".  If you send us code to
-distribute with Tor, it needs to be code that we can distribute
-under those terms.  Please don't send us patches unless you agree
-to allow this.
-
-Some compatible licenses include:
-
-  - 3-clause BSD
-  - 2-clause BSD
-  - CC0 Public Domain Dedication
-
-
-
 How we use Git branches
 =======================
 
@@ -79,17 +49,8 @@ before it gets merged into maint, but that's rare.
 
 If you're working on a bugfix for a bug that occurs in a particular version,
 base your bugfix branch on the "maint" branch for the first supported series
-that has that bug.  (As of June 2013, we're supporting 0.2.3 and later.)
-
-If you're working on a new feature, base it on the master branch. If you're
-working on a new feature and it will take a while to implement and/or you'd
-like to avoid the possibility of unrelated bugs in Tor while you're
-implementing your feature, consider branching off of the latest maint- branch.
-_Never_ branch off a relase- branch. Don't branch off a tag either: they come
-from release branches. Doing so will likely produce a nightmare of merge
-conflicts in the ChangeLog when it comes time to merge your branch into Tor.
-Best advice: don't try to keep an independent branch forked for more than 6
-months and expect it to merge cleanly. Try to merge pieces early and often.
+that has that bug.  (As of June 2013, we're supporting 0.2.3 and later.) If
+you're working on a new feature, base it on the master branch.
 
 
 How we log changes
@@ -113,34 +74,17 @@ you can use `git describe --contains <sha1 of commit>`.
 If at all possible, try to create this file in the same commit where you are
 making the change.  Please give it a distinctive name that no other branch will
 use for the lifetime of your change. To verify the format of the changes file,
-you can use `make check-changes`.  This is run automatically as part of
-`make check` -- if it fails, we must fix it before we release.  These
-checks are implemented in `scripts/maint/lintChanges.py`.
-
-Changes file style guide:
-  * Changes files begin with "  o Header (subheading):".  The header
-    should usually be "Minor/Major bugfixes/features". The subheading is a
-    particular area within Tor.  See the ChangeLog for examples.
-
-  * Make everything terse.
-
-  * Write from the user's point of view: describe the user-visible changes
-    right away.
-
-  * Mention configuration options by name.  If they're rare or unusual,
-    remind people what they're for.
-
-  * Describe changes in the present tense and in the imperative: not past.
-
-  * Every bugfix should have a sentence of the form "Fixes bug 1234; bugfix
-    on 0.1.2.3-alpha", describing what bug was fixed and where it came from.
-
-  * "Relays", not "servers", "nodes", or "Tor relays".
+you can use `make check-changes`.
 
 When we go to make a release, we will concatenate all the entries
 in changes to make a draft changelog, and clear the directory. We'll
 then edit the draft changelog into a nice readable format.
 
+To make sure that stuff is in the right format, we use
+scripts/maint/lintChanges.py to check the changes files for
+(superficial) validity.  You can run this script on your own changes
+files!
+
 What needs a changes file?
 
    * A not-exhaustive list: Anything that might change user-visible
@@ -149,10 +93,6 @@ What needs a changes file?
    rewrites.  Anything about which somebody might plausibly wonder "when
    did that happen, and/or why did we do that" 6 months down the line.
 
-What does not need a changes file?
-
-   * Bugfixes for code that hasn't shipped in any released version of Tor
-
 Why use changes files instead of Git commit messages?
 
    * Git commit messages are written for developers, not users, and they
@@ -185,10 +125,10 @@ deviations from our C whitespace style.  Generally, we use:
      `puts (x)`.
    - Function declarations at the start of the line.
 
-We try hard to build without warnings everywhere.  In particular, if
-you're using gcc, you should invoke the configure script with the
-option `--enable-fatal-warnings`.  This will tell the compiler
-to make all warnings into errors.
+We try hard to build without warnings everywhere.  In particular, if you're
+using gcc, you should invoke the configure script with the option
+`--enable-gcc-warnings`.  This will give a bunch of extra warning flags to
+the compiler, and help us find divergences from our preferred C style.
 
 Functions to use; functions not to use
 --------------------------------------
@@ -209,79 +149,6 @@ old C functions.  Use `strlcat`, `strlcpy`, or `tor_snprintf/tor_asprintf` inste
 We don't call `memcmp()` directly.  Use `fast_memeq()`, `fast_memneq()`,
 `tor_memeq()`, or `tor_memneq()` for most purposes.
 
-Also see a longer list of functions to avoid in:
-https://people.torproject.org/~nickm/tor-auto/internal/this-not-that.html
-
-Floating point math is hard
----------------------------
-
-Floating point arithmetic as typically implemented by computers is
-very counterintuitive.  Failure to adequately analyze floating point
-usage can result in surprising behavior and even security
-vulnerabilities!
-
-General advice:
-
-   - Don't use floating point.
-   - If you must use floating point, document how the limits of
-     floating point precision and calculation accuracy affect function
-     outputs.
-   - Try to do as much as possible of your calculations using integers
-     (possibly acting as fixed-point numbers) and convert to floating
-     point for display.
-   - If you must send floating point numbers on the wire, serialize
-     them in a platform-independent way.  Tor avoids exchanging
-     floating-point values, but when it does, it uses ASCII numerals,
-     with a decimal point (".").
-   - Binary fractions behave very differently from decimal fractions.
-     Make sure you understand how these differences affect your
-     calculations.
-   - Every floating point arithmetic operation is an opportunity to
-     lose precision, overflow, underflow, or otherwise produce
-     undesired results.  Addition and subtraction tend to be worse
-     than multiplication and division (due to things like catastrophic
-     cancellation).  Try to arrange your calculations to minimize such
-     effects.
-   - Changing the order of operations changes the results of many
-     floating-point calculations.  Be careful when you simplify
-     calculations!  If the order is significant, document it using a
-     code comment.
-   - Comparing most floating point values for equality is unreliable.
-     Avoid using `==`, instead, use `>=` or `<=`.  If you use an
-     epsilon value, make sure it's appropriate for the ranges in
-     question.
-   - Different environments (including compiler flags and per-thread
-     state on a single platform!) can get different results from the
-     same floating point calculations.  This means you can't use
-     floats in anything that needs to be deterministic, like consensus
-     generation.  This also makes reliable unit tests of
-     floating-point outputs hard to write.
-
-For additional useful advice (and a little bit of background), see
-[What Every Programmer Should Know About Floating-Point
-Arithmetic](http://floating-point-gui.de/).
-
-A list of notable (and surprising) facts about floating point
-arithmetic is at [Floating-point
-complexities](https://randomascii.wordpress.com/2012/04/05/floating-point-complexities/).
-Most of that [series of posts on floating
-point](https://randomascii.wordpress.com/category/floating-point/) is
-helpful.
-
-For more detailed (and math-intensive) background, see [What Every
-Computer Scientist Should Know About Floating-Point
-Arithmetic](https://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html).
-
-Other C conventions
--------------------
-
-The `a ? b : c` trinary operator only goes inside other expressions;
-don't use it as a replacement for if. (You can ignore this inside macro
-definitions when necessary.)
-
-Assignment operators shouldn't nest inside other expressions.  (You can
-ignore this inside macro definitions when necessary.)
-
 Functions not to write
 ----------------------
 
@@ -343,64 +210,6 @@ end-users that they aren't expected to understand the message (perhaps
 with a string like "internal error"). Option (A) is to be preferred to
 option (B).
 
-Assertions In Tor
------------------
-
-Assertions should be used for bug-detection only.  Don't use assertions to
-detect bad user inputs, network errors, resource exhaustion, or similar
-issues.
-
-Tor is always built with assertions enabled, so try to only use
-`tor_assert()` for cases where you are absolutely sure that crashing is the
-least bad option.  Many bugs have been caused by use of `tor_assert()` when
-another kind of check would have been safer.
-
-If you're writing an assertion to test for a bug that you _can_ recover from,
-use `tor_assert_nonfatal()` in place of `tor_assert()`.  If you'd like to
-write a conditional that incorporates a nonfatal assertion, use the `BUG()`
-macro, as in:
-
-	if (BUG(ptr == NULL))
-		return -1;
-
-Allocator conventions
----------------------
-
-By convention, any tor type with a name like `abc_t` should be allocated
-by a function named `abc_new()`.  This function should never return
-NULL.
-
-Also, a type named `abc_t` should be freed by a function named `abc_free_()`.
-Don't call this `abc_free_()` function directly -- instead, wrap it in a
-macro called `abc_free()`, using the `FREE_AND_NULL` macro:
-
-    void abc_free_(abc_t *obj);
-    #define abc_free(obj) FREE_AND_NULL(abc_t, abc_free_, (obj))
-
-This macro will free the underlying `abc_t` object, and will also set
-the object pointer to NULL.
-
-You should define all `abc_free_()` functions to accept NULL inputs:
-
-    void
-    abc_free_(abc_t *obj)
-    {
-      if (!obj)
-        return;
-      tor_free(obj->name);
-      thing_free(obj->thing);
-      tor_free(obj);
-    }
-
-If you need a free function that takes a `void *` argument (for example,
-to use it as a function callback), define it with a name like
-`abc_free_void()`:
-
-    static void
-    abc_free_void_(void *obj)
-    {
-      abc_free_(obj);
-    }
 
 
 Doxygen comment conventions
@@ -434,4 +243,3 @@ the functions that call your function rely on it doing something, then your
 function should mention that it does that something in the documentation.  If
 you rely on a function doing something beyond what is in its documentation,
 then you should watch out, or it might do something else later.
-

doc/HACKING/CodingStandardsRust.md

@@ -1,523 +0,0 @@
-
- Rust Coding Standards
-=======================
-
-You MUST follow the standards laid out in `.../doc/HACKING/CodingStandards.md`,
-where applicable.
-
- Module/Crate Declarations
----------------------------
-
-Each Tor C module which is being rewritten MUST be in its own crate.
-See the structure of `.../src/rust` for examples.
-
-In your crate, you MUST use `lib.rs` ONLY for pulling in external
-crates (e.g. `extern crate libc;`) and exporting public objects from
-other Rust modules (e.g. `pub use mymodule::foo;`).  For example, if
-you create a crate in `.../src/rust/yourcrate`, your Rust code should
-live in `.../src/rust/yourcrate/yourcode.rs` and the public interface
-to it should be exported in `.../src/rust/yourcrate/lib.rs`.
-
-If your code is to be called from Tor C code, you MUST define a safe
-`ffi.rs`.  See the "Safety" section further down for more details.
-
-For example, in a hypothetical `tor_addition` Rust module:
-
-In `.../src/rust/tor_addition/addition.rs`:
-
-    pub fn get_sum(a: i32, b: i32) -> i32 {
-        a + b
-    }
-
-In `.../src/rust/tor_addition/lib.rs`:
-
-    pub use addition::*;
-
-In `.../src/rust/tor_addition/ffi.rs`:
-
-    #[no_mangle]
-    pub extern "C" fn tor_get_sum(a: c_int, b: c_int) -> c_int {
-        get_sum(a, b)
-    }
-
-If your Rust code must call out to parts of Tor's C code, you must
-declare the functions you are calling in the `external` crate, located
-at `.../src/rust/external`.
-
-<!-- XXX get better examples of how to declare these externs, when/how they -->
-<!-- XXX are unsafe, what they are expected to do —isis                     -->
-
-Modules should strive to be below 500 lines (tests excluded). Single
-responsibility and limited dependencies should be a guiding standard.
-
-If you have any external modules as dependencies (e.g. `extern crate
-libc;`), you MUST declare them in your crate's `lib.rs` and NOT in any
-other module.
-
- Dependencies and versions
----------------------------
-
-In general, we use modules from only the Rust standard library
-whenever possible. We will review including external crates on a
-case-by-case basis.
-
-If a crate only contains traits meant for compatibility between Rust
-crates, such as [the digest crate](https://crates.io/crates/digest) or
-[the failure crate](https://crates.io/crates/failure), it is very likely
-permissible to add it as a dependency.  However, a brief review should
-be conducted as to the usefulness of implementing external traits
-(i.e. how widespread is the usage, how many other crates either
-implement the traits or have trait bounds based upon them), as well as
-the stability of the traits (i.e. if the trait is going to change, we'll
-potentially have to re-do all our implementations of it).
-
-For large external libraries, especially which implement features which
-would be labour-intensive to reproduce/maintain ourselves, such as
-cryptographic or mathematical/statistics libraries, only crates which
-have stabilised to 1.0.0 should be considered, however, again, we may
-make exceptions on a case-by-case basis.
-
-Currently, Tor requires that you use the latest stable Rust version. At
-some point in the future, we will freeze on a given stable Rust version,
-to ensure backward compatibility with stable distributions that ship it.
-
- Updating/Adding Dependencies
-------------------------------
-
-To add/remove/update dependencies, first add your dependencies,
-exactly specifying their versions, into the appropriate *crate-level*
-`Cargo.toml` in `src/rust/` (i.e. *not* `/src/rust/Cargo.toml`, but
-instead the one for your crate).  Also, investigate whether your
-dependency has any optional dependencies which are unnecessary but are
-enabled by default.  If so, you'll likely be able to enable/disable
-them via some feature, e.g.:
-
-```toml
-[dependencies]
-foo = { version = "1.0.0", default-features = false }
-```
-
-Next, run `/scripts/maint/updateRustDependencies.sh`.  Then, go into
-`src/ext/rust` and commit the changes to the `tor-rust-dependencies`
-repo.
-
- Documentation
----------------
-
-You MUST include `#[deny(missing_docs)]` in your crate.
-
-For function/method comments, you SHOULD include a one-sentence, "first person"
-description of function behaviour (see requirements for documentation as
-described in `.../src/HACKING/CodingStandards.md`), then an `# Inputs` section
-for inputs or initialisation values, a `# Returns` section for return
-values/types, a `# Warning` section containing warnings for unsafe behaviours or
-panics that could happen.  For publicly accessible
-types/constants/objects/functions/methods, you SHOULD also include an
-`# Examples` section with runnable doctests.
-
-You MUST document your module with _module docstring_ comments,
-i.e. `//!` at the beginning of each line.
-
- Style
--------
-
-You SHOULD consider breaking up large literal numbers with `_` when it makes it
-more human readable to do so, e.g. `let x: u64 = 100_000_000_000`.
-
- Testing
----------
-
-All code MUST be unittested and integration tested.
-
-Public functions/objects exported from a crate SHOULD include doctests
-describing how the function/object is expected to be used.
-
-Integration tests SHOULD go into a `tests/` directory inside your
-crate.  Unittests SHOULD go into their own module inside the module
-they are testing, e.g. in `.../src/rust/tor_addition/addition.rs` you
-should put:
-
-    #[cfg(test)]
-    mod test {
-        use super::*;
-
-        #[test]
-        fn addition_with_zero() {
-            let sum: i32 = get_sum(5i32, 0i32);
-            assert_eq!(sum, 5);
-        }
-    }
-
- Benchmarking
---------------
-
-The external `test` crate can be used for most benchmarking.  However, using
-this crate requires nightly Rust.  Since we may want to switch to a more
-stable Rust compiler eventually, we shouldn't do things which will automatically
-break builds for stable compilers.  Therefore, you MUST feature-gate your
-benchmarks in the following manner.
-
-If you wish to benchmark some of your Rust code, you MUST put the
-following in the `[features]` section of your crate's `Cargo.toml`:
-
-    [features]
-    bench = []
-
-Next, in your crate's `lib.rs` you MUST put:
-
-    #[cfg(all(test, feature = "bench"))]
-    extern crate test;
-
-This ensures that the external crate `test`, which contains utilities
-for basic benchmarks, is only used when running benchmarks via `cargo
-bench --features bench`.
-
-Finally, to write your benchmark code, in
-`.../src/rust/tor_addition/addition.rs` you SHOULD put:
-
-    #[cfg(all(test, features = "bench"))]
-    mod bench {
-        use test::Bencher;
-        use super::*;
-
-        #[bench]
-        fn addition_small_integers(b: &mut Bencher) {
-            b.iter(| | get_sum(5i32, 0i32));
-        }
-    }
-
- Fuzzing
----------
-
-If you wish to fuzz parts of your code, please see the
-[`cargo fuzz`](https://github.com/rust-fuzz/cargo-fuzz) crate, which uses
-[libfuzzer-sys](https://github.com/rust-fuzz/libfuzzer-sys).
-
- Whitespace & Formatting
--------------------------
-
-You MUST run `rustfmt` (https://github.com/rust-lang-nursery/rustfmt)
-on your code before your code will be merged.  You can install rustfmt
-by doing `cargo install rustfmt-nightly` and then run it with `cargo
-fmt`.
-
- Safety
---------
-
-You SHOULD read [the nomicon](https://doc.rust-lang.org/nomicon/) before writing
-Rust FFI code.  It is *highly advised* that you read and write normal Rust code
-before attempting to write FFI or any other unsafe code.
-
-Here are some additional bits of advice and rules:
-
-0. Any behaviours which Rust considers to be undefined are forbidden
-
-   From https://doc.rust-lang.org/reference/behavior-considered-undefined.html:
-
-   > Behavior considered undefined
-   >
-   > The following is a list of behavior which is forbidden in all Rust code,
-   > including within unsafe blocks and unsafe functions. Type checking provides the
-   > guarantee that these issues are never caused by safe code.
-   > 
-   > * Data races
-   > * Dereferencing a null/dangling raw pointer
-   > * Reads of [undef](http://llvm.org/docs/LangRef.html#undefined-values)
-   >   (uninitialized) memory
-   > * Breaking the
-   >   [pointer aliasing rules](http://llvm.org/docs/LangRef.html#pointer-aliasing-rules)
-   >   with raw pointers (a subset of the rules used by C)
-   > * `&mut T` and `&T` follow LLVM’s scoped noalias model, except if the `&T`
-   >   contains an `UnsafeCell<U>`. Unsafe code must not violate these aliasing
-   >   guarantees.
-   > * Mutating non-mutable data (that is, data reached through a shared
-   >   reference or data owned by a `let` binding), unless that data is
-   >   contained within an `UnsafeCell<U>`.
-   > * Invoking undefined behavior via compiler intrinsics:
-   >     - Indexing outside of the bounds of an object with
-   >       `std::ptr::offset` (`offset` intrinsic), with the exception of
-   >       one byte past the end which is permitted.
-   >     - Using `std::ptr::copy_nonoverlapping_memory` (`memcpy32`/`memcpy64`
-   >       intrinsics) on overlapping buffers
-   > * Invalid values in primitive types, even in private fields/locals:
-   >     - Dangling/null references or boxes
-   >     - A value other than `false` (0) or `true` (1) in a `bool`
-   >     - A discriminant in an `enum` not included in the type definition
-   >     - A value in a `char` which is a surrogate or above `char::MAX`
-   >     - Non-UTF-8 byte sequences in a `str`
-   > * Unwinding into Rust from foreign code or unwinding from Rust into foreign
-   >   code. Rust's failure system is not compatible with exception handling in other
-   >   languages. Unwinding must be caught and handled at FFI boundaries.
-
-1. `unwrap()`
-
-   If you call `unwrap()`, anywhere, even in a test, you MUST include
-   an inline comment stating how the unwrap will either 1) never fail,
-   or 2) should fail (i.e. in a unittest).
-
-   You SHOULD NOT use `unwrap()` anywhere in which it is possible to handle the
-   potential error with either `expect()` or the eel operator, `?`.
-   For example, consider a function which parses a string into an integer:
-
-        fn parse_port_number(config_string: &str) -> u16 {
-            u16::from_str_radix(config_string, 10).unwrap()
-        }
-
-   There are numerous ways this can fail, and the `unwrap()` will cause the
-   whole program to byte the dust!  Instead, either you SHOULD use `expect()`
-   (or another equivalent function which will return an `Option` or a `Result`)
-   and change the return type to be compatible:
-
-        fn parse_port_number(config_string: &str) -> Option<u16> {
-            u16::from_str_radix(config_string, 10).expect("Couldn't parse port into a u16")
-        }
-
-   or you SHOULD use `or()` (or another similar method):
-
-        fn parse_port_number(config_string: &str) -> Option<u16> {
-            u16::from_str_radix(config_string, 10).or(Err("Couldn't parse port into a u16")
-        }
-
-   Using methods like `or()` can be particularly handy when you must do
-   something afterwards with the data, for example, if we wanted to guarantee
-   that the port is high.  Combining these methods with the eel operator (`?`)
-   makes this even easier:
-
-        fn parse_port_number(config_string: &str) -> Result<u16, Err> {
-            let port = u16::from_str_radix(config_string, 10).or(Err("Couldn't parse port into a u16"))?;
-
-            if port > 1024 {
-                return Ok(port);
-            } else {
-                return Err("Low ports not allowed");
-            }
-        }
-
-2. `unsafe`
-
-   If you use `unsafe`, you MUST describe a contract in your
-   documentation which describes how and when the unsafe code may
-   fail, and what expectations are made w.r.t. the interfaces to
-   unsafe code.  This is also REQUIRED for major pieces of FFI between
-   C and Rust.
-
-   When creating an FFI in Rust for C code to call, it is NOT REQUIRED
-   to declare the entire function `unsafe`.  For example, rather than doing:
-
-        #[no_mangle]
-        pub unsafe extern "C" fn increment_and_combine_numbers(mut numbers: [u8; 4]) -> u32 {
-            for number in &mut numbers {
-                *number += 1;
-            }
-            std::mem::transmute::<[u8; 4], u32>(numbers)
-        }
-
-   You SHOULD instead do:
-
-        #[no_mangle]
-        pub extern "C" fn increment_and_combine_numbers(mut numbers: [u8; 4]) -> u32 {
-            for index in 0..numbers.len() {
-                numbers[index] += 1;
-            }
-            unsafe {
-                std::mem::transmute::<[u8; 4], u32>(numbers)
-            }
-        }
-
-3. Pass only C-compatible primitive types and bytes over the boundary
-
-   Rust's C-compatible primitive types are integers and floats.
-   These types are declared in the [libc crate](https://doc.rust-lang.org/libc/x86_64-unknown-linux-gnu/libc/index.html#types).
-   Most Rust objects have different [representations](https://doc.rust-lang.org/libc/x86_64-unknown-linux-gnu/libc/index.html#types)
-   in C and Rust, so they can't be passed using FFI.
-
-   Tor currently uses the following Rust primitive types from libc for FFI:
-   * defined-size integers: `uint32_t`
-   * native-sized integers: `c_int`
-   * native-sized floats: `c_double`
-   * native-sized raw pointers: `* c_void`, `* c_char`, `** c_char`
-
-   TODO: C smartlist to Stringlist conversion using FFI
-
-   The only non-primitive type which may cross the FFI boundary is
-   bytes, e.g. `&[u8]`.  This SHOULD be done on the Rust side by
-   passing a pointer (`*mut libc::c_char`). The length can be passed
-   explicitly (`libc::size_t`), or the string can be NUL-byte terminated
-   C string.
-
-   One might be tempted to do this via doing
-   `CString::new("blah").unwrap().into_raw()`. This has several problems:
-
-   a) If you do `CString::new("bl\x00ah")` then the unwrap() will fail
-      due to the additional NULL terminator, causing a dangling
-      pointer to be returned (as well as a potential use-after-free).
-
-   b) Returning the raw pointer will cause the CString to run its deallocator,
-      which causes any C code which tries to access the contents to dereference a
-      NULL pointer.
-
-   c) If we were to do `as_raw()` this would result in a potential double-free
-      since the Rust deallocator would run and possibly Tor's deallocator.
-
-   d) Calling `into_raw()` without later using the same pointer in Rust to call
-      `from_raw()` and then deallocate in Rust can result in a
-      [memory leak](https://doc.rust-lang.org/std/ffi/struct.CString.html#method.into_raw).
-
-      [It was determined](https://github.com/rust-lang/rust/pull/41074) that this
-      is safe to do if you use the same allocator in C and Rust and also specify
-      the memory alignment for CString (except that there is no way to specify
-      the alignment for CString).  It is believed that the alignment is always 1,
-      which would mean it's safe to dealloc the resulting `*mut c_char` in Tor's
-      C code.  However, the Rust developers are not willing to guarantee the
-      stability of, or a contract for, this behaviour, citing concerns that this
-      is potentially extremely and subtly unsafe.
-
-4. Perform an allocation on the other side of the boundary
-
-   After crossing the boundary, the other side MUST perform an
-   allocation to copy the data and is therefore responsible for
-   freeing that memory later.
-
-5. No touching other language's enums
-
-   Rust enums should never be touched from C (nor can they be safely
-   `#[repr(C)]`) nor vice versa:
-
-   >  "The chosen size is the default enum size for the target platform's C
-   >  ABI. Note that enum representation in C is implementation defined, so this is
-   >  really a "best guess". In particular, this may be incorrect when the C code
-   >  of interest is compiled with certain flags."
-
-   (from https://gankro.github.io/nomicon/other-reprs.html)
-
-6. Type safety
-
-   Wherever possible and sensical, you SHOULD create new types in a
-   manner which prevents type confusion or misuse.  For example,
-   rather than using an untyped mapping between strings and integers
-   like so:
-
-        use std::collections::HashMap;
-
-        pub fn get_elements_with_over_9000_points(map: &HashMap<String, usize>) -> Vec<String> {
-            ...
-        }
-
-   It would be safer to define a new type, such that some other usage
-   of `HashMap<String, usize>` cannot be confused for this type:
-
-        pub struct DragonBallZPowers(pub HashMap<String, usize>);
-
-        impl DragonBallZPowers {
-            pub fn over_nine_thousand<'a>(&'a self) -> Vec<&'a String> {
-                let mut powerful_enough: Vec<&'a String> = Vec::with_capacity(5);
-
-                for (character, power) in &self.0 {
-                    if *power > 9000 {
-                        powerful_enough.push(character);
-                    }
-                }
-                powerful_enough
-            }
-        }
-
-   Note the following code, which uses Rust's type aliasing, is valid
-   but it does NOT meet the desired type safety goals:
-
-        pub type Power = usize;
-
-        pub fn over_nine_thousand(power: &Power) -> bool {
-            if *power > 9000 {
-                return true;
-            }
-            false
-        }
-
-        // We can still do the following:
-        let his_power: usize = 9001;
-        over_nine_thousand(&his_power);
-
-7. Unsafe mucking around with lifetimes
-
-   Because lifetimes are technically, in type theory terms, a kind, i.e. a
-   family of types, individual lifetimes can be treated as types.  For example,
-   one can arbitrarily extend and shorten lifetime using `std::mem::transmute`:
-
-        struct R<'a>(&'a i32);
-
-        unsafe fn extend_lifetime<'b>(r: R<'b>) -> R<'static> {
-            std::mem::transmute::<R<'b>, R<'static>>(r)
-        }
-
-        unsafe fn shorten_invariant_lifetime<'b, 'c>(r: &'b mut R<'static>) -> &'b mut R<'c> {
-            std::mem::transmute::<&'b mut R<'static>, &'b mut R<'c>>(r)
-        }
-
-   Calling `extend_lifetime()` would cause an `R` passed into it to live forever
-   for the life of the program (the `'static` lifetime).  Similarly,
-   `shorten_invariant_lifetime()` could be used to take something meant to live
-   forever, and cause it to disappear!  This is incredibly unsafe.  If you're
-   going to be mucking around with lifetimes like this, first, you better have
-   an extremely good reason, and second, you may as be honest and explicit about
-   it, and for ferris' sake just use a raw pointer.
-
-   In short, just because lifetimes can be treated like types doesn't mean you
-   should do it.
-
-8. Doing excessively unsafe things when there's a safer alternative
-
-   Similarly to #7, often there are excessively unsafe ways to do a task and a
-   simpler, safer way.  You MUST choose the safer option where possible.
-
-   For example, `std::mem::transmute` can be abused in ways where casting with
-   `as` would be both simpler and safer:
-
-        // Don't do this
-        let ptr = &0;
-        let ptr_num_transmute = unsafe { std::mem::transmute::<&i32, usize>(ptr)};
-
-        // Use an `as` cast instead
-        let ptr_num_cast = ptr as *const i32 as usize;
-
-   In fact, using `std::mem::transmute` for *any* reason is a code smell and as
-   such SHOULD be avoided.
-
-9. Casting integers with `as`
-
-   This is generally fine to do, but it has some behaviours which you should be
-   aware of.  Casting down chops off the high bits, e.g.:
-
-        let x: u32 = 4294967295;
-        println!("{}", x as u16); // prints 65535
-
-   Some cases which you MUST NOT do include:
-
-   * Casting an `u128` down to an `f32` or vice versa (e.g.
-     `u128::MAX as f32` but this isn't only a problem with overflowing
-     as it is also undefined behaviour for `42.0f32 as u128`),
-
-   * Casting between integers and floats when the thing being cast
-     cannot fit into the type it is being casted into, e.g.:
-
-         println!("{}", 42949.0f32 as u8); // prints 197 in debug mode and 0 in release
-         println!("{}", 1.04E+17 as u8);   // prints 0 in both modes
-         println!("{}", (0.0/0.0) as i64); // prints whatever the heck LLVM wants
-
-     Because this behaviour is undefined, it can even produce segfaults in
-     safe Rust code.  For example, the following program built in release
-     mode segfaults:
-
-         #[inline(never)]
-         pub fn trigger_ub(sl: &[u8; 666]) -> &[u8] {
-             // Note that the float is out of the range of `usize`, invoking UB when casting.
-             let idx = 1e99999f64 as usize;
-             &sl[idx..] // The bound check is elided due to `idx` being of an undefined value.
-         }
-
-         fn main() {
-             println!("{}", trigger_ub(&[1; 666])[999999]); // ~ out of bound
-         }
-
-      And in debug mode panics with:
-
-         thread 'main' panicked at 'slice index starts at 140721821254240 but ends at 666', /checkout/src/libcore/slice/mod.rs:754:4

doc/HACKING/Fuzzing.md

@@ -1,123 +0,0 @@
-= Fuzzing Tor
-
-== The simple version (no fuzzing, only tests)
-
-Check out fuzzing-corpora, and set TOR_FUZZ_CORPORA to point to the place
-where you checked it out.
-
-To run the fuzzing test cases in a deterministic fashion, use:
-      make test-fuzz-corpora
-
-This won't actually fuzz Tor!  It will just run all the fuzz binaries
-on our existing set of testcases for the fuzzer.
-
-
-== Different kinds of fuzzing
-
-Right now we support three different kinds of fuzzer.
-
-First, there's American Fuzzy Lop (AFL), a fuzzer that works by forking
-a target binary and passing it lots of different inputs on stdin.  It's the
-trickiest one to set up, so I'll be describing it more below.
-
-Second, there's libFuzzer, a llvm-based fuzzer that you link in as a library,
-and it runs a target function over and over.  To use this one, you'll need to
-have a reasonably recent clang and libfuzzer installed.  At that point, you
-just build with --enable-expensive-hardening and --enable-libfuzzer.  That
-will produce a set of binaries in src/test/fuzz/lf-fuzz-* .  These programs
-take as input a series of directories full of fuzzing examples.  For more
-information on libfuzzer, see http://llvm.org/docs/LibFuzzer.html
-
-Third, there's Google's OSS-Fuzz infrastructure, which expects to get all of
-its.  For more on this, see https://github.com/google/oss-fuzz and the
-projects/tor subdirectory.  You'll need to mess around with Docker a bit to
-test this one out; it's meant to run on Google's infrastructure.
-
-In all cases, you'll need some starting examples to give the fuzzer when it
-starts out.  There's a set in the "fuzzing-corpora" git repository.  Try
-setting TOR_FUZZ_CORPORA to point to a checkout of that repository
-
-== Writing Tor fuzzers
-
-A tor fuzzing harness should have:
-* a fuzz_init() function to set up any necessary global state.
-* a fuzz_main() function to receive input and pass it to a parser.
-* a fuzz_cleanup() function to clear global state.
-
-Most fuzzing frameworks will produce many invalid inputs - a tor fuzzing
-harness should rejecting invalid inputs without crashing or behaving badly.
-
-But the fuzzing harness should crash if tor fails an assertion, triggers a
-bug, or accesses memory it shouldn't. This helps fuzzing frameworks detect
-"interesting" cases.
-
-
-== Guided Fuzzing with AFL
-
-There is no HTTPS, hash, or signature for American Fuzzy Lop's source code, so
-its integrity can't be verified. That said, you really shouldn't fuzz on a
-machine you care about, anyway.
-
-To Build:
-  Get AFL from http://lcamtuf.coredump.cx/afl/ and unpack it
-  cd afl
-  make
-  cd ../tor
-  PATH=$PATH:../afl/ CC="../afl/afl-gcc" ./configure --enable-expensive-hardening
-  AFL_HARDEN=1 make clean fuzzers
-
-To Find The ASAN Memory Limit: (64-bit only)
-
-On 64-bit platforms, afl needs to know how much memory ASAN uses,
-because ASAN tends to allocate a ridiculous amount of virtual memory,
-and then not actually use it.
-
-Read afl/docs/notes_for_asan.txt for more details.
-
-  Download recidivm from http://jwilk.net/software/recidivm
-  Download the signature
-  Check the signature
-  tar xvzf recidivm*.tar.gz
-  cd recidivm*
-  make
-  /path/to/recidivm -v src/test/fuzz/fuzz-http
-  Use the final "ok" figure as the input to -m when calling afl-fuzz
-  (Normally, recidivm would output a figure automatically, but in some cases,
-  the fuzzing harness will hang when the memory limit is too small.)
-
-You could also just say "none" instead of the memory limit below, if you
-don't care about memory limits.
-
-
-To Run:
-  mkdir -p src/test/fuzz/fuzz_http_findings
-  ../afl/afl-fuzz -i ${TOR_FUZZ_CORPORA}/http -o src/test/fuzz/fuzz_http_findings -m <asan-memory-limit> -- src/test/fuzz/fuzz-http
-
-
-AFL has a multi-core mode, check the documentation for details.
-You might find the included fuzz-multi.sh script useful for this.
-
-macOS (OS X) requires slightly more preparation, including:
-* using afl-clang (or afl-clang-fast from the llvm directory)
-* disabling external crash reporting (AFL will guide you through this step)
-
-== Triaging Issues
-
-Crashes are usually interesting, particularly if using AFL_HARDEN=1 and --enable-expensive-hardening. Sometimes crashes are due to bugs in the harness code.
-
-Hangs might be interesting, but they might also be spurious machine slowdowns.
-Check if a hang is reproducible before reporting it. Sometimes, processing
-valid inputs may take a second or so, particularly with the fuzzer and
-sanitizers enabled.
-
-To see what fuzz-http is doing with a test case, call it like this:
-  src/test/fuzz/fuzz-http --debug < /path/to/test.case
-
-(Logging is disabled while fuzzing to increase fuzzing speed.)
-
-== Reporting Issues
-
-Please report any issues discovered using the process in Tor's security issue
-policy:
-
-https://trac.torproject.org/projects/tor/wiki/org/meetings/2016SummerDevMeeting/Notes/SecurityIssuePolicy

doc/HACKING/GettingStarted.md

@@ -11,9 +11,8 @@ whole Tor ecosystem.)
 
 
 If you are looking for a more bare-bones, less user-friendly information
-dump of important information, you might like reading the "torguts"
-documents linked to below. You should probably read it before you write
-your first patch.
+dump of important information, you might like reading doc/HACKING
+instead.  You should probably read it before you write your first patch.
 
 
 Required background

doc/HACKING/GettingStartedRust.md

@@ -1,181 +0,0 @@
-
- Hacking on Rust in Tor
-========================
-
- Getting Started
------------------
-
-Please read or review our documentation on Rust coding standards
-(`.../doc/HACKING/CodingStandardsRust.md`) before doing anything.
-
-Please also read
-[the Rust Code of Conduct](https://www.rust-lang.org/en-US/conduct.html). We
-aim to follow the good example set by the Rust community and be
-excellent to one another.  Let's be careful with each other, so we can
-be memory-safe together!
-
-Next, please contact us before rewriting anything!  Rust in Tor is still
-an experiment.  It is an experiment that we very much want to see
-succeed, so we're going slowly and carefully.  For the moment, it's also
-a completely volunteer-driven effort: while many, if not most, of us are
-paid to work on Tor, we are not yet funded to write Rust code for Tor.
-Please be patient with the other people who are working on getting more
-Rust code into Tor, because they are graciously donating their free time
-to contribute to this effort.
-
- Resources for learning Rust
------------------------------
-
-**Beginning resources**
-
-The primary resource for learning Rust is
-[The Book](https://doc.rust-lang.org/book/).  If you'd like to start writing
-Rust immediately, without waiting for anything to install, there is
-[an interactive browser-based playground](https://play.rust-lang.org/).
-
-**Advanced resources**
-
-If you're interested in playing with various Rust compilers and viewing
-a very nicely displayed output of the generated assembly, there is
-[the Godbolt compiler explorer](https://rust.godbolt.org/)
-
-For learning how to write unsafe Rust, read
-[The Rustonomicon](https://doc.rust-lang.org/nomicon/).
-
-For learning everything you ever wanted to know about Rust macros, there
-is
-[The Little Book of Rust Macros](https://danielkeep.github.io/tlborm/book/index.html).
-
-For learning more about FFI and Rust, see Jake Goulding's
-[Rust FFI Omnibus](http://jakegoulding.com/rust-ffi-omnibus/).
-
- Compiling Tor with Rust enabled
----------------------------------
-
-You will need to run the `configure` script with the `--enable-rust`
-flag to explicitly build with Rust. Additionally, you will need to
-specify where to fetch Rust dependencies, as we allow for either
-fetching dependencies from Cargo or specifying a local directory.
-
-**Fetch dependencies from Cargo**
-
-    ./configure --enable-rust --enable-cargo-online-mode
-
-**Using a local dependency cache**
-
-You'll need the following Rust dependencies (as of this writing):
-
-    libc==0.2.39
-
-We vendor our Rust dependencies in a separate repo using
-[cargo-vendor](https://github.com/alexcrichton/cargo-vendor).  To use
-them, do:
-
-    git submodule init
-    git submodule update
-
-To specify the local directory containing the dependencies, (assuming
-you are in the top level of the repository) configure tor with:
-
-    TOR_RUST_DEPENDENCIES='path_to_dependencies_directory' ./configure --enable-rust
-
-(Note that TOR_RUST_DEPENDENCIES must be the full path to the directory; it
-cannot be relative.)
-
-Assuming you used the above `git submodule` commands and you're in the
-topmost directory of the repository, this would be:
-
-    TOR_RUST_DEPENDENCIES=`pwd`/src/ext/rust/crates ./configure --enable-rust
-
-
- Identifying which modules to rewrite
-======================================
-
-The places in the Tor codebase that are good candidates for porting to
-Rust are:
-
-1. loosely coupled to other Tor submodules,
-2. have high test coverage, and
-3. would benefit from being implemented in a memory safe language.
-
-Help in either identifying places such as this, or working to improve
-existing areas of the C codebase by adding regression tests and
-simplifying dependencies, would be really helpful.
-
-Furthermore, as submodules in C are implemented in Rust, this is a good
-opportunity to refactor, add more tests, and split modules into smaller
-areas of responsibility.
-
-A good first step is to build a module-level callgraph to understand how
-interconnected your target module is.
-
-    git clone https://git.torproject.org/user/nickm/calltool.git
-    cd tor
-    CFLAGS=0 ./configure
-    ../calltool/src/main.py module_callgraph
-
-The output will tell you each module name, along with a set of every module that
-the module calls.  Modules which call fewer other modules are better targets.
-
- Writing your Rust module
-==========================
-
-Strive to change the C API as little as possible.
-
-We are currently targeting Rust nightly, *for now*. We expect this to
-change moving forward, as we understand more about which nightly
-features we need. It is on our TODO list to try to cultivate good
-standing with various distro maintainers of `rustc` and `cargo`, in
-order to ensure that whatever version we solidify on is readily
-available.
-
-If parts of your Rust code needs to stay in sync with C code (such as
-handling enums across the FFI boundary), annonotate these places in a
-comment structured as follows:
-
-  /// C_RUST_COUPLED: <path_to_file> `<name_of_c_object>`
-
-Where <name_of_c_object> can be an enum, struct, constant, etc.  Then,
-do the same in the C code, to note that rust will need to be changed
-when the C does.
-
- Adding your Rust module to Tor's build system
------------------------------------------------
-
-0. Your translation of the C module should live in its own crate(s)
-   in the `.../tor/src/rust/` directory.
-1. Add your crate to `.../tor/src/rust/Cargo.toml`, in the
-   `[workspace.members]` section.
-2. Add your crate's files to src/rust/include.am
-
-If your crate should be available to C (rather than just being included as a
-dependency of other Rust modules):
-0. Declare the crate as a dependency of tor_rust in
-   `src/rust/tor_util/Cargo.toml` and include it in
-   `src/rust/tor_rust/lib.rs`
-
- How to test your Rust code
-----------------------------
-
-Everything should be tested full stop.  Even non-public functionality.
-
-Be sure to edit `.../tor/src/test/test_rust.sh` to add the name of your
-crate to the `crates` variable! This will ensure that `cargo test` is
-run on your crate.
-
-Configure Tor's build system to build with Rust enabled:
-
-    ./configure --enable-fatal-warnings --enable-rust --enable-cargo-online-mode
-
-Tor's test should be run by doing:
-
-    make check
-
-Tor's integration tests should also pass:
-
-    make test-stem
-
- Submitting a patch
-=====================
-
-Please follow the instructions in `.../doc/HACKING/GettingStarted.md`.

doc/HACKING/HelpfulTools.md

@@ -4,22 +4,6 @@ Useful tools
 These aren't strictly necessary for hacking on Tor, but they can help track
 down bugs.
 
-Travis CI
----------
-It's CI. Looks like this: https://travis-ci.org/torproject/tor.
-
-Runs automatically on Pull Requests sent to torproject/tor. You can set it up
-for your fork to build commits outside of PRs too:
-
-1. sign up for GitHub: https://github.com/join
-2. fork https://github.com/torproject/tor:
-   https://help.github.com/articles/fork-a-repo/
-3. follow https://docs.travis-ci.com/user/getting-started/#To-get-started-with-Travis-CI.
-   skip steps involving `.travis.yml` (we already have one).
-
-Builds should show up on the web at travis-ci.com and on IRC at #tor-ci on
-OFTC. If they don't, ask #tor-dev (also on OFTC).
-
 Jenkins
 -------
 
@@ -127,18 +111,14 @@ Running gcov for unit test coverage
 
 (On OSX, you'll need to start with `--enable-coverage CC=clang`.)
 
-If that doesn't work:
-
-   * Try configuring Tor with `--disable-gcc-hardening`
-   * You might need to run `make clean` after you run `./configure`.
-
 Then, look at the .gcov files in `coverage-output`.  '-' before a line means
 that the compiler generated no code for that line.  '######' means that the
 line was never reached.  Lines with numbers were called that number of times.
 
-For more details about how to read gcov output, see the [Invoking
-gcov](https://gcc.gnu.org/onlinedocs/gcc/Invoking-Gcov.html) chapter
-of the GCC manual.
+If that doesn't work:
+
+   * Try configuring Tor with `--disable-gcc-hardening`
+   * You might need to run `make clean` after you run `./configure`.
 
 If you make changes to Tor and want to get another set of coverage results,
 you can run `make reset-gcov` to clear the intermediary gcov output.
@@ -148,13 +128,9 @@ a meaningful diff between them, you can run:
 
     ./scripts/test/cov-diff coverage-output1 coverage-output2 | less
 
-In this diff, any lines that were visited at least once will have coverage "1",
-and line numbers are deleted.  This lets you inspect what you (probably) really
-want to know: which untested lines were changed?  Are there any new untested
-lines?
-
-If you run ./scripts/test/cov-exclude, it marks excluded unreached
-lines with 'x', and excluded reached lines with '!!!'.
+In this diff, any lines that were visited at least once will have coverage
+"1".  This lets you inspect what you (probably) really want to know: which
+untested lines were changed?  Are there any new untested lines?
 
 Running integration tests
 -------------------------
@@ -166,12 +142,6 @@ run `make test-network`.
 We also have scripts to run integration tests using Stem.  To try them, set
 `STEM_SOURCE_DIR` to your Stem source directory, and run `test-stem`.
 
-Profiling Tor
--------------
-
-Ongoing notes about Tor profiling can be found at
-https://pad.riseup.net/p/profiling-tor
-
 Profiling Tor with oprofile
 ---------------------------
 
@@ -198,62 +168,20 @@ Here are some basic instructions
    * `opreport -l that_dir/*`
  - Profit
 
-Profiling Tor with perf
------------------------
-
-This works with a running Tor, and requires root.
-
-1. Decide how long you want to profile for. Start with (say) 30 seconds. If that
-   works, try again with longer times.
-
-2. Find the PID of your running tor process.
-
-3. Run `perf record --call-graph dwarf -p <PID> sleep <SECONDS>`
-
-   (You may need to do this as root.)
-
-   You might need to add `-e cpu-clock` as an option to the perf record line
-   above, if you are on an older CPU without access to hardware profiling
-   events, or in a VM, or something.
-
-4. Now you have a perf.data file. Have a look at it with `perf report
-   --no-children --sort symbol,dso` or `perf report --no-children --sort
-   symbol,dso --stdio --header`. How does it look?
-
-5a. Once you have a nice big perf.data file, you can compress it, encrypt it,
-    and send it to your favorite Tor developers.
-
-5b. Or maybe you'd rather not send a nice big perf.data file. Who knows what's
-    in that!? It's kinda scary. To generate a less scary file, you can use `perf
-    report -g > <FILENAME>.out`. Then you can compress that and put it somewhere
-    public.
-
-Profiling Tor with gperftools aka Google-performance-tools
-----------------------------------------------------------
-
-This should work on nearly any unixy system. It doesn't seem to be compatible
-with RunAsDaemon though.
-
-Beforehand, install google-perftools.
-
-1. You need to rebuild Tor, hack the linking steps to add `-lprofiler` to the
-   libs. You can do this by adding `LIBS=-lprofiler` when you call `./configure`.
-
-Now you can run Tor with profiling enabled, and use the pprof utility to look at
-performance! See the gperftools manual for more info, but basically:
-
-2. Run `env CPUPROFILE=/tmp/profile src/or/tor -f <path/torrc>`. The profile file
-   is not written to until Tor finishes execuction.
-
-3. Run `pprof src/or/tor /tm/profile` to start the REPL.
-
 Generating and analyzing a callgraph
 ------------------------------------
 
-0. Build Tor on linux or mac, ideally with -O0 or -fno-inline.
+1. Run `./scripts/maint/generate_callgraph.sh`.  This will generate a
+   bunch of files in a new ./callgraph directory.
 
-1. Clone 'https://gitweb.torproject.org/user/nickm/calltool.git/' .
-   Follow the README in that repository.
+2. Run `./scripts/maint/analyze_callgraph.py callgraph/src/*/*`.  This
+   will do a lot of graph operations and then dump out a new
+   `callgraph.pkl` file, containing data in Python's 'pickle' format.
+
+3. Run `./scripts/maint/display_callgraph.py`.  It will display:
+    - the number of functions reachable from each function.
+    - all strongly-connnected components in the Tor callgraph
+    - the largest bottlenecks in the largest SCC in the Tor callgraph.
 
 Note that currently the callgraph generator can't detect calls that pass
 through function pointers.

doc/HACKING/HowToReview.md

@@ -15,12 +15,10 @@ Top-level smell-checks
 
 (Difficulty: easy)
 
-- Does it compile with `--enable-fatal-warnings`?
+- Does it compile with `--enable-gcc-warnings`?
 
 - Does `make check-spaces` pass?
 
-- Does `make check-changes` pass?
-
 - Does it have a reasonable amount of tests?  Do they pass?  Do they leak
   memory?
 
@@ -34,7 +32,6 @@ Top-level smell-checks
 
 - If this changes Tor's behavior on the wire, is there a design proposal?
 
-- If this changes anything in the code, is there a "changes" file?
 
 
 Let's look at the code!

doc/HACKING/Module.md

@@ -1,111 +0,0 @@
-# Modules in Tor #
-
-This document describes the build system and coding standards when writing a
-module in Tor.
-
-## What is a module? ##
-
-In the context of the tor code base, a module is a subsystem that we can
-selectively enable or disable, at `configure` time.
-
-Currently, there is only one module:
-
-  - Directory Authority subsystem (dirauth)
-
-It is located in its own directory in `src/or/dirauth/`. To disable it, one
-need to pass `--disable-module-dirauth` at configure time. All modules are
-currently enabled by default.
-
-## Build System ##
-
-The changes to the build system are pretty straightforward.
-
-1. Locate in the `configure.ac` file this define: `m4_define(MODULES`. It
-   contains a list (white-space separated) of the module in tor. Add yours to
-   the list.
-
-2. Use the `AC_ARG_ENABLE([module-dirauth]` template for your new module. We
-   use the "disable module" approach instead of enabling them one by one. So,
-   by default, tor will build all the modules.
-
-   This will define the `HAVE_MODULE_<name>` statement which can be used in
-   the C code to conditionally compile things for your module. And the
-   `BUILD_MODULE_<name>` is also defined for automake files (e.g: include.am).
-
-3. In the `src/or/include.am` file, locate the `MODULE_DIRAUTH_SOURCES` value.
-   You need to create your own `_SOURCES` variable for your module and then
-   conditionally add the it to `LIBTOR_A_SOURCES` if you should build the
-   module.
-
-   It is then **very** important to add your SOURCES variable to
-   `src_or_libtor_testing_a_SOURCES` so the tests can build it.
-
-4. Do the same for header files, locate `ORHEADERS +=` which always add all
-   headers of all modules so the symbol can be found for the module entry
-   points.
-
-Finally, your module will automatically be included in the
-`TOR_MODULES_ALL_ENABLED` variable which is used to build the unit tests. They
-always build everything in order to tests everything.
-
-## Coding ##
-
-As mentioned above, a module must be isolated in its own directory (name of
-the module) in `src/or/`.
-
-There are couples of "rules" you want to follow:
-
-* Minimize as much as you can the number of entry points into your module.
-  Less is always better but of course that doesn't work out for every use
-  case. However, it is a good thing to always keep that in mind.
-
-* Do **not** use the `HAVE_MODULE_<name>` define outside of the module code
-  base. Every entry point should have a second definition if the module is
-  disabled. For instance:
-
-  ```
-  #ifdef HAVE_MODULE_DIRAUTH
-
-  int sr_init(int save_to_disk);
-
-  #else /* HAVE_MODULE_DIRAUTH */
-
-  static inline int
-  sr_init(int save_to_disk)
-  {
-    (void) save_to_disk;
-    return 0;
-  }
-
-  #endif /* HAVE_MODULE_DIRAUTH */
-
-  ```
-
-  The main reason for this approach is to avoid having conditional code
-  everywhere in the code base. It should be centralized as much as possible
-  which helps maintainability but also avoids conditional spaghetti code
-  making the code much more difficult to follow/understand.
-
-* It is possible that you end up with code that needs to be used by the rest
-  of the code base but is still part of your module. As a good example, if you
-  look at `src/or/shared_random_client.c`: it contains code needed by the hidden
-  service subsystem but mainly related to the shared random subsystem very
-  specific to the dirauth module.
-
-  This is fine but try to keep it as lean as possible and never use the same
-  filename as the one in the module. For example, this is a bad idea and
-  should never be done:
-
-    - `src/or/shared_random.c`
-    - `src/or/dirauth/shared_random.c`
-
-* When you include headers from the module, **always** use the full module
-  path in your statement. Example:
-
-  `#include "dirauth/dirvote.h"`
-
-  The main reason is that we do **not** add the module include path by default
-  so it needs to be specified. But also, it helps our human brain understand
-  which part comes from a module or not.
-
-  Even **in** the module itself, use the full include path like above.

doc/HACKING/ReleasingTor.md

@@ -2,69 +2,33 @@
 Putting out a new release
 -------------------------
 
-Here are the steps that the maintainer should take when putting out a
-new Tor release:
-
-=== 0. Preliminaries
-
-1. Get at least three of weasel/arma/Sebastian/Sina to put the new
-   version number in their approved versions list.  Give them a few
-   days to do this if you can.
-
-2. If this is going to be an important security release, give the packagers
-   some advance warning: See this list of packagers in IV.3 below.
-
-3. Given the release date for Tor, ask the TB team about the likely release
-   date of a TB that contains it.  See note below in "commit, upload,
-   announce".
-
-=== I. Make sure it works
+Here are the steps Roger takes when putting out a new Tor release:
 
 1. Use it for a while, as a client, as a relay, as a hidden service,
    and as a directory authority. See if it has any obvious bugs, and
    resolve those.
 
    As applicable, merge the `maint-X` branch into the `release-X` branch.
-   But you've been doing that all along, right?
 
-2. Are all of the jenkins builders happy?  See jenkins.torproject.org.
-
-   What about the bsd buildbots?
-         See http://buildbot.pixelminers.net/builders/
-
-   What about Coverity Scan?
-
-   What about clang scan-build?
-
-   Does 'make distcheck' complain?
-
-   How about 'make test-stem' and 'make test-network' and
-   `make test-network-full`?
-
-       - Are all those tests still happy with --enable-expensive-hardening ?
-
-   Any memory leaks?
-
-
-=== II. Write a changelog
-
-
-1a. (Alpha release variant)
-
-   Gather the `changes/*` files into a changelog entry, rewriting many
+2. Gather the `changes/*` files into a changelog entry, rewriting many
    of them and reordering to focus on what users and funders would find
    interesting and understandable.
 
-   To do this, first run `./scripts/maint/lintChanges.py changes/*` and
-   fix as many warnings as you can.  Then run `./scripts/maint/sortChanges.py
-   changes/* > changelog.in` to combine headings and sort the entries.
-   After that, it's time to hand-edit and fix the issues that lintChanges
-   can't find:
+   1. Make sure that everything that wants a bug number has one.
+      Make sure that everything which is a bugfix says what version
+      it was a bugfix on.
 
-   1. Within each section, sort by "version it's a bugfix on", else by
-      numerical ticket order.
+   2. Concatenate them.
 
-   2. Clean them up:
+   3. Sort them by section. Within each section, sort by "version it's
+      a bugfix on", else by numerical ticket order.
+
+   4. Clean them up:
+
+      Standard idioms:
+      `Fixes bug 9999; bugfix on 0.3.3.3-alpha.`
+
+      One space after a period.
 
       Make stuff very terse
 
@@ -92,32 +56,19 @@ new Tor release:
       maint-0.2.1), be sure to make the stanzas identical (so people can
       distinguish if these are the same change).
 
-   3. Clean everything one last time.
+   5. Merge them in.
 
-   4. Run `./scripts/maint/format_changelog.py --inplace` to make it prettier
+   6. Clean everything one last time.
 
-1b. (old-stable release variant)
+   7. Run `./scripts/maint/format_changelog.py` to make it prettier.
 
-   For stable releases that backport things from later, we try to compose
-   their releases, we try to make sure that we keep the changelog entries
-   identical to their original versions, with a 'backport from 0.x.y.z'
-   note added to each section.  So in this case, once you have the items
-   from the changes files copied together, don't use them to build a new
-   changelog: instead, look up the corrected versions that were merged
-   into ChangeLog in the master branch, and use those.
-
-2. Compose a short release blurb to highlight the user-facing
+3. Compose a short release blurb to highlight the user-facing
    changes. Insert said release blurb into the ChangeLog stanza. If it's
    a stable release, add it to the ReleaseNotes file too. If we're adding
-   to a release-* branch, manually commit the changelogs to the later
+   to a release-0.2.x branch, manually commit the changelogs to the later
    git branches too.
 
-3. If there are changes that require or suggest operator intervention
-   before or during the update, mail operators (either dirauth or relays
-   list) with a headline that indicates that an action is required or
-   appreciated.
-
-4. If you're doing the first stable release in a series, you need to
+   If you're doing the first stable release in a series, you need to
    create a ReleaseNotes for the series as a whole.  To get started
    there, copy all of the Changelog entries from the series into a new
    file, and run `./scripts/maint/sortChanges.py` on it.  That will
@@ -127,96 +78,66 @@ new Tor release:
    to start sorting and condensing entries.  (Generally, we don't edit the
    text of existing entries, though.)
 
-
-=== III. Making the source release.
-
-1. In `maint-0.?.x`, bump the version number in `configure.ac` and run
-   `perl scripts/maint/updateVersions.pl` to update version numbers in other
-   places, and commit.  Then merge `maint-0.?.x` into `release-0.?.x`.
+4. In `maint-0.2.x`, bump the version number in `configure.ac` and run
+   `scripts/maint/updateVersions.pl` to update version numbers in other
+   places, and commit.  Then merge `maint-0.2.x` into `release-0.2.x`.
 
    (NOTE: To bump the version number, edit `configure.ac`, and then run
    either `make`, or `perl scripts/maint/updateVersions.pl`, depending on
    your version.)
 
-   When you merge the maint branch forward to the next maint branch, or into
-   master, merge it with "-s ours" to avoid a needless version bump.
+5. Make distcheck, put the tarball up somewhere, and tell `#tor` about
+   it. Wait a while to see if anybody has problems building it. Try to
+   get Sebastian or somebody to try building it on Windows.
 
-2. Make distcheck, put the tarball up in somewhere (how about your
-   homedir on your homedir on people.torproject.org?) , and tell `#tor`
-   about it. Wait a while to see if anybody has problems building it.
-   (Though jenkins is usually pretty good about catching these things.)
+6. Get at least two of weasel/arma/Sebastian to put the new version number
+   in their approved versions list.
 
-=== IV. Commit, upload, announce
-
-1. Sign the tarball, then sign and push the git tag:
+7. Sign the tarball, then sign and push the git tag:
 
         gpg -ba <the_tarball>
-        git tag -u <keyid> tor-0.3.x.y-status
-        git push origin tag tor-0.3.x.y-status
+        git tag -u <keyid> tor-0.2.x.y-status
+        git push origin tag tor-0.2.x.y-status
 
-   (You must do this before you update the website: it relies on finding
-   the version by tag.)
-
-2. scp the tarball and its sig to the dist website, i.e.
+8. scp the tarball and its sig to the dist website, i.e.
    `/srv/dist-master.torproject.org/htdocs/` on dist-master. When you want
    it to go live, you run "static-update-component dist.torproject.org"
    on dist-master.
 
-   In the webwml.git repository, `include/versions.wmi` and `Makefile`
-   to note the new version.
+   Edit `include/versions.wmi` and `Makefile` to note the new version.
 
    (NOTE: Due to #17805, there can only be one stable version listed at
    once.  Nonetheless, do not call your version "alpha" if it is stable,
    or people will get confused.)
 
-3. Email the packagers (cc'ing tor-team) that a new tarball is up.
+9. Email the packagers (cc'ing tor-assistants) that a new tarball is up.
    The current list of packagers is:
 
        - {weasel,gk,mikeperry} at torproject dot org
        - {blueness} at gentoo dot org
        - {paul} at invizbox dot io
-       - {vincent} at invizbox dot com
+       - {ondrej.mikle} at gmail dot com
        - {lfleischer} at archlinux dot org
-       - {Nathan} at freitas dot net
-       - {mike} at tig dot as
-       - {tails-rm} at boum dot org
-       - {simon} at sdeziel.info
-       - {yuri} at freebsd.org
-       - {mh+tor} at scrit.ch
+       - {tails-dev} at boum dot org
 
-   Also, email tor-packagers@lists.torproject.org.
-
-4. Add the version number to Trac.  To do this, go to Trac, log in,
+10. Add the version number to Trac.  To do this, go to Trac, log in,
     select "Admin" near the top of the screen, then select "Versions" from
     the menu on the left.  At the right, there will be an "Add version"
     box.  By convention, we enter the version in the form "Tor:
     0.2.2.23-alpha" (or whatever the version is), and we select the date as
     the date in the ChangeLog.
 
-5. Double-check: did the version get recommended in the consensus yet?  Is
-   the website updated?  If not, don't announce until they have the
-   up-to-date versions, or people will get confused.
+11. Forward-port the ChangeLog (and ReleaseNotes if appropriate).
 
-6. Mail the release blurb and ChangeLog to tor-talk (development release) or
-   tor-announce (stable).
+12. Wait up to a day or two (for a development release), or until most
+    packages are up (for a stable release), and mail the release blurb and
+    changelog to tor-talk or tor-announce.
 
-   Post the changelog on the blog as well. You can generate a
-   blog-formatted version of the changelog with the -B option to
-   format-changelog.
-
-   When you post, include an estimate of when the next TorBrowser
-   releases will come out that include this Tor release.  This will
-   usually track https://wiki.mozilla.org/RapidRelease/Calendar , but it
-   can vary.
-
-
-=== V. Aftermath and cleanup
-
-1. If it's a stable release, bump the version number in the
-    `maint-x.y.z` branch to "newversion-dev", and do a `merge -s ours`
-    merge to avoid taking that change into master.
-
-2. Forward-port the ChangeLog (and ReleaseNotes if appropriate).
-
-3. Keep an eye on the blog post, to moderate comments and answer questions.
+   (We might be moving to faster announcements, but don't announce until
+   the website is at least updated.)
 
+13. If it's a stable release, bump the version number in the `maint-x.y.z`
+    branch to "newversion-dev", and do a `merge -s ours` merge to avoid
+    taking that change into master.  Do a similar `merge -s theirs`
+    merge to get the change (and only that change) into release.  (Some
+    of the build scripts require that maint merge cleanly into release.)

doc/HACKING/Tracing.md

@@ -1,91 +0,0 @@
-# Tracing #
-
-This document describes how the event tracing subsystem works in tor so
-developers can add events to the code base but also hook them to an event
-tracing framework.
-
-## Basics ###
-
-Event tracing is separated in two concepts, trace events and a tracer. The
-tracing subsystem can be found in `src/trace`. The `events.h` header file is
-the main file that maps the different tracers to trace events.
-
-### Events ###
-
-A trace event is basically a function from which we can pass any data that
-we want to collect. In addition, we specify a context for the event such as
-a subsystem and an event name.
-
-A trace event in tor has the following standard format:
-
-	tor_trace(subsystem, event\_name, args...)
-
-The `subsystem` parameter is the name of the subsytem the trace event is in.
-For example that could be "scheduler" or "vote" or "hs". The idea is to add
-some context to the event so when we collect them we know where it's coming
-from. The `event_name` is the name of the event which helps a lot with
-adding some semantic to the event. Finally, `args` is any number of
-arguments we want to collect.
-
-Here is an example of a possible tracepoint in main():
-
-	tor_trace(main, init_phase, argc)
-
-The above is a tracepoint in the `main` subsystem with `init_phase` as the
-event name and the `int argc` is passed to the event as well.
-
-How `argc` is collected or used has nothing to do with the instrumentation
-(adding trace events to the code). It is the work of the tracer so this is why
-the trace events and collection framework (tracer) are decoupled. You _can_
-have trace events without a tracer.
-
-### Tracer ###
-
-In `src/trace/events.h`, we map the `tor_trace()` function to the right
-tracer. A tracer support is only enabled at compile time. For instance, the
-file `src/trace/debug.h` contains the mapping of the generic tracing function
-`tor_trace()` to the `log_debug()` function. More specialized function can be
-mapped depending on the tracepoint.
-
-## Build System ##
-
-This section describes how it is integrated into the build system of tor.
-
-By default, every tracing events are disabled in tor that is `tor_trace()`
-is a NOP.
-
-To enable a tracer, there is a configure option on the form of:
-
-	--enable-tracing-<tracer>
-
-We have an option that will send every trace events to a `log_debug()` (as
-mentionned above) which will print you the subsystem and name of the event but
-not the arguments for technical reasons. This is useful if you want to quickly
-see if your trace event is being hit or well written. To do so, use this
-configure option:
-
-	--enable-tracing-debug
-
-## Instrument Tor ##
-
-This is pretty easy. Let's say you want to add a trace event in
-`src/or/rendcache.c`, you only have to add this include statement:
-
-	#include "trace/events.h"
-
-Once done, you can add as many as you want `tor_trace()` that you need.
-Please use the right subsystem (here it would be `hs`) and a unique name that
-tells what the event is for. For example:
-
-	tor_trace(hs, store_desc_as_client, desc, desc_id);
-
-If you look in `src/trace/events.h`, you'll see that if tracing is enabled it
-will be mapped to a function called:
-
-	tor_trace_hs_store_desc_as_client(desc, desc_id)
-
-And the point of all this is for that function to be defined in a new file
-that you might want to add named `src/trace/hs.{c|h}` which would defined how
-to collect the data for the `tor_trace_hs_store_desc_as_client()` function
-like for instance sending it to a `log_debug()` or do more complex operations
-or use a userspace tracer like LTTng (https://lttng.org).

doc/HACKING/WritingTests.md

@@ -48,7 +48,7 @@ isolation, you just run `./src/test/test-memwipe`.
 To run tests within the unit test programs, you can specify the name
 of the test.  The string ".." can be used as a wildcard at the end of the
 test name.  For example, to run all the cell format tests, enter
-`./src/test/test cellfmt/..`.
+`./src/test/test cellfmt/..`.  To run
 
 Many tests that need to mess with global state run in forked subprocesses in
 order to keep from contaminating one another.  But when debugging a failing test,
@@ -91,9 +91,6 @@ coverage percentage.
 For a summary of the test coverage for each _function_, run
 `./scripts/test/cov-display -f ${TMPDIR}/*`.
 
-For more details on using gcov, including the helper scripts in
-scripts/test, see HelpfulTools.md.
-
 ### Comparing test coverage
 
 Sometimes it's useful to compare test coverage for a branch you're writing to
@@ -112,20 +109,6 @@ To count new or modified uncovered lines in D2, you can run:
 
     ./scripts/test/cov-diff ${D1} ${D2}" | grep '^+ *\#' | wc -l
 
-### Marking lines as unreachable by tests
-
-You can mark a specific line as unreachable by using the special
-string LCOV_EXCL_LINE.  You can mark a range of lines as unreachable
-with LCOV_EXCL_START... LCOV_EXCL_STOP.  Note that older versions of
-lcov don't understand these lines.
-
-You can post-process .gcov files to make these lines 'unreached' by
-running ./scripts/test/cov-exclude on them.  It marks excluded
-unreached lines with 'x', and excluded reached lines with '!!!'.
-
-Note: you should never do this unless the line is meant to 100%
-unreachable by actual code.
-
 
 What kinds of test should I write?
 ----------------------------------
@@ -156,11 +139,6 @@ or create a new C file there.  Each test is a single function that must
 be indexed in the table at the end of the file.  We use the label "done:" as
 a cleanup point for all test functions.
 
-If you have created a new test file, you will need to:
-1. Add the new test file to include.am
-2. In `test.h`, include the new test cases (testcase_t)
-3. In `test.c`, add the new test cases to testgroup_t testgroups
-
 (Make sure you read `tinytest-manual.md` before proceeding.)
 
 I use the term "unit test" and "regression tests" very sloppily here.
@@ -439,50 +417,18 @@ makefile exports them.
 Writing integration tests with Stem
 -----------------------------------
 
-The 'stem' library includes extensive tests for the Tor controller protocol.
+The 'stem' library includes extensive unit tests for the Tor controller
+protocol.
+
+For more information on writing new tests for stem, have a look around
+the `test/*` directory in stem, and find a good example to emulate.  You
+might want to start with
+`https://gitweb.torproject.org/stem.git/tree/test/integ/control/controller.py`
+to improve Tor's test coverage.
+
 You can run stem tests from tor with `make test-stem`, or see
 `https://stem.torproject.org/faq.html#how-do-i-run-the-tests`.
 
-To see what tests are available, have a look around the `test/*` directory in
-stem. The first thing you'll notice is that there are both `unit` and `integ`
-tests. The former are for tests of the facilities provided by stem itself that
-can be tested on their own, without the need to hook up a tor process. These
-are less relevant, unless you want to develop a new stem feature. The latter,
-however, are a very useful tool to write tests for controller features. They
-provide a default environment with a connected tor instance that can be
-modified and queried. Adding more integration tests is a great way to increase
-the test coverage inside Tor, especially for controller features.
-
-Let's assume you actually want to write a test for a previously untested
-controller feature. I'm picking the `exit-policy/*` GETINFO queries. Since
-these are a controller feature that we want to write an integration test for,
-the right file to modify is
-`https://gitweb.torproject.org/stem.git/tree/test/integ/control/controller.py`.
-
-First off we notice that there is an integration test called
-`test_get_exit_policy()` that's already written. This exercises the interaction
-of stem's `Controller.get_exit_policy()` method, and is not relevant for our
-test since there are no stem methods to make use of all `exit-policy/*`
-queries (if there were, likely they'd be tested already. Maybe you want to
-write a stem feature, but I chose to just add tests).
-
-Our test requires a tor controller connection, so we'll use the
-`@require_controller` annotation for our `test_exit_policy()` method. We need a
-controller instance, which we get from
-`test.runner.get_runner().get_tor_controller()`. The attached Tor instance is
-configured as a client, but the exit-policy GETINFO queries need a relay to
-work, so we have to change the config (using `controller.set_options()`). This
-is OK for us to do, we just have to remember to set DisableNetwork so we don't
-actually start an exit relay and also to undo the changes we made (by calling
-`controller.reset_conf()` at the end of our test). Additionally, we have to
-configure a static Address for Tor to use, because it refuses to build a
-descriptor when it can't guess a suitable IP address. Unfortunately, these
-kinds of tripwires are everywhere. Don't forget to file appropriate tickets if
-you notice any strange behaviour that seems totally unreasonable.
-
-Check out the `test_exit_policy()` function in abovementioned file to see the
-final implementation for this test.
-
 System testing with Chutney
 ---------------------------
 

doc/HACKING/android/Simpleperf.md

@@ -1,98 +0,0 @@
-# Using `simpleperf` to collect CPU profiling on Android
-
-This document describes how you can use Android's `simpleperf`
-command-line tool to get CPU profiling information from Tor via the
-Orbot application. The tool is particularly useful for Tor development
-because it is able to profile native applications on the platform
-whereas a lot of the normal tooling for the Android platform is only
-able to collect information from Java-based applications.
-
-## Prerequisites
-
-Before using `simpleperf` there is a couple of steps that must be
-followed. You should make sure you have both a recent installation of
-the Android Software Development Kit (SDK) and Native Development Kit
-(NDK) installed. These can be found on the Android Developers website.
-
-1. Follow the build instructions from the `BUILD` file in the Orbot
-   repository and build an Orbot APK (Android Package) file with
-   debugging enabled. Make sure that when you build the native content of
-   the Orbot application that you run the `make -C external` command with
-   an additional `DEBUG=1` as parameter to ensure that the Orbot build
-   process does not strip the debug symbols from the Tor binary.
-
-2. (Optional) Uninstall and clean-up your old Orbot installation that
-   is most likely downloaded from Google's Play Store or via fdroid:
-
-       $ adb shell pm clear org.torproject.android
-       $ adb uninstall org.torproject.android
-
-3. Install the Android Package you generated in step 1:
-
-       $ adb install /path/to/your/app-fullperm-debug.apk
-
-4. Check on your device that the newly installed Orbot actually works
-   and behaves in the way you expect it to.
-
-## Profiling using `simpleperf`
-
-The `simpleperf` tool can be found in the `simpleperf/` directory in
-the directory where you installed the Android NDK to. In this
-directory there is a set of Python files that will help you deploy the
-tool to a device and collect the measurement data such that you can
-analyze the results on your computer rather than on your phone.
-
-1. Change directory to the location of the `simpleperf` directory.
-2. Open the `app_profiler.config` file and change
-   `app_package_name` to `org.torproject.android`, `apk_file_path` to
-   the path of your Orbot Android Package (APK file).
-3. Optionally change the duration parameter in the `record_options`
-   variable in `app_profiler.config` to the duration which you would like
-   to collect samples in. The value is specified in seconds.
-4. Run the app profiler using `python app_profiler.py`. This helper
-   script will push the `simpleperf` tool to your device, start the
-   profiler, and once it has completed copy the generated `perf.data`
-   file over to your computer with the results.
-
-### Analyzing the results
-
-You can inspect your resulting `perf.data` file via a simple GUI
-program `python report.py` or via the command-line tool `simpleperf
-report`. I've found the GUI tool to be easier to navigate around with
-than the command-line tool.
-
-The `-g` option can be passed to the command line `simpleperf report`
-tool allows you to see the call graph of functions and how much time
-was spend on the call.
-
-## Tips & Tricks
-
-- When you have installed Orbot the first time, you will notice that
-  if you get a shell on the Android device that there is no Tor binary
-  available. This is because Orbot unpacks the Tor binary first time it
-  is executed and places it under the `app_bin/` directory on the
-  device.
-
-  To access binaries, `torrc` files, and other useful information on
-  the device do the following:
-
-      $ adb shell
-      (device):/ $ run-as org.torproject.android
-      (device):/data/data/org.torproject.android $ ls
-      app_bin app_data cache databases files lib shared_prefs
-
-  Descriptors, control authentication cookie, state, and other files can be
-  found in the `app_data` directory. The `torrc` can be found in the `app_bin/`
-  directory.
-
-- You can enable logging in Tor via the syslog (or android) log
-  mechanism with:
-
-      $ adb shell
-      (device):/ $ run-as org.torproject.android
-      (device):/data/data/org.torproject.android $ echo -e "\nLog info syslog" >> app_bin/torrc
-
-  Start Tor the normal way via Orbot and collect the logs from your computer using
-
-      $ adb logcat
-

doc/include.am

@@ -12,11 +12,17 @@
 # part of the source distribution, so that people without asciidoc can
 # just use the .1 and .html files.
 
-all_mans = doc/tor doc/tor-gencert doc/tor-resolve doc/torify
+base_mans = doc/tor doc/tor-gencert doc/tor-resolve doc/torify
+all_mans = $(base_mans)
+if USE_FW_HELPER
+install_mans = $(all_mans)
+else
+install_mans = $(base_mans)
+endif
 
 if USE_ASCIIDOC
-nodist_man1_MANS = $(all_mans:=.1)
-doc_DATA = $(all_mans:=.html)
+nodist_man1_MANS = $(install_mans:=.1)
+doc_DATA = $(install_mans:=.html)
 html_in = $(all_mans:=.html.in)
 man_in = $(all_mans:=.1.in)
 txt_in = $(all_mans:=.1.txt)
@@ -35,15 +41,10 @@ EXTRA_DIST+= doc/asciidoc-helper.sh			\
 	     doc/TUNING						\
 	     doc/HACKING/README.1st.md				\
 	     doc/HACKING/CodingStandards.md 			\
-	     doc/HACKING/CodingStandardsRust.md			\
-	     doc/HACKING/Fuzzing.md				\
 	     doc/HACKING/GettingStarted.md 			\
-	     doc/HACKING/GettingStartedRust.md 			\
 	     doc/HACKING/HelpfulTools.md 			\
 	     doc/HACKING/HowToReview.md  			\
-	     doc/HACKING/Module.md				\
 	     doc/HACKING/ReleasingTor.md                        \
-	     doc/HACKING/Tracing.md				\
 	     doc/HACKING/WritingTests.md
 
 docdir = @docdir@

doc/tor-gencert.1.txt

@@ -68,7 +68,7 @@ OPTIONS
     Number of months that the certificate should be valid. Default: 12.
 
 **--passphrase-fd** __FILEDES__::
-    Filedescriptor to read the passphrase from. Ends at the first NUL or
+    Filedescriptor to read the file descriptor from. Ends at the first NUL or
     newline. Default: read from the terminal.
 
 **-a** __address__:__port__::

doc/torify.1.txt

@@ -17,23 +17,25 @@ SYNOPSIS
 
 DESCRIPTION
 -----------
-**torify** is a simple wrapper that calls torsocks with a tor-specific
-configuration file.
+**torify** is a simple wrapper that attempts to find the best underlying Tor
+wrapper available on a system. It calls torsocks with a tor specific
+configuration file. +
 
-It is provided for backward compatibility; instead you should use torsocks.
+torsocks is an improved wrapper that explicitly rejects UDP, safely resolves DNS
+lookups and properly socksifies your TCP connections. +
+
+Please note that since both method use LD_PRELOAD, torify cannot be applied to
+suid binaries.
 
 WARNING
 -------
-When used with torsocks, torify should not leak DNS requests or UDP data.
+When used with torsocks, torify should not leak DNS requests or UDP data. +
 
-torify can leak ICMP data.
-
-torify will not ensure that different requests are processed on
-different circuits.
+Both will leak ICMP data.
 
 SEE ALSO
 --------
-**tor**(1), **torsocks**(1)
+**tor**(1), **tor-resolve**(1), **torsocks**(1)
 
 AUTHORS
 -------

doc/torrc_format.txt

@@ -18,10 +18,9 @@ does, not what it should do.
    ; specified in RFC5234.
 
    ; A file is interpreted as every Entry in the file, in order.
-   TorrcFile = *Line [ UnterminatedLine ]
+   TorrcFile = *Line
 
-   Line = BlankLine LF / Entry LF
-   UnterminatedLine = BlankLine / Entry
+   Line = BlankLine / Entry
 
    BlankLine =  *WSP OptComment LF
    BlankLine =/ *WSP LF
@@ -70,12 +69,6 @@ does, not what it should do.
    ; Anything besides NUL and LF
    NonLF = %x01-%x09 / %x0b - %xff
 
-   ; Note that on windows, we open our configuration files in "text" mode,
-   ; which causes CRLF pairs to be interpreted as LF.  So, on windows:
-   ;         LF = [ %x0d ] %x0a
-   ; but everywhere else,
-   LF = %0x0a
-
    OCTDIG = '0' - '7'
 
    KC = Any character except an isspace() character or '#' or NUL
@@ -182,7 +175,7 @@ and\
 friends
 
 # Backslashes in the middle of a line are included as-is.  The key of
-# this one is "Too" and the value is "Many\\Backsl\ashes \here" (with
+# this one is "Too" and the value is "Many\\Backsl\ashes here" (with
 # backslashes in that last string as-is)
 Too \
 Many\\\
@@ -192,7 +185,7 @@ here
 # And here's the really yucky part. If a comment appears in a multi-line
 # entry, the entry is still able to continue on the next line, as in the
 # following, where the key is "This" and the value is
-# "entry        and some        are  silly"
+# "entry   and some        are  silly"
 This entry      \
  # has comments \
  and some       \

m4/ax_check_sign.m4

@@ -41,8 +41,8 @@ AU_ALIAS([VL_CHECK_SIGN], [AX_CHECK_SIGN])
 AC_DEFUN([AX_CHECK_SIGN], [
  typename=`echo $1 | sed "s/@<:@^a-zA-Z0-9_@:>@/_/g"`
  AC_CACHE_CHECK([whether $1 is signed], ax_cv_decl_${typename}_signed, [
-   AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$4]],
-     [[ int foo @<:@ 1 - 2 * !((($1) -1) < 0) @:>@ ]])],
+   AC_TRY_COMPILE([$4],
+     [ int foo @<:@ 1 - 2 * !((($1) -1) < 0) @:>@ ],
      [ eval "ax_cv_decl_${typename}_signed=\"yes\"" ],
      [ eval "ax_cv_decl_${typename}_signed=\"no\"" ])])
  symbolname=`echo $1 | sed "s/@<:@^a-zA-Z0-9_@:>@/_/g" | tr "a-z" "A-Z"`

m4/pc_from_ucontext.m4

@@ -79,29 +79,32 @@ AC_DEFUN([AC_PC_FROM_UCONTEXT],
      if ! $pc_field_found; then
        # Prefer sys/ucontext.h to ucontext.h, for OS X's sake.
        if test "x$ac_cv_header_cygwin_signal_h" = xyes; then
-         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <cygwin/signal.h>]],
-                        [[ucontext_t u; return u.$pc_field == 0;]])],
+         AC_TRY_COMPILE([#define _GNU_SOURCE 1
+                         #include <cygwin/signal.h>],
+                        [ucontext_t u; return u.$pc_field == 0;],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
                         pc_field_found=true)
        elif test "x$ac_cv_header_sys_ucontext_h" = xyes; then
-         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/ucontext.h>]],
-                        [[ucontext_t u; return u.$pc_field == 0;]])],
+         AC_TRY_COMPILE([#define _GNU_SOURCE 1
+                         #include <sys/ucontext.h>],
+                        [ucontext_t u; return u.$pc_field == 0;],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
                         pc_field_found=true)
        elif test "x$ac_cv_header_ucontext_h" = xyes; then
-         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <ucontext.h>]],
-                        [[ucontext_t u; return u.$pc_field == 0;]])],
+         AC_TRY_COMPILE([#define _GNU_SOURCE 1
+                         #include <ucontext.h>],
+                        [ucontext_t u; return u.$pc_field == 0;],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
                         pc_field_found=true)
        else     # hope some standard header gives it to us
-         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]],
-                        [[ucontext_t u; return u.$pc_field == 0;]])],
+         AC_TRY_COMPILE([],
+                        [ucontext_t u; return u.$pc_field == 0;],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
@@ -114,8 +117,8 @@ AC_DEFUN([AC_PC_FROM_UCONTEXT],
      pc_fields="$pc_fields sc_rip"  # OpenBSD (x86_64)
      for pc_field in $pc_fields; do
        if ! $pc_field_found; then
-         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <signal.h>]],
-                        [[ucontext_t u; return u.$pc_field == 0;]])],
+         AC_TRY_COMPILE([#include <signal.h>],
+                        [ucontext_t u; return u.$pc_field == 0;],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])

scripts/coccinelle/ceil_div.cocci

@@ -1,6 +0,0 @@
-@@
-expression n, d;
-@@
-
-- (((n) + (d) - 1) / (d))
-+ CEIL_DIV(n, d)

scripts/coccinelle/test-operator-cleanup

@@ -1,11 +0,0 @@
-#!/usr/bin/perl -w -p -i
-
-next if m#^ */\*# or m#^ *\* #;
-
-s/<([,)])/OP_LT$1/;
-s/(?<=[\s,])>([,)])/OP_GT$1/;
-#s/>([,)])/OP_GT$1/;
-s/==([,)])/OP_EQ$1/;
-s/>=([,)])/OP_GE$1/;
-s/<=([,)])/OP_LE$1/;
-s/!=([,)])/OP_NE$1/;

scripts/coccinelle/test_assert_int.cocci

@@ -1,49 +0,0 @@
-@@
-int e;
-constant c;
-@@
-
-(
-- tt_assert(e == c)
-+ tt_int_op(e, OP_EQ, c)
-|
-- tt_assert(e != c)
-+ tt_int_op(e, OP_NE, c)
-|
-- tt_assert(e < c)
-+ tt_int_op(e, OP_LT, c)
-|
-- tt_assert(e <= c)
-+ tt_int_op(e, OP_LE, c)
-|
-- tt_assert(e > c)
-+ tt_int_op(e, OP_GT, c)
-|
-- tt_assert(e >= c)
-+ tt_int_op(e, OP_GE, c)
-)
-
-@@
-unsigned int e;
-constant c;
-@@
-
-(
-- tt_assert(e == c)
-+ tt_uint_op(e, OP_EQ, c)
-|
-- tt_assert(e != c)
-+ tt_uint_op(e, OP_NE, c)
-|
-- tt_assert(e < c)
-+ tt_uint_op(e, OP_LT, c)
-|
-- tt_assert(e <= c)
-+ tt_uint_op(e, OP_LE, c)
-|
-- tt_assert(e > c)
-+ tt_uint_op(e, OP_GT, c)
-|
-- tt_assert(e >= c)
-+ tt_uint_op(e, OP_GE, c)
-)

scripts/coccinelle/test_assert_null.cocci

@@ -1,11 +0,0 @@
-@@
-expression * e;
-@@
-
-(
-- tt_assert(e != NULL)
-+ tt_ptr_op(e, OP_NE, NULL)
-|
-- tt_assert(e == NULL)
-+ tt_ptr_op(e, OP_EQ, NULL)
-)

scripts/coccinelle/test_assert_zero.cocci

@@ -1,5 +0,0 @@
-@@
-@@
-
-- tt_assert(0)
-+ tt_abort()

scripts/codegen/fuzzing_include_am.py

@@ -1,160 +0,0 @@
-#!/usr/bin/python
-
-FUZZERS = """
-	consensus
-	descriptor
-	diff
-	diff-apply
-	extrainfo
-	hsdescv2
-	hsdescv3
-	http
-	http-connect
-	iptsv2
-	microdesc
-	vrs
-"""
-
-
-PREAMBLE = r"""
-FUZZING_CPPFLAGS = \
-	$(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS)
-FUZZING_CFLAGS = \
-	$(AM_CFLAGS) $(TEST_CFLAGS)
-FUZZING_LDFLAG = \
-	@TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@
-FUZZING_LIBS = \
-	src/or/libtor-testing.a \
-	src/common/libor-crypto-testing.a \
-	$(LIBKECCAK_TINY) \
-	$(LIBDONNA) \
-	src/common/libor-testing.a \
-	src/common/libor-ctime-testing.a \
-	src/common/libor-event-testing.a \
-	src/trunnel/libor-trunnel-testing.a \
-	$(rust_ldadd) \
-	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
-	@TOR_LIBEVENT_LIBS@ \
-	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
-	@CURVE25519_LIBS@ \
-	@TOR_SYSTEMD_LIBS@ \
-	@TOR_LZMA_LIBS@ \
-	@TOR_ZSTD_LIBS@
-
-oss-fuzz-prereqs: \
-	src/or/libtor-testing.a \
-	src/common/libor-crypto-testing.a \
-	$(LIBKECCAK_TINY) \
-	$(LIBDONNA) \
-	src/common/libor-testing.a \
-	src/common/libor-ctime-testing.a \
-	src/common/libor-event-testing.a \
-	src/trunnel/libor-trunnel-testing.a
-
-noinst_HEADERS += \
-	src/test/fuzz/fuzzing.h
-
-LIBFUZZER = -lFuzzer
-LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ
-LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS)
-LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG)
-LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++
-
-LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ
-LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS)
-"""
-
-POSTAMBLE = r"""
-noinst_PROGRAMS += $(FUZZERS) $(LIBFUZZER_FUZZERS)
-noinst_LIBRARIES += $(OSS_FUZZ_FUZZERS)
-oss-fuzz-fuzzers:  oss-fuzz-prereqs $(OSS_FUZZ_FUZZERS)
-fuzzers: $(FUZZERS) $(LIBFUZZER_FUZZERS)
-
-test-fuzz-corpora: $(FUZZERS)
-	$(top_srcdir)/src/test/fuzz_static_testcases.sh
-"""
-
-########### No user serviceable parts will follow.
-
-PREAMBLE = PREAMBLE.strip()
-POSTAMBLE = POSTAMBLE.strip()  # If I use it, it's a word!
-FUZZERS = FUZZERS.split()
-FUZZERS.sort()
-
-WARNING = """
-# This file was generated by fuzzing_include_am.py; do not hand-edit unless
-# you enjoy having your changes erased.
-""".strip()
-
-print(WARNING)
-
-print(PREAMBLE)
-
-print("\n# ===== AFL fuzzers")
-
-def get_id_name(s):
-    return s.replace("-", "_")
-
-for fuzzer in FUZZERS:
-    idname = get_id_name(fuzzer)
-    print("""\
-src_test_fuzz_fuzz_{name}_SOURCES = \\
-	src/test/fuzz/fuzzing_common.c \\
-	src/test/fuzz/fuzz_{name}.c
-src_test_fuzz_fuzz_{name}_CPPFLAGS = $(FUZZING_CPPFLAGS)
-src_test_fuzz_fuzz_{name}_CFLAGS = $(FUZZING_CFLAGS)
-src_test_fuzz_fuzz_{name}_LDFLAGS = $(FUZZING_LDFLAG)
-src_test_fuzz_fuzz_{name}_LDADD = $(FUZZING_LIBS)
-""".format(name=idname))
-
-print("FUZZERS = \\")
-print(" \\\n".join("\tsrc/test/fuzz/fuzz-{name}".format(name=fuzzer)
-                   for fuzzer in FUZZERS))
-
-print("\n# ===== libfuzzer")
-print("\nif LIBFUZZER_ENABLED")
-
-for fuzzer in FUZZERS:
-    idname = get_id_name(fuzzer)
-    print("""\
-src_test_fuzz_lf_fuzz_{name}_SOURCES = \\
-	$(src_test_fuzz_fuzz_{name}_SOURCES)
-src_test_fuzz_lf_fuzz_{name}_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
-src_test_fuzz_lf_fuzz_{name}_CFLAGS = $(LIBFUZZER_CFLAGS)
-src_test_fuzz_lf_fuzz_{name}_LDFLAGS = $(LIBFUZZER_LDFLAG)
-src_test_fuzz_lf_fuzz_{name}_LDADD = $(LIBFUZZER_LIBS)
-""".format(name=idname))
-
-print("LIBFUZZER_FUZZERS = \\")
-print(" \\\n".join("\tsrc/test/fuzz/lf-fuzz-{name}".format(name=fuzzer)
-                   for fuzzer in FUZZERS))
-
-print("""
-else
-LIBFUZZER_FUZZERS =
-endif""")
-
-print("\n# ===== oss-fuzz\n")
-print("if OSS_FUZZ_ENABLED")
-
-for fuzzer in FUZZERS:
-    idname = get_id_name(fuzzer)
-    print("""\
-src_test_fuzz_liboss_fuzz_{name}_a_SOURCES = \\
-	$(src_test_fuzz_fuzz_{name}_SOURCES)
-src_test_fuzz_liboss_fuzz_{name}_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
-src_test_fuzz_liboss_fuzz_{name}_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
-""".format(name=idname))
-
-print("OSS_FUZZ_FUZZERS = \\")
-print(" \\\n".join("\tsrc/test/fuzz/liboss-fuzz-{name}.a".format(name=fuzzer)
-                   for fuzzer in FUZZERS))
-
-print("""
-else
-OSS_FUZZ_FUZZERS =
-endif""")
-
-print("")
-
-print(POSTAMBLE)

scripts/codegen/gen_server_ciphers.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2014-2017, The Tor Project, Inc
+# Copyright 2014-2015, The Tor Project, Inc
 # See LICENSE for licensing information
 
 # This script parses openssl headers to find ciphersuite names, determines
@@ -13,13 +13,13 @@ import sys
 
 EPHEMERAL_INDICATORS = [ "_EDH_", "_DHE_", "_ECDHE_" ]
 BAD_STUFF = [ "_DES_40_", "MD5", "_RC4_", "_DES_64_",
-              "_SEED_", "_CAMELLIA_", "_NULL",
-              "_CCM_8", "_DES_", ]
+              "_SEED_", "_CAMELLIA_", "_NULL" ]
 
 # these never get #ifdeffed.
 MANDATORY = [
     "TLS1_TXT_DHE_RSA_WITH_AES_256_SHA",
     "TLS1_TXT_DHE_RSA_WITH_AES_128_SHA",
+    "SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA",
 ]
 
 def find_ciphers(filename):
@@ -48,23 +48,15 @@ def usable_cipher(ciph):
 # All fields we sort on, in order of priority.
 FIELDS = [ 'cipher', 'fwsec', 'mode',  'digest', 'bitlength' ]
 # Map from sorted fields to recognized value in descending order of goodness
-FIELD_VALS = { 'cipher' : [ 'AES', 'CHACHA20' ],
+FIELD_VALS = { 'cipher' : [ 'AES', 'DES'],
                'fwsec' : [ 'ECDHE', 'DHE' ],
-               'mode' : [ 'POLY1305', 'GCM', 'CCM', 'CBC', ],
-               'digest' : [ 'n/a', 'SHA384', 'SHA256', 'SHA', ],
+               'mode' : [ 'GCM', 'CBC' ],
+               'digest' : [ 'SHA384', 'SHA256', 'SHA' ],
                'bitlength' : [ '256', '128', '192' ],
 }
 
 class Ciphersuite(object):
     def __init__(self, name, fwsec, cipher, bitlength, mode, digest):
-        if fwsec == 'EDH':
-            fwsec = 'DHE'
-
-        if mode in [ '_CBC3', '_CBC', '' ]:
-            mode = 'CBC'
-        elif mode == '_GCM':
-            mode = 'GCM'
-
         self.name = name
         self.fwsec = fwsec
         self.cipher = cipher
@@ -82,50 +74,42 @@ class Ciphersuite(object):
 def parse_cipher(ciph):
     m = re.match('(?:TLS1|SSL3)_TXT_(EDH|DHE|ECDHE)_RSA(?:_WITH)?_(AES|DES)_(256|128|192)(|_CBC|_CBC3|_GCM)_(SHA|SHA256|SHA384)$', ciph)
 
-    if m:
-        fwsec, cipher, bits, mode, digest = m.groups()
-        return Ciphersuite(ciph, fwsec, cipher, bits, mode, digest)
+    if not m:
+        print "/* Couldn't parse %s ! */"%ciph
+        return None
 
-    m = re.match('(?:TLS1|SSL3)_TXT_(EDH|DHE|ECDHE)_RSA(?:_WITH)?_(AES|DES)_(256|128|192)_CCM', ciph)
-    if m:
-        fwsec, cipher, bits = m.groups()
-        return Ciphersuite(ciph, fwsec, cipher, bits, "CCM", "n/a")
+    fwsec, cipher, bits, mode, digest = m.groups()
+    if fwsec == 'EDH':
+        fwsec = 'DHE'
 
-    m = re.match('(?:TLS1|SSL3)_TXT_(EDH|DHE|ECDHE)_RSA(?:_WITH)?_CHACHA20_POLY1305', ciph)
-    if m:
-        fwsec, = m.groups()
-        return Ciphersuite(ciph, fwsec, "CHACHA20", "256", "POLY1305", "n/a")
-
-    print "/* Couldn't parse %s ! */"%ciph
-    return None
+    if mode in [ '_CBC3', '_CBC', '' ]:
+        mode = 'CBC'
+    elif mode == '_GCM':
+        mode = 'GCM'
 
+    return Ciphersuite(ciph, fwsec, cipher, bits, mode, digest)
 
 ALL_CIPHERS = []
 
 for fname in sys.argv[1:]:
-    for c in find_ciphers(fname):
-        if usable_cipher(c):
-            parsed = parse_cipher(c)
-            if parsed != None:
-                ALL_CIPHERS.append(parsed)
+    ALL_CIPHERS += (parse_cipher(c)
+                           for c in find_ciphers(fname)
+                           if usable_cipher(c) )
 
 ALL_CIPHERS.sort(key=Ciphersuite.sort_key)
 
-indent = " "*7
-
 for c in ALL_CIPHERS:
     if c is ALL_CIPHERS[-1]:
-        colon = ''
+        colon = ';'
     else:
         colon = ' ":"'
 
     if c.name in MANDATORY:
-        print "%s/* Required */"%indent
-        print '%s%s%s'%(indent,c.name,colon)
+        print "       /* Required */"
+        print '       %s%s'%(c.name,colon)
     else:
         print "#ifdef %s"%c.name
-        print '%s%s%s'%(indent,c.name,colon)
+        print '       %s%s'%(c.name,colon)
         print "#endif"
 
-print '%s;'%indent
 

scripts/codegen/get_mozilla_ciphers.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python
 # coding=utf-8
-# Copyright 2011-2017, The Tor Project, Inc
+# Copyright 2011-2015, The Tor Project, Inc
 # original version by Arturo Filastò
 # See LICENSE for licensing information
 
@@ -127,9 +127,9 @@ for k, v in enabled_ciphers.items():
 #oSSLinclude = ('/usr/include/openssl/ssl3.h', '/usr/include/openssl/ssl.h',
 #               '/usr/include/openssl/ssl2.h', '/usr/include/openssl/ssl23.h',
 #               '/usr/include/openssl/tls1.h')
-oSSLinclude = ['ssl3.h', 'ssl.h'
-               'ssl2.h', 'ssl23.h',
-               'tls1.h']
+oSSLinclude = ('ssl/ssl3.h', 'ssl/ssl.h',
+               'ssl/ssl2.h', 'ssl/ssl23.h',
+               'ssl/tls1.h')
 
 #####
 # This reads the hex code for the ciphers that are used by firefox.
@@ -155,12 +155,9 @@ for x in used_ciphers:
 openssl_macro_by_hex = {}
 all_openssl_macros = {}
 for fl in oSSLinclude:
-    fname = ossl("include/openssl/"+fl)
-    if not os.path.exists(fname):
-        continue
-    fp = open(fname, 'r')
+    fp = open(ossl(fl), 'r')
     for line in fp.readlines():
-        m = re.match('# *define\s+(\S+)\s+(\S+)', line)
+        m = re.match('#define\s+(\S+)\s+(\S+)', line)
         if m:
             value,key = m.groups()
             if key.startswith('0x') and "_CK_" in value:

scripts/codegen/makedesc.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2014-2017, The Tor Project, Inc.
+# Copyright 2014-2015, The Tor Project, Inc.
 # See LICENSE for license information
 
 # This is a kludgey python script that uses ctypes and openssl to sign

scripts/codegen/run_trunnel.sh

@@ -5,13 +5,7 @@ if test "x$TRUNNEL_PATH" != "x"; then
   export PYTHONPATH
 fi
 
-OPTIONS="--require-version=1.5.1"
+python -m trunnel --require-version=1.4 ./src/trunnel/*.trunnel
 
-# Get all .trunnel files recursively from that directory so we can support
-# multiple sub-directories.
-for file in `find ./src/trunnel/ -name '*.trunnel'`; do
-  python -m trunnel ${OPTIONS} $file
-done
-
-python -m trunnel ${OPTIONS} --write-c-files --target-dir=./src/ext/trunnel/
+python -m trunnel --require-version=1.4 --write-c-files --target-dir=./src/ext/trunnel/
 

scripts/maint/analyze_callgraph.py

@@ -0,0 +1,259 @@
+#!/usr/bin/python
+
+import re
+import sys
+import copy
+import cPickle
+import os
+
+class Parser:
+  def __init__(self):
+    self.calls = {}
+    self.definedIn = {}
+
+  def enter_func(self, name):
+    if self.infunc and not self.extern and self.calledfns:
+      if self.infunc in self.definedIn:
+        #print "{}: {} or {}?".format(
+        #  self.infunc, self.definedIn[self.infunc], self.module)
+        self.definedIn[self.infunc] = 'nil'
+      else:
+        self.definedIn[self.infunc] = self.module
+      self.calls.setdefault(self.infunc, set()).update( self.calledfns )
+
+    self.calledfns = set()
+    self.infunc = name
+    self.extern = False
+
+  def parse_callgraph_file(self, inp, module):
+    self.infunc = None
+    self.extern = False
+    self.calledfns = set()
+    self.module = module
+
+    for line in inp:
+       m = re.match(r"Call graph node for function: '([^']+)'", line)
+       if m:
+           self.enter_func(m.group(1))
+           continue
+       m = re.match(r"  CS<[^>]+> calls external node", line)
+       if m:
+           self.extern = True
+       m = re.match(r"  CS<[^>]+> calls function '([^']+)'", line)
+       if m:
+           self.calledfns.add(m.group(1))
+    self.enter_func(None)
+
+  def extract_callgraph(self):
+    c = self.calls
+    self.calls = {}
+    return c
+
+
+def transitive_closure(g):
+    passno = 0
+    changed = True
+    g = copy.deepcopy(g)
+    import random
+    while changed:
+      passno += 1
+      changed = False
+      keys = g.keys()
+      idx = 0
+      for k in keys:
+         idx += 1
+         print "Pass %d/?: %d/%d\r" %(passno, idx, len(keys)),
+         sys.stdout.flush()
+         newset = g[k].copy()
+         for fn in g[k]:
+            newset.update(g.get(fn, set()))
+         if len(newset) != len(g[k]):
+            g[k].update( newset )
+            changed = True
+
+      print
+
+    return g
+
+def strongly_connected_components(g):
+  # From https://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm, done stupidly.
+  index_of = {}
+  index = [ 0 ]
+  lowlink = {}
+  S = []
+  onStack = set()
+
+  all_sccs = []
+
+  def strongconnect(fn):
+    index_of[fn] = index[0]
+    lowlink[fn] = index[0]
+    index[0] += 1
+    S.append(fn)
+    onStack.add(fn)
+
+    for w in g.get(fn, []):
+      if w not in index_of:
+        strongconnect(w)
+        lowlink[fn] = min(lowlink[fn], lowlink[w])
+      elif w in onStack:
+        lowlink[fn] = min(lowlink[fn], index_of[w])
+
+    if lowlink[fn] == index_of[fn]:
+      this_scc = []
+      all_sccs.append(this_scc)
+      while True:
+        w = S.pop()
+        onStack.remove(w)
+        this_scc.append(w)
+        if w == fn:
+          break
+
+  for v in g.keys():
+    if v not in index_of:
+      strongconnect(v)
+
+  return all_sccs
+
+def biggest_component(sccs):
+  return max(len(c) for c in sccs)
+
+def connection_bottlenecks(callgraph):
+
+  callers = {}
+  for fn in callgraph:
+    for fn2 in callgraph[fn]:
+      callers.setdefault(fn2, set()).add(fn)
+
+  components = strongly_connected_components(callgraph)
+  components.sort(key=len)
+  big_component_fns = components[-1]
+  size = len(big_component_fns)
+
+  function_bottlenecks = fn_results = []
+
+  total = len(big_component_fns)
+  idx = 0
+  for fn in big_component_fns:
+    idx += 1
+    print "Pass 1/3: %d/%d\r"%(idx, total),
+    sys.stdout.flush()
+    cg2 = copy.deepcopy(callgraph)
+    del cg2[fn]
+
+    fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), fn) )
+
+  print
+  bcf_set = set(big_component_fns)
+
+  call_bottlenecks = fn_results = []
+  result_set = set()
+  total = len(big_component_fns)
+  idx = 0
+  for fn in big_component_fns:
+    fn_callers = callers[fn].intersection(bcf_set)
+    idx += 1
+    if len(fn_callers) != 1:
+      continue
+
+    print "Pass 2/3: %d/%d\r"%(idx, total),
+    sys.stdout.flush()
+
+    caller = fn_callers.pop()
+    assert len(fn_callers) == 0
+    cg2 = copy.deepcopy(callgraph)
+    cg2[caller].remove(fn)
+
+    fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), fn, "called by", caller) )
+    result_set.add( (caller, fn) )
+
+  print
+
+  total = len(big_component_fns)
+  idx = 0
+  for fn in big_component_fns:
+    fn_calls = callgraph[fn].intersection(bcf_set)
+    idx += 1
+    if len(fn_calls) != 1:
+      continue
+
+    print "Pass 3/3: %d/%d\r"%(idx, total),
+    sys.stdout.flush()
+
+    callee = fn_calls.pop()
+    if (fn, callee) in result_set:
+      continue
+
+    assert len(fn_calls) == 0
+    cg2 = copy.deepcopy(callgraph)
+    cg2[fn].remove(callee)
+
+    fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), callee, "called by", fn) )
+
+  print
+
+
+  return (function_bottlenecks, call_bottlenecks)
+
+if __name__ == '__main__':
+    p = Parser()
+    for fname in sys.argv[1:]:
+      modname = re.sub(r'.*/', '', fname).replace('.callgraph', '.c')
+      with open(fname, 'r') as f:
+        p.parse_callgraph_file(f, modname)
+
+    sys.stdout.flush()
+
+    print "Building callgraph"
+    callgraph = p.extract_callgraph()
+    inModule = p.definedIn
+
+    print "Deriving module callgraph"
+    modCallgraph = {}
+    for fn in callgraph:
+      fnMod = inModule[fn]
+      for called in callgraph[fn]:
+        try:
+          calledMod = inModule[called]
+        except KeyError:
+            continue
+        modCallgraph.setdefault(fnMod, set()).add(calledMod)
+    del modCallgraph['nil']
+
+    print "Finding strongly connected components"
+    sccs = strongly_connected_components(callgraph)
+
+    print "Finding the transitive closure of the callgraph.."
+    closure = transitive_closure(callgraph)
+
+    print "Finding bottlenecks..."
+    bottlenecks = connection_bottlenecks(callgraph)
+
+    print "Finding module SCCs"
+    modSCCS = strongly_connected_components(modCallgraph)
+
+    print "Finding module TC"
+    modTC = transitive_closure(modCallgraph)
+
+    print "Finding module bottlenecks"
+    modB = connection_bottlenecks(modCallgraph)
+
+    data = {
+      'callgraph' : callgraph,
+      'sccs' : sccs,
+      'closure' : closure,
+      'bottlenecks' : bottlenecks,
+      'modules' : p.definedIn,
+      'modItems' : {
+        'callgraph' : modCallgraph,
+        'sccs' : modSCCS,
+        'closure' : modTC,
+        'bottlenecks' : modB,
+      }
+    }
+
+    with open('callgraph.pkl', 'w') as f:
+      cPickle.dump(data, f)
+
+
+

scripts/maint/annotate_ifdef_directives

@@ -1,74 +0,0 @@
-#!/usr/bin/python
-# Copyright (c) 2017, The Tor Project, Inc.
-# See LICENSE for licensing information
-
-import re
-
-LINE_OBVIOUSNESS_LIMIT = 4
-
-class Problem(Exception):
-    pass
-
-def uncomment(s):
-    s = re.sub(r'//.*','',s)
-    s = re.sub(r'/\*.*','',s)
-    return s.strip()
-
-def translate(f_in, f_out):
-    whole_file = []
-    stack = []
-    cur_level = whole_file
-    lineno = 0
-    for line in f_in:
-        lineno += 1
-        m = re.match(r'\s*#\s*(if|ifdef|ifndef|else|endif|elif)\b\s*(.*)',
-                     line)
-        if not m:
-            f_out.write(line)
-            continue
-        command,rest = m.groups()
-        if command in ("if", "ifdef", "ifndef"):
-            # The #if directive pushes us one level lower on the stack.
-            if command == 'ifdef':
-                rest = "defined(%s)"%uncomment(rest)
-            elif command == 'ifndef':
-                rest = "!defined(%s)"%uncomment(rest)
-            elif rest.endswith("\\"):
-                rest = rest[:-1]+"..."
-
-            rest = uncomment(rest)
-
-            new_level = [ (command, rest, lineno) ]
-            stack.append(cur_level)
-            cur_level = new_level
-            f_out.write(line)
-        elif command in ("else", "elif"):
-            if len(cur_level) == 0 or cur_level[-1][0] == 'else':
-                raise Problem("Unexpected #%s on %d"% (command,lineno))
-            if (len(cur_level) == 1 and command == 'else' and
-                lineno > cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT):
-                f_out.write("#else /* !(%s) */\n"%cur_level[0][1])
-            else:
-                f_out.write(line)
-            cur_level.append((command, rest, lineno))
-        else:
-            assert command == 'endif'
-            if len(stack) == 0:
-                raise Problem("Unmatched #%s on %s"% (command,lineno))
-            if lineno <= cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT:
-                f_out.write(line)
-            elif len(cur_level) == 1 or (
-                    len(cur_level) == 2 and cur_level[1][0] == 'else'):
-                f_out.write("#endif /* %s */\n"%cur_level[0][1])
-            else:
-                f_out.write("#endif /* %s || ... */\n"%cur_level[0][1])
-            cur_level = stack.pop()
-    if len(stack) or cur_level != whole_file:
-        raise Problem("Missing #endif")
-
-import sys,os
-for fn in sys.argv[1:]:
-    with open(fn+"_OUT", 'w') as output_file:
-        translate(open(fn, 'r'), output_file)
-    os.rename(fn+"_OUT", fn)
-

scripts/maint/checkSpace.pl

@@ -1,59 +1,48 @@
-#!/usr/bin/perl
-
-use strict;
-use warnings;
-
-my $found = 0;
-sub msg {
-  $found = 1;
-  print "$_[0]";
-}
-
-my $C = 0;
+#!/usr/bin/perl -w
 
 if ($ARGV[0] =~ /^-/) {
-    my $lang = shift @ARGV;
+    $lang = shift @ARGV;
     $C = ($lang eq '-C');
+#    $TXT = ($lang eq '-txt');
 }
 
-for my $fn (@ARGV) {
+for $fn (@ARGV) {
     open(F, "$fn");
-    my $lastnil = 0;
-    my $lastline = "";
-    my $incomment = 0;
-    my $in_func_head = 0;
+    $lastnil = 0;
+    $lastline = "";
+    $incomment = 0;
     while (<F>) {
         ## Warn about windows-style newlines.
-        #    (We insist on lines that end with a single LF character, not
-        #    CR LF.)
+	#    (We insist on lines that end with a single LF character, not
+	#    CR LF.)
         if (/\r/) {
-            msg "       CR:$fn:$.\n";
+            print "       CR:$fn:$.\n";
         }
         ## Warn about tabs.
-        #    (We only use spaces)
+	#    (We only use spaces)
         if (/\t/) {
-            msg "      TAB:$fn:$.\n";
+            print "      TAB:$fn:$.\n";
         }
         ## Warn about labels that don't have a space in front of them
-        #    (We indent every label at least one space)
+	#    (We indent every label at least one space)
         if (/^[a-zA-Z_][a-zA-Z_0-9]*:/) {
-            msg "nosplabel:$fn:$.\n";
+            print "nosplabel:$fn:$.\n";
         }
         ## Warn about trailing whitespace.
-        #    (We don't allow whitespace at the end of the line; make your
-        #    editor highlight it for you so you can stop adding it in.)
+	#    (We don't allow whitespace at the end of the line; make your
+	#    editor highlight it for you so you can stop adding it in.)
         if (/ +$/) {
-            msg "Space\@EOL:$fn:$.\n";
+            print "Space\@EOL:$fn:$.\n";
         }
         ## Warn about control keywords without following space.
-        #    (We put a space after every 'if', 'while', 'for', 'switch', etc)
+	#    (We put a space after every 'if', 'while', 'for', 'switch', etc)
         if ($C && /\s(?:if|while|for|switch)\(/) {
-            msg "      KW(:$fn:$.\n";
+            print "      KW(:$fn:$.\n";
         }
         ## Warn about #else #if instead of #elif.
         #    (We only allow #elif)
         if (($lastline =~ /^\# *else/) and ($_ =~ /^\# *if/)) {
-            msg " #else#if:$fn:$.\n";
+            print " #else#if:$fn:$.\n";
         }
         ## Warn about some K&R violations
         #    (We use K&R-style C, where open braces go on the same line as
@@ -64,23 +53,23 @@ for my $fn (@ARGV) {
         #            other stuff;
         #          }
         if (/^\s+\{/ and $lastline =~ /^\s*(if|while|for|else if)/ and
-            $lastline !~ /\{$/) {
-            msg "non-K&R {:$fn:$.\n";
-        }
+	    $lastline !~ /\{$/) {
+            print "non-K&R {:$fn:$.\n";
+	}
         if (/^\s*else/ and $lastline =~ /\}$/) {
-            msg "  }\\nelse:$fn:$.\n";
-        }
+	    print "  }\\nelse:$fn:$.\n";
+	}
         $lastline = $_;
         ## Warn about unnecessary empty lines.
         #   (Don't put an empty line before a line that contains nothing
         #   but a closing brace.)
         if ($lastnil && /^\s*}\n/) {
-            msg "  UnnecNL:$fn:$.\n";
+            print "  UnnecNL:$fn:$.\n";
         }
         ## Warn about multiple empty lines.
         #   (At most one blank line in a row.)
         if ($lastnil && /^$/) {
-            msg " DoubleNL:$fn:$.\n";
+            print " DoubleNL:$fn:$.\n";
         } elsif (/^$/) {
             $lastnil = 1;
         } else {
@@ -90,7 +79,7 @@ for my $fn (@ARGV) {
         ## accept double-line lines.
         #   (Don't make lines wider than 80 characters, including newline.)
         if (/^.{80}/) {
-            msg "     Wide:$fn:$.\n";
+            print "     Wide:$fn:$.\n";
         }
         ### Juju to skip over comments and strings, since the tests
         ### we're about to do are okay there.
@@ -113,52 +102,48 @@ for my $fn (@ARGV) {
             s!"(?:[^\"]+|\\.)*"!"X"!g;
             next if /^\#/;
             ## Warn about C++-style comments.
-            #   (Use C style comments only.)
+	    #   (Use C style comments only.)
             if (m!//!) {
-                #    msg "       //:$fn:$.\n";
+                #    print "       //:$fn:$.\n";
                 s!//.*!!;
             }
             ## Warn about unquoted braces preceded by non-space.
-            #   (No character except a space should come before a {)
+	    #   (No character except a space should come before a {)
             if (/([^\s'])\{/) {
-                msg "       $1\{:$fn:$.\n";
+                print "       $1\{:$fn:$.\n";
             }
-            ## Warn about double semi-colons at the end of a line.
-            if (/;;$/) {
-                msg "       double semi-colons at the end of $. in $fn\n"
-            }            
             ## Warn about multiple internal spaces.
             #if (/[^\s,:]\s{2,}[^\s\\=]/) {
-            #    msg "     X  X:$fn:$.\n";
+            #    print "     X  X:$fn:$.\n";
             #}
             ## Warn about { with stuff after.
             #s/\s+$//;
             #if (/\{[^\}\\]+$/) {
-            #    msg "     {X:$fn:$.\n";
+            #    print "     {X:$fn:$.\n";
             #}
             ## Warn about function calls with space before parens.
-            #   (Don't put a space between the name of a function and its
-            #   arguments.)
+	    #   (Don't put a space between the name of a function and its
+	    #   arguments.)
             if (/(\w+)\s\(([A-Z]*)/) {
                 if ($1 ne "if" and $1 ne "while" and $1 ne "for" and
                     $1 ne "switch" and $1 ne "return" and $1 ne "int" and
                     $1 ne "elsif" and $1 ne "WINAPI" and $2 ne "WINAPI" and
                     $1 ne "void" and $1 ne "__attribute__" and $1 ne "op" and
-                    $1 ne "size_t" and $1 ne "double" and $1 ne "uint64_t" and
+                    $1 ne "size_t" and $1 ne "double" and
                     $1 ne "workqueue_reply_t") {
-                    msg "     fn ():$fn:$.\n";
+                    print "     fn ():$fn:$.\n";
                 }
             }
             ## Warn about functions not declared at start of line.
-            #    (When you're declaring functions, put "static" and "const"
-            #    and the return type on one line, and the function name at
-            #    the start of a new line.)
+	    #    (When you're declaring functions, put "static" and "const"
+	    #    and the return type on one line, and the function name at
+	    #    the start of a new line.)
             if ($in_func_head ||
                 ($fn !~ /\.h$/ && /^[a-zA-Z0-9_]/ &&
                  ! /^(?:const |static )*(?:typedef|struct|union)[^\(]*$/ &&
                  ! /= *\{$/ && ! /;$/)) {
                 if (/.\{$/){
-                    msg "fn() {:$fn:$.\n";
+                    print "fn() {:$fn:$.\n";
                     $in_func_head = 0;
                 } elsif (/^\S[^\(]* +\**[a-zA-Z0-9_]+\(/) {
                     $in_func_head = -1; # started with tp fn
@@ -166,38 +151,18 @@ for my $fn (@ARGV) {
                     $in_func_head = 0;
                 } elsif (/\{/) {
                     if ($in_func_head == -1) {
-                        msg "tp fn():$fn:$.\n";
+                        print "tp fn():$fn:$.\n";
                     }
                     $in_func_head = 0;
                 }
             }
-
-            ## Check for forbidden functions except when they are
-            # explicitly permitted
-            if (/\bassert\(/ && not /assert OK/) {
-                msg "assert :$fn:$.   (use tor_assert)\n";
-            }
-            if (/\bmemcmp\(/ && not /memcmp OK/) {
-                msg "memcmp :$fn:$.   (use {tor,fast}_mem{eq,neq,cmp}\n";
-            }
-            # always forbidden.
-            if (not /\ OVERRIDE\ /) {
-                if (/\bstrcat\(/ or /\bstrcpy\(/ or /\bsprintf\(/) {
-                    msg "$& :$fn:$.\n";
-                }
-                if (/\bmalloc\(/ or /\bfree\(/ or /\brealloc\(/ or
-                    /\bstrdup\(/ or /\bstrndup\(/ or /\bcalloc\(/) {
-                    msg "$& :$fn:$.    (use tor_malloc, tor_free, etc)\n";
-                }
-            }
         }
     }
     ## Warn if the file doesn't end with a blank line.
     #    (End each file with a single blank line.)
     if (! $lastnil) {
-        msg "  EOL\@EOF:$fn:$.\n";
+        print "  EOL\@EOF:$fn:$.\n";
     }
     close(F);
 }
 
-exit $found;

scripts/maint/display_callgraph.py

@@ -0,0 +1,41 @@
+#!/usr/bin/python
+
+import cPickle
+
+data = cPickle.load(open("callgraph.pkl"))
+
+# data = data['modItems']
+
+callgraph = data['callgraph']
+closure = data['closure']
+sccs = data['sccs']
+fn_bottle, call_bottle = data['bottlenecks']
+
+for n_reachable, fn in sorted(list((len(r), fn) for fn, r in closure.iteritems())):
+   print "%s can reach %s other functions." %(fn, n_reachable)
+
+
+c = [ (len(component), component) for component in sccs ]
+c.sort()
+
+print "\n================================"
+
+for n, component in c:
+   if n < 2:
+      continue
+   print "Strongly connected component of size %d:"%n
+   print component
+
+
+print "\n================================"
+
+print "====== Number of functions pulled into blob, by function in blob."
+fn_bottle.sort()
+for n, fn in fn_bottle[-30:]:
+   print "%3d: %s"%(n, fn)
+
+print "====== Number of functions pulled into blob, by call in blob."
+call_bottle.sort()
+for n, fn1, _, fn2 in call_bottle[-30:]:
+   print "%3d: %s -> %s "%(n, fn2, fn1)
+

scripts/maint/fallback.blacklist

@@ -3,8 +3,6 @@
 # Format:
 # [ IPv4[:DirPort] ] [ orport=<ORPort> ] [ id=<ID> ] ...
 #   [ ipv6=<IPv6>[:<IPv6 ORPort>] ]
-# or use:
-# scripts/maint/generateFallbackDirLine.py fingerprint ...
 #
 # If a sufficiently specific group of attributes matches, the directory mirror
 # will be excluded: (each group is listed on its own line)
@@ -29,6 +27,11 @@
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008384.html
 80.82.215.199:80 orport=443 id=3BEFAB76461B6B99DCF34C285E933562F5712AE4 ipv6=[2001:4ba0:cafe:a18::1]:443
 
+# https://lists.torproject.org/pipermail/tor-relays/2016-January/008515.html
+# later opt-out in
+# https://lists.torproject.org/pipermail/tor-relays/2016-January/008521.html
+5.9.158.75:80 orport=443 id=F1BE15429B3CE696D6807F4D4A58B1BFEC45C822 ipv6=[2a01:4f8:190:514a::2]:443
+
 # Email sent directly to teor, verified using relay contact info
 5.34.183.168:80 orport=443 id=601C92108A568742A7A6D9473FE3A414F7149070
 217.12.199.208:8080 orport=22 id=BCFB0933367D626715DA32A147F417194A5D48D6
@@ -37,6 +40,14 @@
 62.210.207.124:9030 orport=9001 id=58938B1A5C4029B4415D38A4F36B7724273F4755 ipv6=[2001:bc8:31eb:100::1]:9001
 62.210.207.124:9130 orport=9101 id=338D0AB6DBAB7B529B9C91B2FD770658000693C4 ipv6=[2001:bc8:31eb:100::1]:9101
 
+# these fallback candidates fail the consensus download test in a way that
+# causes stem to hang (and not respond to ^C, at least on OS X)
+# (Is something sending weird responses to DirPort traffic?)
+#217.23.14.190:1194
+#151.80.164.147:80
+#148.251.255.92:80
+#78.142.19.59:80
+
 # Email sent directly to teor, verified using relay contact info
 216.17.99.183:80 orport=443 id=D52CD431CEF28E01B11F545A84347EE45524BCA7
 216.17.99.183:8080 orport=9001 id=EE21F83AB6F76E3B3FFCBA5C2496F789CB84E7C6
@@ -68,6 +79,9 @@
 # Email sent directly to teor, verified using relay contact info
 185.21.216.140:9030 orport=9001 id=921DA852C95141F8964B359F774B35502E489869
 
+# Email sent directly to teor, verified using relay contact info
+62.210.82.44:143 orport=21 id=1C90D3AEADFF3BCD079810632C8B85637924A58E ipv6=[2001:bc8:3d7c::]:21
+
 # Email sent directly to teor, verified using relay contact info
 46.101.220.161:80 orport=443 id=7DDFE5B2C306B19A79832FBE581EAA245BAE90C6 ipv6=[2a03:b0c0:3:d0::8b:3001]:443
 
@@ -118,6 +132,7 @@
 85.114.135.20:9030 orport=9001 id=ED8A9291A3139E34BBD35037B082081EC6C26C80 ipv6=[2001:4ba0:fff5:2d::8]:9001
 148.251.128.156:9030 orport=9001 id=E382042E06A0A68AFC533E5AD5FB6867A12DF9FF ipv6=[2a01:4f8:210:238a::8]:9001
 62.210.115.147:9030 orport=9001 id=7F1D94E2C36F8CC595C2AB00022A5AE38171D50B ipv6=[2001:bc8:3182:101::8]:9001
+212.47.250.24:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB
 
 # Email sent directly to teor, verified using relay contact info
 74.208.220.222:60000 orport=59999 id=4AA22235F0E9B3795A33930343CBB3EDAC60C5B0
@@ -191,42 +206,24 @@
 # Email sent directly to teor, verified using relay contact info
 88.190.208.4:30555 orport=30556 id=030A6EB24725C05D8E0FCE21923CBA5223E75E0E
 
+# Fallback was on 0.2.8.2-alpha list, but changed fingerprint before 0.2.8.5
+46.101.102.71:80 orport=443 id=9504CB22EEB25D344DE63CB7A6F2C46F895C3686 ipv6=[2a03:b0c0:3:d0::2ed:7001]:9050
+# Also blacklist anything with the new fingerprint
+id=9C8A123081EFBE022EF795630F447839DDFDDDEC
+
+# Fallbacks were on 0.2.8.2-alpha list, but downloads were slow before 0.2.8.5
+185.96.88.29:80 orport=443 id=86C281AD135058238D7A337D546C902BE8505DDE
+178.62.36.64:9030 orport=9001 id=B87C84E38DAECFFFFDE98E5AEE5786AFDC748F2C
+
+# Fallback was on 0.2.8.2-alpha list, but changed address before 0.2.8.5
+84.219.173.60:9030 orport=443 id=855BC2DABE24C861CD887DB9B2E950424B49FC34
+# Also blacklist anything with the new address
+84.216.235.55:9030 orport=443
+
+# Fallbacks were on 0.2.8.2-alpha list, but disappeared before 0.2.8.5
+81.7.17.171:80 orport=443 id=CFECDDCA990E3EF7B7EC958B22441386B6B8D820 ipv6=[2a02:180:1:1::517:11ab]:443
+51.254.215.121:80 orport=443 id=262B66AD25C79588AD1FC8ED0E966395B47E5C1D
+185.100.85.138:80 orport=46356 id=5C4DF16A0029CC4F67D3E127356E68F219269859
+
 # Fallback was on 0.2.8.2-alpha list, but opted-out before 0.2.8.6
 37.187.1.149:9030 orport=9001 id=08DC0F3C6E3D9C527C1FC8745D35DD1B0DE1875D ipv6=[2001:41d0:a:195::1]:9001
-
-# Email sent directly to teor, verified using relay contact info
-195.154.15.227:9030 orport=9001 id=6C3E3AB2F5F03CD71B637D433BAD924A1ECC5796
-
-# Fallback was on 0.2.8.6 list, but opted-out before 0.2.9
-144.76.73.140:9030 orport=9001 id=6A640018EABF3DA9BAD9321AA37C2C87BBE1F907
-
-# https://lists.torproject.org/pipermail/tor-relays/2016-December/011114.html
-# no dirport
-86.107.110.34:0 orport=9001 id=A0E3D30A660DB70CA0B6D081BA54D094DED6F28D
-94.242.59.147:80 orport=9001 id=674DCBB0D9C1C4C4DBFB4A9AE024AF59FE4E7F46 ipv6=[2a00:1838:35:42::b648]:9001
-
-# Email sent directly to teor, verified using relay contact info
-167.114.152.100:9030 orport=443 id=0EF5E5FFC5D1EABCBDA1AFF6F6D6325C5756B0B2 ipv6=[2607:5300:100:200::1608]:443
-
-# Email sent directly to teor, verified using relay contact info
-163.172.35.245:80 orport=443 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24
-
-# Email sent directly to teor, verified using relay contact info
-104.243.35.196:9030 orport=9001 id=FA3415659444AE006E7E9E5375E82F29700CFDFD
-
-# Emails sent directly to teor, verified using relay contact info
-217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05 ipv6=[2a02:27a8:0:2::7e]:443
-
-# Emails sent directly to teor, verified using relay contact info
-195.154.75.84:9030 orport=9001 id=F80FDE27EFCB3F6A7B4E2CC517133DBFFA78BA2D
-195.154.127.246:9030 orport=9001 id=4FEE77AFFD157BBCF2D896AE417FBF647860466C
-
-# Email sent directly to teor, verified using relay contact info
-5.35.251.247:9030 orport=9001 id=9B1F5187DFBA89DC24B37EA7BF896C12B43A27AE
-
-#https://lists.torproject.org/pipermail/tor-relays/2017-May/012281.html
-62.210.124.124:9030 orport=9001 id=86E78DD3720C78DA8673182EF96C54B162CD660C ipv6=[2001:bc8:3f23:100::1]:9001
-62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E ipv6=[2001:bc8:3f23:100::1]:9101
-
-# Email sent directly to teor
-212.51.156.193:995 orport=110 id=32E7AAF1F602814D699BEF6761AD03E387758D49 ipv6=[2a02:168:4a01::49]:110

scripts/maint/fallback.whitelist

@@ -2,8 +2,6 @@
 #
 # Format:
 # IPv4:DirPort orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ]
-# or use:
-# scripts/maint/generateFallbackDirLine.py fingerprint ...
 #
 # All attributes must match for the directory mirror to be included.
 # If the fallback has an ipv6 key, the whitelist line must also have
@@ -30,9 +28,8 @@
 # <IPv4>:<DirPort> orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ]
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008362.html
-# https://trac.torproject.org/projects/tor/ticket/22321#comment:22
-78.47.18.110:443 orport=80 id=F8D27B163B9247B232A2EEE68DD8B698695C28DE ipv6=[2a01:4f8:120:4023::110]:80 # fluxe3
-131.188.40.188:1443 orport=80 id=EBE718E1A49EE229071702964F8DB1F318075FF8 ipv6=[2001:638:a000:4140::ffff:188]:80 # fluxe4
+78.47.18.110:443 orport=80 id=F8D27B163B9247B232A2EEE68DD8B698695C28DE
+131.188.40.188:443 orport=80 id=EBE718E1A49EE229071702964F8DB1F318075FF8
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008366.html
 5.39.88.19:9030 orport=9001 id=7CB8C31432A796731EA7B6BF4025548DFEB25E0C ipv6=[2001:41d0:8:9a13::1]:9050
@@ -40,6 +37,8 @@
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008370.html
 # https://lists.torproject.org/pipermail/tor-relays/2016-January/008517.html
 # https://lists.torproject.org/pipermail/tor-relays/2016-January/008555.html
+62.210.124.124:9030 orport=9001 id=86E78DD3720C78DA8673182EF96C54B162CD660C ipv6=[2001:bc8:3f23:100::1]:9001
+62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E ipv6=[2001:bc8:3f23:100::1]:9101
 212.47.237.95:9030 orport=9001 id=3F5D8A879C58961BB45A3D26AC41B543B40236D6
 212.47.237.95:9130 orport=9101 id=6FB38EB22E57EF7ED5EF00238F6A48E553735D88
 
@@ -50,26 +49,27 @@
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008373.html
 167.114.35.28:9030 orport=9001 id=E65D300F11E1DB12C534B0146BDAB6972F1A8A48
 
+# https://lists.torproject.org/pipermail/tor-relays/2015-December/008374.html
+170.130.1.7:9030 orport=9001 id=FA3415659444AE006E7E9E5375E82F29700CFDFD
+
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008378.html
 144.76.14.145:110 orport=143 id=14419131033443AE6E21DA82B0D307F7CAE42BDB ipv6=[2a01:4f8:190:9490::dead]:443
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html
 # Email sent directly to teor, verified using relay contact info
 91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F
-91.121.84.137:4952 orport=4052 id=9FBEB75E8BC142565F12CBBE078D63310236A334
+
+# https://lists.torproject.org/pipermail/tor-relays/2015-December/008380.html
+5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008381.html
-# Sent additional emails to teor with updated relays
-81.7.11.96:9030 orport=9001 id=8FA37B93397015B2BC5A525C908485260BE9F422 # Doedel22
-# 9F5068310818ED7C70B0BC4087AB55CB12CB4377 not found in current consensus
-178.254.19.101:80 orport=443 id=F9246DEF2B653807236DA134F2AEAB103D58ABFE # Freebird31
-178.254.19.101:9030 orport=9001 id=0C475BA4D3AA3C289B716F95954CAD616E50C4E5 # Freebird32
-81.7.14.253:9001 orport=443 id=1AE039EE0B11DB79E4B4B29CBA9F752864A0259E # Ichotolot60
-81.7.11.186:1080 orport=443 id=B86137AE9681701901C6720E55C16805B46BD8E3 # BeastieJoy60
-85.25.213.211:465 orport=80 id=CE47F0356D86CF0A1A2008D97623216D560FB0A8 # BeastieJoy61
-85.25.159.65:995 orport=80 id=52BFADA8BEAA01BA46C8F767F83C18E2FE50C1B9 # BeastieJoy63
-81.7.3.67:993 orport=443 id=A2E6BB5C391CD46B38C55B4329C35304540771F1 # BeastieJoy62
-81.7.14.31:9001 orport=443 id=7600680249A22080ECC6173FBBF64D6FCF330A61 # Ichotolot62
+# Sent additional email to teor with more relays
+178.254.44.135:9030 orport=9001 id=8FA37B93397015B2BC5A525C908485260BE9F422
+178.254.20.134:80 orport=443 id=9F5068310818ED7C70B0BC4087AB55CB12CB4377
+178.254.20.134:9030 orport=9001 id=2CE96A8A1DA032664C90F574AFFBECE18A6E8DFC
+178.254.44.135:80 orport=443 id=AE6A8C18E7499B586CD36246AC4BCAFFBBF93AB2
+178.254.13.126:80 orport=443 id=F9246DEF2B653807236DA134F2AEAB103D58ABFE
+178.254.13.126:9030 orport=9001 id=0C475BA4D3AA3C289B716F95954CAD616E50C4E5
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008382.html
 51.255.33.237:9091 orport=9001 id=A360C21FA87FFA2046D92C17086A6B47E5C68109
@@ -98,28 +98,33 @@
 # https://lists.torproject.org/pipermail/tor-relays/2016-January/008542.html
 178.62.199.226:80 orport=443 id=CBEFF7BA4A4062045133C053F2D70524D8BBE5BE ipv6=[2a03:b0c0:2:d0::b7:5001]:443
 
+# Emails sent directly to teor, verified using relay contact info
+217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05
+
 # Email sent directly to teor, verified using relay contact info
 94.23.204.175:9030 orport=9001 id=5665A3904C89E22E971305EE8C1997BCA4123C69
 
+# https://twitter.com/binarytenshi/status/717952514327453697
+94.126.23.174:9030 orport=9001 id=6FC6F08270D565BE89B7C819DD8E2D487397C073
+
 # Email sent directly to teor, verified using relay contact info
-171.25.193.77:80 orport=443 id=A10C4F666D27364036B562823E5830BC448E046A ipv6=[2001:67c:289c:3::77]:443
 171.25.193.78:80 orport=443 id=A478E421F83194C114F41E94F95999672AED51FE ipv6=[2001:67c:289c:3::78]:443
+171.25.193.77:80 orport=443 id=A10C4F666D27364036B562823E5830BC448E046A ipv6=[2001:67c:289c:3::77]:443
+171.25.193.131:80 orport=443 id=79861CF8522FC637EF046F7688F5289E49D94576
 171.25.193.20:80 orport=443 id=DD8BD7307017407FCC36F8D04A688F74A0774C02 ipv6=[2001:67c:289c::20]:443
-# same machine as DD8BD7307017407FCC36F8D04A688F74A0774C02
-171.25.193.25:80 orport=443 id=185663B7C12777F052B2C2D23D7A239D8DA88A0F ipv6=[2001:67c:289c::25]:443
+# OK, but same machine as 79861CF8522FC637EF046F7688F5289E49D94576
+#171.25.193.132:80 orport=443 id=01C67E0CA8F97111E652C7564CB3204361FFFAB8
+# OK, but same machine as DD8BD7307017407FCC36F8D04A688F74A0774C02
+#171.25.193.25:80 orport=443 id=185663B7C12777F052B2C2D23D7A239D8DA88A0F ipv6=[2001:67c:289c::25]:443
 
 # Email sent directly to teor, verified using relay contact info
-212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68 ipv6=[2001:bc8:4400:2100::f03]:9001
+212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68
 93.115.97.242:9030 orport=9001 id=B5212DB685A2A0FCFBAE425738E478D12361710D
-46.28.109.231:9030 orport=9001 id=F70B7C5CD72D74C7F9F2DC84FA9D20D51BA13610 ipv6=[2a02:2b88:2:1::4205:1]:9001
+46.28.109.231:9030 orport=9001 id=F70B7C5CD72D74C7F9F2DC84FA9D20D51BA13610 ipv6=[2a02:2b88:2:1::4205:42]:9001
 
 # Email sent directly to teor, verified using relay contact info
-85.235.250.88:80 orport=443 id=72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE # TykRelay01
-185.96.88.29:80 orport=443 id=86C281AD135058238D7A337D546C902BE8505DDE # TykRelay051
-# This fallback opted-in in previous releases, then changed its details,
-# and so we blacklisted it. Now we want to whitelist changes.
-# Assume details update is permanent
-185.96.180.29:80 orport=443 id=F93D8F37E35C390BCAD9F9069E13085B745EC216 # TykRelay06
+85.235.250.88:80 orport=443 id=72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE
+185.96.180.29:80 orport=443 id=F93D8F37E35C390BCAD9F9069E13085B745EC216
 
 # Email sent directly to teor, verified using relay contact info
 185.11.180.67:80 orport=9001 id=794D8EA8343A4E820320265D05D4FA83AB6D1778
@@ -149,20 +154,31 @@
 178.16.208.59:80 orport=443 id=136F9299A5009A4E0E96494E723BDB556FB0A26B ipv6=[2a00:1c20:4089:1234:bff6:e1bb:1ce3:8dc6]:443
 
 # Email sent directly to teor, verified using relay contact info
+195.154.8.111:80 orport=443 id=FCB6695F8F2DC240E974510A4B3A0F2B12AB5B64
+51.255.235.246:80 orport=443 id=9B99C72B02AF8E3E5BE3596964F9CACD0090D132
 5.39.76.158:80 orport=443 id=C41F60F8B00E7FEF5CCC5BC6BB514CA1B8AAB651
 
 # Email sent directly to teor, verified using relay contact info
-109.163.234.2:80 orport=443 id=14F92FF956105932E9DEC5B82A7778A0B1BD9A52
-109.163.234.4:80 orport=443 id=4888770464F0E900EFEF1BA181EA873D13F7713C
-109.163.234.5:80 orport=443 id=5EB8D862E70981B8690DEDEF546789E26AB2BD24
-109.163.234.7:80 orport=443 id=23038A7F2845EBA2234ECD6651BD4A7762F51B18
-109.163.234.8:80 orport=443 id=0818DAE0E2DDF795AEDEAC60B15E71901084F281
-109.163.234.9:80 orport=443 id=ABF7FBF389C9A747938B639B20E80620B460B2A9
+109.163.234.5:80 orport=443 id=5C84C35936B7100B949AC75764EEF1352550550B
+109.163.234.7:80 orport=443 id=C46524E586E1B997329703D356C07EE12B28C722
+109.163.234.9:80 orport=443 id=5714542DCBEE1DD9864824723638FD44B2122CEA
+77.247.181.162:80 orport=443 id=7BB160A8F54BD74F3DA5F2CE701E8772B841859D
+109.163.234.4:80 orport=443 id=6B1E001929AF4DDBB747D02EC28340792B7724A6
+77.247.181.164:80 orport=443 id=10E13E340651D0EF66B4DEBF610B3C0981168107
+109.163.234.8:80 orport=443 id=20B0038D7A2FD73C696922551B8344CB0893D1F8
+77.247.181.166:80 orport=443 id=06E123865C590189B3181114F23F0F13A7BC0E69
+109.163.234.2:80 orport=443 id=B4F883DB3D478C7AE569C9F6CB766FD58650DC6A
+62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF
+109.163.234.5:80 orport=443 id=5C84C35936B7100B949AC75764EEF1352550550B
+109.163.234.7:80 orport=443 id=C46524E586E1B997329703D356C07EE12B28C722
+109.163.234.9:80 orport=443 id=5714542DCBEE1DD9864824723638FD44B2122CEA
+77.247.181.162:80 orport=443 id=7BB160A8F54BD74F3DA5F2CE701E8772B841859D
+109.163.234.4:80 orport=443 id=6B1E001929AF4DDBB747D02EC28340792B7724A6
+77.247.181.164:80 orport=443 id=10E13E340651D0EF66B4DEBF610B3C0981168107
+109.163.234.8:80 orport=443 id=20B0038D7A2FD73C696922551B8344CB0893D1F8
+77.247.181.166:80 orport=443 id=06E123865C590189B3181114F23F0F13A7BC0E69
+109.163.234.2:80 orport=443 id=B4F883DB3D478C7AE569C9F6CB766FD58650DC6A
 62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF
-# Assume details update is permanent
-77.247.181.166:80 orport=443 id=77131D7E2EC1CA9B8D737502256DA9103599CE51 # CriticalMass
-77.247.181.164:80 orport=443 id=204DFD2A2C6A0DC1FA0EACB495218E0B661704FD # HaveHeart
-77.247.181.162:80 orport=443 id=7BFB908A3AA5B491DA4CA72CCBEE0E1F2A939B55 # sofia
 
 # https://twitter.com/biotimylated/status/718994247500718080
 212.47.252.149:9030 orport=9001 id=2CAC39BAA996791CEFAADC9D4754D65AF5EB77C0
@@ -199,7 +215,9 @@
 
 # Email sent directly to teor, verified using relay contact info
 86.59.119.88:80 orport=443 id=ACD889D86E02EDDAB1AFD81F598C0936238DC6D0
-86.59.119.83:80 orport=443 id=FC9AC8EA0160D88BCCFDE066940D7DD9FA45495B
+
+# Email sent directly to teor, verified using relay contact info
+144.76.73.140:9030 orport=9001 id=6A640018EABF3DA9BAD9321AA37C2C87BBE1F907
 
 # Email sent directly to teor, verified using relay contact info
 193.11.164.243:9030 orport=9001 id=FFA72BD683BC2FCF988356E6BEC1E490F313FB07 ipv6=[2001:6b0:7:125::243]:9001
@@ -214,19 +232,20 @@
 # Email sent directly to teor, verified using relay contact info
 89.187.142.208:80 orport=443 id=64186650FFE4469EBBE52B644AE543864D32F43C
 
-# Email sent directly to teor
-# Assume details update is permanent
-212.51.134.123:9030 orport=9001 id=50586E25BE067FD1F739998550EDDCB1A14CA5B2 # Jans
+# Email sent directly to teor, verified using relay contact info
+212.51.134.123:9030 orport=9001 id=50586E25BE067FD1F739998550EDDCB1A14CA5B2 ipv6=[2a02:168:6e00:0:3a60:77ff:fe9c:8bd1]:9001
 
 # Email sent directly to teor, verified using relay contact info
 46.101.143.173:80 orport=443 id=F960DF50F0FD4075AC9B505C1D4FFC8384C490FB
 
+# Email sent directly to teor, verified using relay contact info
+217.79.190.25:9030 orport=9090 id=361D33C96D0F161275EE67E2C91EE10B276E778B
+
 # Email sent directly to teor, verified using relay contact info
 193.171.202.146:9030 orport=9001 id=01A9258A46E97FF8B2CAC7910577862C14F2C524
 
 # Email sent directly to teor, verified using relay contact info
-# Assume details update is permanent
-197.231.221.211:9030 orport=443 id=BC630CBBB518BE7E9F4E09712AB0269E9DC7D626 # IPredator
+197.231.221.211:9030 orport=9001 id=BC630CBBB518BE7E9F4E09712AB0269E9DC7D626
 
 # Email sent directly to teor, verified using relay contact info
 185.61.138.18:8080 orport=4443 id=2541759BEC04D37811C2209A88E863320271EC9C
@@ -237,7 +256,7 @@
 193.11.114.46:9032 orport=9003 id=B83DC1558F0D34353BB992EF93AFEAFDB226A73E
 
 # Email sent directly to teor, verified using relay contact info
-138.201.250.33:9012 orport=9011 id=2BA2C8E96B2590E1072AECE2BDB5C48921BF8510
+144.76.26.175:9012 orport=9011 id=2BA2C8E96B2590E1072AECE2BDB5C48921BF8510
 
 # Email sent directly to teor, verified using relay contact info
 37.221.162.226:9030 orport=9001 id=D64366987CB39F61AD21DBCF8142FA0577B92811
@@ -255,12 +274,12 @@
 134.119.3.164:9030 orport=9001 id=D1B8AAA98C65F3DF7D8BB3AF881CAEB84A33D8EE
 
 # Email sent directly to teor, verified using relay contact info
-173.212.254.192:31336 orport=31337 id=99E246DB480B313A3012BC3363093CC26CD209C7
+81.7.10.93:31336 orport=31337 id=99E246DB480B313A3012BC3363093CC26CD209C7
 
 # Email sent directly to teor, verified using relay contact info
 178.62.22.36:80 orport=443 id=A0766C0D3A667A3232C7D569DE94A28F9922FCB1 ipv6=[2a03:b0c0:1:d0::174:1]:9050
-188.166.23.127:80 orport=443 id=8672E8A01B4D3FA4C0BBE21C740D4506302EA487 ipv6=[2a03:b0c0:2:d0::27b:7001]:9050
-198.199.64.217:80 orport=443 id=B1D81825CFD7209BD1B4520B040EF5653C204A23 ipv6=[2604:a880:400:d0::1a9:b001]:9050
+188.166.23.127:80 orport=443 id=3771A8154DEA98D551607806C80A209CDAA74535 ipv6=[2a03:b0c0:2:d0::27b:7001]:9050
+198.199.64.217:80 orport=443 id=FAD306BAA59F6A02783F8606BDAA431F5FF7D1EA ipv6=[2604:a880:400:d0::1a9:b001]:9050
 159.203.32.149:80 orport=443 id=55C7554AFCEC1062DCBAC93E67B2E03C6F330EFC ipv6=[2604:a880:cad:d0::105:f001]:9050
 
 # Email sent directly to teor, verified using relay contact info
@@ -281,6 +300,9 @@
 # Email sent directly to teor, verified using relay contact info
 212.47.230.49:9030 orport=9001 id=3D6D0771E54056AEFC28BB1DE816951F11826E97
 
+# Email sent directly to teor, verified using relay contact info
+176.31.180.157:143 orport=22 id=E781F4EC69671B3F1864AE2753E0890351506329 ipv6=[2001:41d0:8:eb9d::1]:22
+
 # Email sent directly to teor, verified using relay contact info
 192.99.55.69:80 orport=443 id=0682DE15222A4A4A0D67DBA72A8132161992C023
 192.99.59.140:80 orport=443 id=3C9148DA49F20654730FAC83FFF693A4D49D0244
@@ -296,7 +318,7 @@
 151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605 ipv6=[2001:41d0:e:f67::114]:9001
 
 # Email sent directly to teor, verified using relay contact info
-5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6 ipv6=[2001:41d0:8:b1c7::1]:443
+5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6
 
 # Email sent directly to teor, verified using relay contact info
 176.31.159.231:80 orport=443 id=D5DBCC0B4F029F80C7B8D33F20CF7D97F0423BB1
@@ -306,15 +328,17 @@
 46.148.18.74:8080 orport=443 id=6CACF0B5F03C779672F3C5C295F37C8D234CA3F7
 
 # Email sent directly to teor, verified using relay contact info
-37.187.102.108:80 orport=443 id=F4263275CF54A6836EE7BD527B1328836A6F06E1 ipv6=[2001:41d0:a:266c::1]:443 # EvilMoe
-212.47.241.21:80 orport=443 id=892F941915F6A0C6E0958E52E0A9685C190CF45C # EvilMoe
+37.187.102.108:9090 orport=5550 id=F4263275CF54A6836EE7BD527B1328836A6F06E1
+212.47.241.21:80 orport=443 id=892F941915F6A0C6E0958E52E0A9685C190CF45C
 
 # Email sent directly to teor, verified using relay contact info
-212.129.38.254:9030 orport=9001 id=FDF845FC159C0020E2BDDA120C30C5C5038F74B4
+195.191.233.221:80 orport=443 id=DE134FC8E5CC4EC8A5DE66934E70AC9D70267197
 
-# Email sent directly to teor
-37.157.195.87:8030 orport=443 id=12FD624EE73CEF37137C90D38B2406A66F68FAA2 # thanatosCZ
-5.189.169.190:8030 orport=8080 id=8D79F73DCD91FC4F5017422FAC70074D6DB8DD81 # thanatosDE
+# Email sent directly to teor, verified using relay contact info
+62.210.238.33:9030 orport=9001 id=FDF845FC159C0020E2BDDA120C30C5C5038F74B4
+
+# Email sent directly to teor, verified using relay contact info
+37.157.195.87:8030 orport=443 id=12FD624EE73CEF37137C90D38B2406A66F68FAA2
 
 # Email sent directly to teor, verified using relay contact info
 37.187.7.74:80 orport=443 id=AEA43CB1E47BE5F8051711B2BF01683DB1568E05 ipv6=[2001:41d0:a:74a::1]:443
@@ -331,6 +355,9 @@
 # Email sent directly to teor, verified using relay contact info
 37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001
 
+# Email sent directly to teor, verified using relay contact info
+5.35.251.247:9030 orport=9001 id=9B1F5187DFBA89DC24B37EA7BF896C12B43A27AE
+
 # Email sent directly to teor, verified using relay contact info
 198.96.155.3:8080 orport=5001 id=BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E
 
@@ -377,6 +404,14 @@
 # Email sent directly to teor, verified using relay contact info
 91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7
 
+# Email sent directly to teor, verified using relay contact info
+# Suitable, check with operator before adding
+#212.47.240.10:82 orport=443 id=2A4C448784F5A83AFE6C78DA357D5E31F7989DEB
+212.47.240.10:81 orport=993 id=72527E3242CB15AADE28374AE0D35833FC083F60
+163.172.131.88:80 orport=443 id=AD253B49E303C6AB1E048B014392AC569E8A7DAE ipv6=[2001:bc8:4400:2100::2:1009]:443
+# Suitable, check with operator before adding
+#163.172.131.88:81 orport=993 id=D5F3FB17504744FB7ECEF46F4B1D155258A6D942 ipv6=D5F3FB17504744FB7ECEF46F4B1D155258A6D942
+
 # Email sent directly to teor, verified using relay contact info
 46.101.151.222:80 orport=443 id=1DBAED235E3957DE1ABD25B4206BE71406FB61F8
 178.62.60.37:80 orport=443 id=175921396C7C426309AB03775A9930B6F611F794
@@ -405,8 +440,11 @@
 # Email sent directly to teor, verified using relay contact info
 5.199.142.236:9030 orport=9001 id=F4C0EDAA0BF0F7EC138746F8FEF1CE26C7860265
 
-# Email sent directly to teor
-188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::26c0:1]:9001 # dropsy
+# Email sent directly to teor, verified using relay contact info
+188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::5:f001]:9001
+
+# Email sent directly to teor, verified using relay contact info
+5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484
 
 # Email sent directly to teor, verified using relay contact info
 46.8.249.10:80 orport=443 id=31670150090A7C3513CB7914B9610E786391A95D
@@ -418,10 +456,12 @@
 46.4.24.161:9030 orport=9001 id=DB4C76A3AD7E234DA0F00D6F1405D8AFDF4D8DED
 46.4.24.161:9031 orport=9002 id=7460F3D12EBE861E4EE073F6233047AACFE46AB4
 46.38.51.132:9030 orport=9001 id=810DEFA7E90B6C6C383C063028EC397A71D7214A
-163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451 ipv6=[2001:bc8:225f:142:6c69:7461:7669:73]:9001
+163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451
 
 # Email sent directly to teor, verified using relay contact info
 176.10.107.180:9030 orport=9001 id=3D7E274A87D9A89AF064C13D1EE4CA1F184F2600
+195.154.75.84:9030 orport=9001 id=F80FDE27EFCB3F6A7B4E2CC517133DBFFA78BA2D
+195.154.127.246:9030 orport=9001 id=4FEE77AFFD157BBCF2D896AE417FBF647860466C
 
 # Email sent directly to teor, verified using relay contact info
 46.28.207.19:80 orport=443 id=5B92FA5C8A49D46D235735504C72DBB3472BA321
@@ -445,10 +485,11 @@
 5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11
 
 # Email sent directly to teor, verified using relay contact info
-# Updated details from atlas based on ticket #20010
-163.172.176.167:80 orport=443 id=230A8B2A8BA861210D9B4BA97745AEC217A94207
-163.172.149.155:80 orport=443 id=0B85617241252517E8ECF2CFC7F4C1A32DCD153F
-163.172.149.122:80 orport=443 id=A9406A006D6E7B5DA30F2C6D4E42A338B5E340B2
+167.114.152.100:9030 orport=443 id=0EF5E5FFC5D1EABCBDA1AFF6F6D6325C5756B0B2 ipv6=[2607:5300:100:200::1608]:443
+
+# Email sent directly to teor, verified using relay contact info
+192.99.168.102:80 orport=443 id=230A8B2A8BA861210D9B4BA97745AEC217A94207
+167.114.153.21:80 orport=443 id=0B85617241252517E8ECF2CFC7F4C1A32DCD153F
 
 # Email sent directly to teor, verified using relay contact info
 204.11.50.131:9030 orport=9001 id=185F2A57B0C4620582602761097D17DB81654F70
@@ -456,11 +497,14 @@
 # Email sent directly to teor, verified using relay contact info
 151.236.222.217:44607 orport=9001 id=94D58704C2589C130C9C39ED148BD8EA468DBA54
 
+# Email sent directly to teor, verified using relay contact info
+194.150.168.79:11112 orport=11111 id=29F1020B94BE25E6BE1AD13E93CE19D2131B487C
+
 # Email sent directly to teor, verified using relay contact info
 185.35.202.221:9030 orport=9001 id=C13B91384CDD52A871E3ECECE4EF74A7AC7DCB08 ipv6=[2a02:ed06::221]:9001
 
 # Email sent directly to teor, verified using relay contact info
-5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A ipv6=[2a01:4f8:190:34f0::2]:4223
+5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A
 
 # Email sent directly to teor, verified using relay contact info
 89.40.71.149:8081 orport=8080 id=EC639EDAA5121B47DBDF3D6B01A22E48A8CB6CC7
@@ -469,7 +513,7 @@
 92.222.20.130:80 orport=443 id=0639612FF149AA19DF3BCEA147E5B8FED6F3C87C
 
 # Email sent directly to teor, verified using relay contact info
-80.112.155.100:9030 orport=9001 id=53B000310984CD86AF47E5F3CD0BFF184E34B383 ipv6=[2001:470:7b02::38]:9001
+80.112.155.100:9030 orport=9001 id=1163378F239C36CA1BDC730AC50BF4F2976141F5 ipv6=[2001:470:7b02::38]:9001
 
 # Email sent directly to teor, verified using relay contact info
 83.212.99.68:80 orport=443 id=DDBB2A38252ADDA53E4492DDF982CA6CC6E10EC0 ipv6=[2001:648:2ffc:1225:a800:bff:fe3d:67b5]:443
@@ -477,6 +521,9 @@
 # Email sent directly to teor, verified using relay contact info
 95.130.11.147:9030 orport=443 id=6B697F3FF04C26123466A5C0E5D1F8D91925967A
 
+# Email sent directly to teor, verified using relay contact info
+176.31.191.26:9030 orport=9001 id=7350AB9ED7568F22745198359373C04AC783C37C
+
 # Email sent directly to teor, verified using relay contact info
 128.199.55.207:9030 orport=9001 id=BCEF908195805E03E92CCFE669C48738E556B9C5 ipv6=[2a03:b0c0:2:d0::158:3001]:9001
 
@@ -493,17 +540,16 @@
 80.240.139.111:80 orport=443 id=DD3BE7382C221F31723C7B294310EF9282B9111B
 
 # Email sent directly to teor, verified using relay contact info
-185.97.32.18:9030 orport=9001 id=04250C3835019B26AA6764E85D836088BE441088
-
-# Email sent directly to teor
-149.56.45.200:9030 orport=9001 id=FE296180018833AF03A8EACD5894A614623D3F76 ipv6=[2607:5300:201:3000::17d3]:9002 # PiotrTorpotkinOne
+185.97.32.18:9030 orport=9001 id=3BAB316CAAEC47E71905EB6C65584636D5689A8A
 
 # Email sent directly to teor, verified using relay contact info
-81.2.209.10:443 orport=80 id=B6904ADD4C0D10CDA7179E051962350A69A63243 ipv6=[2001:15e8:201:1::d10a]:80
+149.56.45.200:9030 orport=9001 id=FE296180018833AF03A8EACD5894A614623D3F76
 
 # Email sent directly to teor, verified using relay contact info
-# IPv6 address unreliable
-195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C #ipv6=[2001:bc8:399f:f000::1]:993
+81.2.209.10:443 orport=80 id=B6904ADD4C0D10CDA7179E051962350A69A63243
+
+# Email sent directly to teor, verified using relay contact info
+195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C ipv6=[2001:bc8:399f:f000::1]:993
 138.201.26.2:80 orport=443 id=6D3A3ED5671E4E3F58D4951438B10AE552A5FA0F
 81.7.16.182:80 orport=443 id=51E1CF613FD6F9F11FE24743C91D6F9981807D82 ipv6=[2a02:180:1:1::517:10b6]:993
 134.119.36.135:80 orport=443 id=763C9556602BD6207771A7A3D958091D44C43228 ipv6=[2a00:1158:3::2a8]:993
@@ -517,7 +563,7 @@
 217.12.208.117:80 orport=443 id=E6E18151300F90C235D3809F90B31330737CEB43 ipv6=[2a00:1ca8:a7::1bb]:993
 81.7.10.251:80 orport=443 id=8073670F8F852971298F8AF2C5B23AE012645901 ipv6=[2a02:180:1:1::517:afb]:993
 46.36.39.50:80 orport=443 id=ED4B0DBA79AEF5521564FA0231455DCFDDE73BB6 ipv6=[2a02:25b0:aaaa:aaaa:8d49:b692:4852:0]:995
-91.194.90.103:80 orport=443 id=75C4495F4D80522CA6F6A3FB349F1B009563F4B7 ipv6=[2a02:c205:3000:5449::1]:993
+91.194.90.103:80 orport=443 id=75C4495F4D80522CA6F6A3FB349F1B009563F4B7 ipv6=[2a02:c200:0:10:3:0:5449:1]:993
 163.172.25.118:80 orport=22 id=0CF8F3E6590F45D50B70F2F7DA6605ECA6CD408F
 188.138.88.42:80 orport=443 id=70C55A114C0EF3DC5784A4FAEE64388434A3398F
 81.7.13.84:80 orport=443 id=0C1E7DD9ED0676C788933F68A9985ED853CA5812 ipv6=[2a02:180:1:1::5b8f:538c]:993
@@ -534,17 +580,18 @@
 # Email sent directly to teor, verified using relay contact info
 212.238.208.48:9030 orport=9001 id=F406219CDD339026D160E53FCA0EF6857C70F109 ipv6=[2001:984:a8fb:1:ba27:ebff:feac:c109]:9001
 
-# Email sent directly to teor
-176.158.236.102:9030 orport=9001 id=DC163DDEF4B6F0C6BC226F9F6656A5A30C5C5686 # Underworld
+# Email sent directly to teor, verified using relay contact info
+176.158.132.12:9030 orport=9001 id=DC163DDEF4B6F0C6BC226F9F6656A5A30C5C5686
 
 # Email sent directly to teor, verified using relay contact info
 91.229.20.27:9030 orport=9001 id=9A0D54D3A6D2E0767596BF1515E6162A75B3293F
 
 # Email sent directly to teor, verified using relay contact info
+# Awaiting confirmation of new ORPort from relay operator
 80.127.137.19:80 orport=443 id=6EF897645B79B6CB35E853B32506375014DE3621 ipv6=[2001:981:47c1:1::6]:443
 
-# Email sent directly to teor
-163.172.138.22:80 orport=443 id=16102E458460349EE45C0901DAA6C30094A9BBEA ipv6=[2001:bc8:4400:2100::1:3]:443 # mkultra
+# Email sent directly to teor, verified using relay contact info
+163.172.138.22:80 orport=443 id=8664DC892540F3C789DB37008236C096C871734D
 
 # Email sent directly to teor, verified using relay contact info
 97.74.237.196:9030 orport=9001 id=2F0F32AB1E5B943CA7D062C03F18960C86E70D94
@@ -556,11 +603,10 @@
 178.62.98.160:9030 orport=9001 id=8B92044763E880996A988831B15B2B0E5AD1544A
 
 # Email sent directly to teor, verified using relay contact info
-163.172.217.50:9030 orport=9001 id=02ECD99ECD596013A8134D46531560816ECC4BE6
+195.154.15.227:9030 orport=9001 id=6C3E3AB2F5F03CD71B637D433BAD924A1ECC5796
 
 # Email sent directly to teor, verified using relay contact info
 185.100.86.100:80 orport=443 id=0E8C0C8315B66DB5F703804B3889A1DD66C67CE0
-185.100.84.82:80 orport=443 id=7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E
 
 # Email sent directly to teor, verified using relay contact info
 164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B
@@ -571,15 +617,13 @@
 178.62.86.96:9030 orport=9001 id=439D0447772CB107B886F7782DBC201FA26B92D1 ipv6=[2a03:b0c0:1:d0::3cf:7001]:9050
 
 # Email sent directly to teor, verified using relay contact info
-# Very low bandwidth, stale consensues, excluded to cut down on warnings
-#91.233.106.121:80 orport=443 id=896364B7996F5DFBA0E15D1A2E06D0B98B555DD6
+91.233.106.121:80 orport=443 id=896364B7996F5DFBA0E15D1A2E06D0B98B555DD6
 
 # Email sent directly to teor, verified using relay contact info
-167.114.113.48:9030 orport=403 id=2EC0C66EA700C44670444280AABAB1EC78B722A0
+167.114.113.48:9030 orport=443 id=2EC0C66EA700C44670444280AABAB1EC78B722A0
 
 # Email sent directly to teor, verified using relay contact info
-# Assume details update is permanent
-213.141.138.174:9030 orport=9001 id=BD552C165E2ED2887D3F1CCE9CFF155DDA2D86E6 # Schakalium
+79.120.16.42:9030 orport=9001 id=BD552C165E2ED2887D3F1CCE9CFF155DDA2D86E6
 
 # Email sent directly to teor, verified using relay contact info
 95.128.43.164:80 orport=443 id=616081EC829593AF4232550DE6FFAA1D75B37A90 ipv6=[2a02:ec0:209:10::4]:443
@@ -608,13 +652,10 @@
 # Email sent directly to teor, verified using relay contact info
 31.31.78.49:80 orport=443 id=46791D156C9B6C255C2665D4D8393EC7DBAA7798
 
-# Email sent directly to teor
-192.160.102.169:80 orport=9001 id=C0192FF43E777250084175F4E59AC1BA2290CE38 ipv6=[2620:132:300c:c01d::9]:9002 # manipogo
-192.160.102.166:80 orport=9001 id=547DA56F6B88B6C596B3E3086803CDA4F0EF8F21 ipv6=[2620:132:300c:c01d::6]:9002 # chaucer
-192.160.102.170:80 orport=9001 id=557ACEC850F54EEE65839F83CACE2B0825BE811E ipv6=[2620:132:300c:c01d::a]:9002 # ogopogo
-192.160.102.164:80 orport=9001 id=823AA81E277F366505545522CEDC2F529CE4DC3F ipv6=[2620:132:300c:c01d::4]:9002 # snowfall
-192.160.102.165:80 orport=9001 id=C90CA3B7FE01A146B8268D56977DC4A2C024B9EA ipv6=[2620:132:300c:c01d::5]:9002 # cowcat
-192.160.102.168:80 orport=9001 id=F6A358DD367B3282D6EF5824C9D45E1A19C7E815 ipv6=[2620:132:300c:c01d::8]:9002 # prawksi
+# Email sent directly to teor, verified using relay contact info
+96.47.231.214:9030 orport=8080 id=F843CB5729575D76FF1FFBB2179BDCF52C0C6387
+192.99.246.48:9030 orport=9001 id=CD6B149BED1BB254EF6DFF9D75DDB11E7F8A38A4 ipv6=[2607:5300:100:200::de3]:9002
+192.160.102.164:80 orport=9001 id=823AA81E277F366505545522CEDC2F529CE4DC3F ipv6=[2605:e200:d00c:c01d::1111]:9002
 
 # Email sent directly to teor, verified using relay contact info
 136.243.214.137:80 orport=443 id=B291D30517D23299AD7CEE3E60DFE60D0E3A4664
@@ -625,14 +666,17 @@
 
 # Email sent directly to teor, verified using relay contact info
 192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024
-# same machine as ED2338CAC2711B3E331392E1ED2831219B794024
-192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7
+# OK, but same machine as ED2338CAC2711B3E331392E1ED2831219B794024
+#192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7
 
 # https://twitter.com/kosjoli/status/719507270904758272
 85.10.202.87:9030 orport=9001 id=971AFB23C168DCD8EDA17473C1C452B359DE3A5A
 176.9.5.116:9030 orport=9001 id=A1EB8D8F1EE28DB98BBB1EAA3B4BEDD303BAB911
 46.4.111.124:9030 orport=9001 id=D9065F9E57899B3D272AA212317AF61A9B14D204
 
+# Email sent directly to teor, verified using relay contact info
+78.46.164.129:9030 orport=9001 id=52AEA31188331F421B2EDB494DB65CD181E5B257
+
 # Email sent directly to teor, verified using relay contact info
 185.100.85.61:80 orport=443 id=025B66CEBC070FCB0519D206CF0CF4965C20C96E
 
@@ -640,12 +684,11 @@
 108.166.168.158:80 orport=443 id=CDAB3AE06A8C9C6BF817B3B0F1877A4B91465699
 
 # Email sent directly to teor, verified using relay contact info
-91.219.236.222:80 orport=443 id=20704E7DD51501DC303FA51B738D7B7E61397CF6
+91.219.236.222:80 orport=443 id=EC413181CEB1C8EDC17608BBB177CD5FD8535E99
 
 # Email sent directly to teor, verified using relay contact info
 185.14.185.240:9030 orport=443 id=D62FB817B0288085FAC38A6DC8B36DCD85B70260
 192.34.63.137:9030 orport=443 id=ABCB4965F1FEE193602B50A365425105C889D3F8
-128.199.197.16:9030 orport=443 id=DEE5298B3BA18CDE651421CD2DCB34A4A69F224D
 
 # Email sent directly to teor, verified using relay contact info
 185.13.38.75:9030 orport=9001 id=D2A1703758A0FBBA026988B92C2F88BAB59F9361
@@ -676,14 +719,13 @@
 166.70.207.2:9030 orport=9001 id=E3DB2E354B883B59E8DC56B3E7A353DDFD457812
 
 # Emails sent directly to teor, verified using relay contact info
-69.162.139.9:9030 orport=9001 id=4791FC0692EAB60DF2BCCAFF940B95B74E7654F6 ipv6=[2607:f128:40:1212::45a2:8b09]:9001
+#69.162.139.9:9030 orport=9001 id=4791FC0692EAB60DF2BCCAFF940B95B74E7654F6 ipv6=[2607:f128:40:1212::45a2:8b09]:9001
 
 # Email sent directly to teor, verified using relay contact info
 213.239.217.18:1338 orport=1337 id=C37BC191AC389179674578C3E6944E925FE186C2 ipv6=[2a01:4f8:a0:746a:101:1:1:1]:1337
 
 # Email sent directly to teor, verified using relay contact info
-# Assume details update is permanent
-188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601 ipv6=[2a01:4f8:221:1ac1:dead:beef:7005:9001]:9001 # sputnik
+188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601
 
 # Email sent directly to teor, verified using relay contact info
 88.198.253.13:9030 orport=9001 id=DF924196D69AAE3C00C115A9CCDF7BB62A175310 ipv6=[2a01:4f8:11a:b1f::2]:9001
@@ -701,44 +743,20 @@
 # Email sent directly to teor, verified using relay contact info
 107.170.101.39:9030 orport=443 id=30973217E70AF00EBE51797FF6D9AA720A902EAA
 
-# Email sent directly to teor
-193.70.112.165:80 orport=443 id=F10BDE279AE71515DDCCCC61DC19AC8765F8A3CC # ParkBenchInd001
+# Email sent directly to teor, verified using relay contact info
+192.99.212.139:80 orport=443 id=F10BDE279AE71515DDCCCC61DC19AC8765F8A3CC
 
-# Email sent directly to teor
-185.220.101.6:10006 orport=20006 id=C08DE49658E5B3CFC6F2A952B453C4B608C9A16A # niftyvolcanorabbit
-185.220.101.13:10013 orport=20013 id=71AB4726D830FAE776D74AEF790CF04D8E0151B4 # niftycottontail
-185.220.101.5:10005 orport=20005 id=1084200B44021D308EA4253F256794671B1D099A # niftyhedgehog
-185.220.101.9:10009 orport=20009 id=14877C6384A9E793F422C8D1DDA447CACA4F7C4B # niftywoodmouse
-185.220.101.8:10008 orport=20008 id=24E91955D969AEA1D80413C64FE106FAE7FD2EA9 # niftymouse
-185.220.101.1:10001 orport=20001 id=28F4F392F8F19E3FBDE09616D9DB8143A1E2DDD3 # niftycottonmouse
-185.220.101.21:10021 orport=20021 id=348B89013EDDD99E4755951D1EC284D9FED71226 # niftysquirrel
-185.220.101.10:10010 orport=20010 id=4031460683AE9E0512D3620C2758D98758AC6C93 # niftyeuropeanrabbit
-185.220.101.34:10034 orport=20034 id=47C42E2094EE482E7C9B586B10BABFB67557030B # niftyquokka
-185.220.101.18:10018 orport=20018 id=5D5006E4992F2F97DF4F8B926C3688870EB52BD8 # niftyplagiodontia
-185.220.101.28:10028 orport=20028 id=609E598FB6A00BCF7872906B602B705B64541C50 # niftychipmunk
-185.220.101.20:10020 orport=20020 id=619349D82424C601CAEB94161A4CF778993DAEE7 # niftytucotuco
-185.220.101.17:10017 orport=20017 id=644DECC5A1879C0FE23DE927DD7049F58BBDF349 # niftyhutia
-185.220.101.0:10000 orport=20000 id=6E94866ED8CA098BACDFD36D4E8E2B459B8A734E # niftybeaver
-185.220.101.30:10030 orport=20030 id=71CFDEB4D9E00CCC3E31EC4E8A29E109BBC1FB36 # niftypedetidae
-185.220.101.29:10029 orport=20029 id=7DC52AE6667A30536BA2383CD102CFC24F20AD71 # niftyllipika
-185.220.101.41:10041 orport=20041 id=7E281CD2C315C4F7A84BC7C8721C3BC974DDBFA3 # niftyporcupine
-185.220.101.25:10025 orport=20025 id=8EE0534532EA31AA5172B1892F53B2F25C76EB02 # niftyjerboa
-185.220.101.33:10033 orport=20033 id=906DCB390F2BA987AE258D745E60BAAABAD31DE8 # niftyquokka
-185.220.101.26:10026 orport=20026 id=92A6085EABAADD928B6F8E871540A1A41CBC08BA # niftypedetes
-185.220.101.40:10040 orport=20040 id=9A857254F379194D1CD76F4A79A20D2051BEDA3F # niftynutria
-185.220.101.42:10042 orport=20042 id=9B816A5B3EB20B8E4E9B9D1FBA299BD3F40F0320 # niftypygmyjerboa
-185.220.101.2:10002 orport=20002 id=B740BCECC4A9569232CDD45C0E1330BA0D030D33 # niftybunny
-185.220.101.32:10032 orport=20032 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24 # niftypika
-185.220.101.12:10012 orport=20012 id=BC82F2190DE2E97DE65F49B4A95572374BDC0789 # niftycapybara
-185.220.101.22:10022 orport=20022 id=CA37CD46799449D83B6B98B8C22C649906307888 # niftyjackrabbit
-185.220.101.4:10004 orport=20004 id=CDA2EA326E2272C57ACB26773D7252C211795B78 # niftygerbil
-185.220.101.14:10014 orport=20014 id=E7EBA5D8A4E09684D11A1DF24F75362817333768 # niftyhare
-185.220.101.16:10016 orport=20016 id=EC1997D51892E4607C68E800549A1E7E4694005A # niftyguineapig
-185.220.101.24:10024 orport=20024 id=FDA70EC93DB01E3CB418CB6943B0C68464B18B4C # niftyrat
+# Email sent directly to teor, verified using relay contact info
+163.172.35.249:80 orport=443 id=C08DE49658E5B3CFC6F2A952B453C4B608C9A16A
+163.172.35.247:80 orport=443 id=71AB4726D830FAE776D74AEF790CF04D8E0151B4
+163.172.13.124:80 orport=443 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24
 
 # Email sent directly to teor, verified using relay contact info
 64.113.32.29:9030 orport=9001 id=30C19B81981F450C402306E2E7CFB6C3F79CB6B2
 
+# Email sent directly to teor, verified using relay contact info
+212.51.156.193:995 orport=110 id=32E7AAF1F602814D699BEF6761AD03E387758D49 ipv6=[2a02:168:4a01::49]:110
+
 # Emails sent directly to teor, verified using relay contact info
 51.254.101.242:9002 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF
 
@@ -750,247 +768,3 @@
 
 # Email sent directly to teor, verified using relay contact info
 62.216.5.120:9030 orport=9001 id=D032D4D617140D6B828FC7C4334860E45E414FBE
-
-# Email sent directly to teor, verified using relay contact info
-51.254.136.195:80 orport=443 id=7BB70F8585DFC27E75D692970C0EEB0F22983A63
-
-# Email sent directly to teor, verified using relay contact info
-163.172.13.165:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB ipv6=[2001:bc8:38cb:201::8]:9001
-
-# Email sent directly to teor, verified using relay contact info
-5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484 ipv6=[2001:41d0:a:fb7a::1]:9001
-
-# Email sent directly to teor, verified using relay contact info
-5.9.158.75:80 orport=443 id=1AF72E8906E6C49481A791A6F8F84F8DFEBBB2BA ipv6=[2a01:4f8:190:514a::2]:443
-
-# Email sent directly to teor, verified using relay contact info
-46.101.169.151:9030 orport=9001 id=D760C5B436E42F93D77EF2D969157EEA14F9B39C ipv6=[2a03:b0c0:3:d0::74f:a001]:9001
-
-# Email sent directly to teor, verified using relay contact info
-199.249.223.81:80 orport=443 id=F7447E99EB5CBD4D5EB913EE0E35AC642B5C1EF3
-199.249.223.79:80 orport=443 id=D33292FEDE24DD40F2385283E55C87F85C0943B6
-199.249.223.78:80 orport=443 id=EC15DB62D9101481F364DE52EB8313C838BDDC29
-199.249.223.77:80 orport=443 id=CC4A3AE960E3617F49BF9887B79186C14CBA6813
-199.249.223.76:80 orport=443 id=43209F6D50C657A56FE79AF01CA69F9EF19BD338
-199.249.223.75:80 orport=443 id=60D3667F56AEC5C69CF7E8F557DB21DDF6C36060
-199.249.223.74:80 orport=443 id=5F4CD12099AF20FAF9ADFDCEC65316A376D0201C
-199.249.223.73:80 orport=443 id=5649CB2158DA94FB747415F26628BEC07FA57616
-199.249.223.72:80 orport=443 id=B028707969D8ED84E6DEA597A884F78AAD471971
-199.249.223.71:80 orport=443 id=B6320E44A230302C7BF9319E67597A9B87882241
-199.249.223.60:80 orport=443 id=B7047FBDE9C53C39011CA84E5CB2A8E3543066D0
-199.249.223.61:80 orport=443 id=40E7D6CE5085E4CDDA31D51A29D1457EB53F12AD
-199.249.223.62:80 orport=443 id=0077BCBA7244DB3E6A5ED2746E86170066684887
-199.249.223.63:80 orport=443 id=1DB25DF59DAA01B5BE3D3CEB8AFED115940EBE8B
-199.249.223.64:80 orport=443 id=9F2856F6D2B89AD4EF6D5723FAB167DB5A53519A
-199.249.223.65:80 orport=443 id=9D21F034C3BFF4E7737D08CF775DC1745706801F
-199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2
-199.249.223.67:80 orport=443 id=155D6F57425F16C0624D77777641E4EB1B47C6F0
-199.249.223.68:80 orport=443 id=DF20497E487A979995D851A5BCEC313DF7E5BC51
-199.249.223.69:80 orport=443 id=7FA8E7E44F1392A4E40FFC3B69DB3B00091B7FD3
-
-# https://lists.torproject.org/pipermail/tor-relays/2016-December/011114.html
-86.105.212.130:9030 orport=443 id=9C900A7F6F5DD034CFFD192DAEC9CCAA813DB022
-
-# Email sent directly to teor, verified using relay contact info
-178.33.183.251:80 orport=443 id=DD823AFB415380A802DCAEB9461AE637604107FB ipv6=[2001:41d0:2:a683::251]:443
-
-# Email sent directly to teor, verified using relay contact info
-31.185.104.19:80 orport=443 id=9EAD5B2D3DBD96DBC80DCE423B0C345E920A758D
-# same machine as 9EAD5B2D3DBD96DBC80DCE423B0C345E920A758D
-31.185.104.20:80 orport=443 id=ADB2C26629643DBB9F8FE0096E7D16F9414B4F8D
-31.185.104.21:80 orport=443 id=C2AAB088555850FC434E68943F551072042B85F1
-31.185.104.22:80 orport=443 id=5BA3A52760A0EABF7E7C3ED3048A77328FF0F148
-
-# Email sent directly to teor, verified using relay contact info
-185.34.60.114:80 orport=443 id=7F7A695DF6F2B8640A70B6ADD01105BC2EBC5135
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013939.html
-94.142.242.84:80 orport=443 id=AA0D167E03E298F9A8CD50F448B81FBD7FA80D56 ipv6=[2a02:898:24:84::1]:443 # rejozenger
-
-# Email sent directly to teor, verified using relay contact info
-185.129.62.62:9030 orport=9001 id=ACDD9E85A05B127BA010466C13C8C47212E8A38F ipv6=[2a06:d380:0:3700::62]:9001
-
-# Email sent directly to teor, verified using relay contact info
-# The e84 part of the IPv6 address does not have a leading 0 in the consensus
-81.30.158.213:9030 orport=9001 id=789EA6C9AE9ADDD8760903171CFA9AC5741B0C70 ipv6=[2001:4ba0:cafe:e84::1]:9001
-
-# https://lists.torproject.org/pipermail/tor-relays/2016-December/011209.html
-5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797
-
-# Email sent directly to teor, verified using relay contact info
-45.62.255.25:80 orport=443 id=3473ED788D9E63361D1572B7E82EC54338953D2A
-
-# Email sent directly to teor, verified using relay contact info
-217.79.179.177:9030 orport=9001 id=3E53D3979DB07EFD736661C934A1DED14127B684 ipv6=[2001:4ba0:fff9:131:6c4f::90d3]:9001
-
-# Email sent directly to teor, verified using relay contact info
-212.47.244.38:8080 orport=443 id=E81EF60A73B3809F8964F73766B01BAA0A171E20
-163.172.157.213:8080 orport=443 id=4623A9EC53BFD83155929E56D6F7B55B5E718C24
-163.172.139.104:8080 orport=443 id=68F175CCABE727AA2D2309BCD8789499CEE36ED7
-
-# Email sent directly to teor, verified using relay contact info
-163.172.223.200:80 orport=443 id=998BF3ED7F70E33D1C307247B9626D9E7573C438
-195.154.122.54:80 orport=443 id=64E99CB34C595A02A3165484BD1215E7389322C6
-
-# Email sent directly to teor, verified using relay contact info
-185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883
-185.100.85.101:9030 orport=9001 id=4061C553CA88021B8302F0814365070AAE617270
-31.171.155.108:9030 orport=9001 id=D3E5EDDBE5159388704D6785BE51930AAFACEC6F
-
-# Email sent directly to teor, verified using relay contact info
-89.163.247.43:9030 orport=9001 id=BC7ACFAC04854C77167C7D66B7E471314ED8C410 ipv6=[2001:4ba0:fff7:25::5]:9001
-
-# Email sent directly to teor, verified using relay contact info
-95.85.8.226:80 orport=443 id=1211AC1BBB8A1AF7CBA86BCE8689AA3146B86423
-
-# Email sent directly to teor, verified using relay contact info
-85.214.151.72:9030 orport=9001 id=722D365140C8C52DBB3C9FF6986E3CEFFE2BA812
-
-# email sent directly to teor
-72.52.75.27:9030 orport=9001 id=8567AD0A6369ED08527A8A8533A5162AC00F7678 # piecoopdotnet
-
-# Email sent directly to teor, verified using relay contact info
-5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11
-5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797
-
-# Email sent directly to teor, verified using relay contact info
-# Assume details update is permanent
-5.9.147.226:9030 orport=9001 id=B0553175AADB0501E5A61FC61CEA3970BE130FF2 ipv6=[2a01:4f8:190:30e1::2]:9001 # zwiubel
-
-# https://trac.torproject.org/projects/tor/ticket/22527#comment:1
-199.184.246.250:80 orport=443 id=1F6ABD086F40B890A33C93CC4606EE68B31C9556 ipv6=[2620:124:1009:1::171]:443
-
-# https://trac.torproject.org/projects/tor/ticket/24695
-163.172.53.84:143 orport=21 id=1C90D3AEADFF3BCD079810632C8B85637924A58E ipv6=[2001:bc8:24f8::]:21 # Multivac
-
-# Email sent directly to teor
-54.36.237.163:80 orport=443 id=DB2682153AC0CCAECD2BD1E9EBE99C6815807A1E # GermanCraft2
-
-# Email sent directly to teor
-62.138.7.171:9030 orport=9001 id=9844B981A80B3E4B50897098E2D65167E6AEF127 # 0x3d004
-62.138.7.171:8030 orport=8001 id=9285B22F7953D7874604EEE2B470609AD81C74E9 # 0x3d005
-91.121.23.100:9030 orport=9001 id=3711E80B5B04494C971FB0459D4209AB7F2EA799 # 0x3d002
-91.121.23.100:8030 orport=8001 id=CFBBA0D858F02E40B1432A65F6D13C9BDFE7A46B # 0x3d001
-51.15.13.245:9030 orport=9001 id=CED527EAC230E7B56E5B363F839671829C3BA01B # 0x3d006
-51.15.13.245:8030 orport=8001 id=8EBB8D1CF48FE2AB95C451DA8F10DB6235F40F8A # 0x3d007
-
-# Email sent directly to teor
-104.192.5.248:9030 orport=9001 id=BF735F669481EE1CCC348F0731551C933D1E2278 # Freeway11
-
-# Email sent directly to teor
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013961.html
-178.17.174.14:9030 orport=9001 id=B06F093A3D4DFAD3E923F4F28A74901BD4F74EB1 # TorExitMoldova
-178.17.170.156:9030 orport=9001 id=41C59606AFE1D1AA6EC6EF6719690B856F0B6587 # TorExitMoldova2
-
-# Email sent directly to teor
-163.172.221.44:59030 orport=59001 id=164604F5C86FC8CC9C0288BD9C02311958427597 # altego
-
-# Email sent directly to teor
-46.38.237.221:9030 orport=9001 id=D30E9D4D639068611D6D96861C95C2099140B805 # mine
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013911.html
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013912.html
-199.249.223.62:80 orport=443 id=0077BCBA7244DB3E6A5ED2746E86170066684887 # Quintex13
-199.249.224.45:80 orport=443 id=041646640AB306EA74B001966E86169B04CC88D2 # QuintexAirVPN26
-199.249.223.67:80 orport=443 id=155D6F57425F16C0624D77777641E4EB1B47C6F0 # Quintex18
-199.249.223.45:80 orport=443 id=1AE949967F82BBE7534A3D6BA77A7EBE1CED4369 # Quintex36
-199.249.223.63:80 orport=443 id=1DB25DF59DAA01B5BE3D3CEB8AFED115940EBE8B # Quintex14
-199.249.224.63:80 orport=443 id=1E5136DDC52FAE1219208F0A6BADB0BA62587EE6 # Quintex43
-199.249.224.46:80 orport=443 id=2ED4D25766973713EB8C56A290BF07E06B85BF12 # QuintexAirVPN27
-199.249.223.42:80 orport=443 id=3687FEC7E73F61AC66F7AE251E7DEE6BBD8C0252 # Quintex33
-199.249.223.49:80 orport=443 id=36D68478366CB8627866757EBCE7FB3C17FC1CB8 # Quintex40
-199.249.224.49:80 orport=443 id=3CA0D15567024D2E0B557DC0CF3E962B37999A79 # QuintexAirVPN30
-199.249.223.61:80 orport=443 id=40E7D6CE5085E4CDDA31D51A29D1457EB53F12AD # Quintex12
-199.249.223.76:80 orport=443 id=43209F6D50C657A56FE79AF01CA69F9EF19BD338 # QuintexAirVPN5
-199.249.224.41:80 orport=443 id=54A4820B46E65509BF3E2B892E66930A41759DE9 # QuintexAirVPN22
-199.249.223.73:80 orport=443 id=5649CB2158DA94FB747415F26628BEC07FA57616 # QuintexAirVPN8
-199.249.223.74:80 orport=443 id=5F4CD12099AF20FAF9ADFDCEC65316A376D0201C # QuintexAirVPN7
-199.249.223.75:80 orport=443 id=60D3667F56AEC5C69CF7E8F557DB21DDF6C36060 # QuintexAirVPN6
-199.249.223.46:80 orport=443 id=66E19E8C4773086F669A1E06A3F8C23B6C079129 # Quintex37
-199.249.224.65:80 orport=443 id=764BF8A03868F84C8F323C1A676AA254B80DC3BF # Quintex45
-199.249.223.48:80 orport=443 id=7A3DD280EA4CD4DD16EF8C67B93D9BDE184D1A81 # Quintex39
-199.249.224.68:80 orport=443 id=7E6E9A6FDDB8DC7C92F0CFCC3CBE76C29F061799 # Quintex48
-199.249.223.69:80 orport=443 id=7FA8E7E44F1392A4E40FFC3B69DB3B00091B7FD3 # Quintex20
-199.249.223.44:80 orport=443 id=8B80169BEF71450FC4069A190853523B7AEA45E1 # Quintex35
-199.249.224.60:80 orport=443 id=9314BD9503B9014261A65C221D77E57389DBCCC1 # Quintex50
-199.249.224.40:80 orport=443 id=9C1E7D92115D431385B8CAEA6A7C15FB89CE236B # QuintexAirVPN21
-199.249.223.65:80 orport=443 id=9D21F034C3BFF4E7737D08CF775DC1745706801F # Quintex16
-199.249.224.67:80 orport=443 id=9E2D7C6981269404AA1970B53891701A20424EF8 # Quintex47
-199.249.223.64:80 orport=443 id=9F2856F6D2B89AD4EF6D5723FAB167DB5A53519A # Quintex15
-199.249.224.48:80 orport=443 id=A0DB820FEC87C0405F7BF05DEE5E4ADED2BB9904 # QuintexAirVPN29
-199.249.224.64:80 orport=443 id=A4A393FEF48640961AACE92D041934B55348CEF9 # Quintex44
-199.249.223.72:80 orport=443 id=B028707969D8ED84E6DEA597A884F78AAD471971 # QuintexAirVPN9
-199.249.223.40:80 orport=443 id=B0CD9F9B5B60651ADC5919C0F1EAA87DBA1D9249 # Quintex31
-199.249.224.61:80 orport=443 id=B2197C23A4FF5D1C49EE45BA7688BA8BCCD89A0B # Quintex41
-199.249.223.71:80 orport=443 id=B6320E44A230302C7BF9319E67597A9B87882241 # QuintexAirVPN10
-199.249.223.60:80 orport=443 id=B7047FBDE9C53C39011CA84E5CB2A8E3543066D0 # Quintex11
-199.249.224.66:80 orport=443 id=C78AFFEEE320EA0F860961763E613FD2FAC855F5 # Quintex46
-199.249.224.44:80 orport=443 id=CB7C0D841FE376EF43F7845FF201B0290C0A239E # QuintexAirVPN25
-199.249.223.47:80 orport=443 id=CC14C97F1D23EE97766828FC8ED8582E21E11665 # Quintex38
-199.249.223.77:80 orport=443 id=CC4A3AE960E3617F49BF9887B79186C14CBA6813 # QuintexAirVPN4
-199.249.223.41:80 orport=443 id=D25210CE07C49F2A4F2BC7A506EB0F5EA7F5E2C2 # Quintex32
-199.249.223.79:80 orport=443 id=D33292FEDE24DD40F2385283E55C87F85C0943B6 # QuintexAirVPN2
-199.249.224.47:80 orport=443 id=D6FF2697CEA5C0C7DA84797C2E71163814FC2466 # QuintexAirVPN28
-199.249.223.68:80 orport=443 id=DF20497E487A979995D851A5BCEC313DF7E5BC51 # Quintex19
-199.249.223.43:80 orport=443 id=E480D577F58E782A5BC4FA6F49A6650E9389302F # Quintex34
-199.249.224.69:80 orport=443 id=EABC2DD0D47B5DB11F2D37EB3C60C2A4D91C10F2 # Quintex49
-199.249.223.78:80 orport=443 id=EC15DB62D9101481F364DE52EB8313C838BDDC29 # QuintexAirVPN3
-199.249.224.42:80 orport=443 id=F21DE9C7DE31601D9716781E17E24380887883D1 # QuintexAirVPN23
-199.249.223.81:80 orport=443 id=F7447E99EB5CBD4D5EB913EE0E35AC642B5C1EF3 # QuintexAirVPN1
-199.249.224.43:80 orport=443 id=FDD700C791CC6BB0AC1C2099A82CBC367AD4B764 # QuintexAirVPN24
-199.249.224.62:80 orport=443 id=FE00A3A835680E67FBBC895A724E2657BB253E97 # Quintex42
-199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2 # Quintex17
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013914.html
-5.196.23.64:9030 orport=9001 id=775B0FAFDE71AADC23FFC8782B7BEB1D5A92733E # Aerodynamik01
-217.182.75.181:9030 orport=9001 id=EFEACD781604EB80FBC025EDEDEA2D523AEAAA2F # Aerodynamik02
-193.70.43.76:9030 orport=9001 id=484A10BA2B8D48A5F0216674C8DD50EF27BC32F3 # Aerodynamik03
-149.56.141.138:9030 orport=9001 id=1938EBACBB1A7BFA888D9623C90061130E63BB3F # Aerodynamik04
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013917.html
-104.200.20.46:80 orport=9001 id=78E2BE744A53631B4AAB781468E94C52AB73968B # bynumlawtor
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013929.html
-139.99.130.178:80 orport=443 id=867B95CACD64653FEEC4D2CEFC5C49B4620307A7 # coffswifi2
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013946.html
-172.98.193.43:80 orport=443 id=5E56738E7F97AA81DEEF59AF28494293DFBFCCDF # Backplane
-
-# Email sent directly to teor
-62.210.254.132:80 orport=443 id=8456DFA94161CDD99E480C2A2992C366C6564410 # turingmachine
-
-# Email sent directly to teor
-80.127.117.180:80 orport=443 id=328E54981C6DDD7D89B89E418724A4A7881E3192 ipv6=[2001:985:e77:10::4]:443 # sjc01
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013960.html
-51.15.205.214:9030 orport=9001 id=8B6556601612F1E2AFCE2A12FFFAF8482A76DD1F ipv6=[2001:bc8:4400:2500::5:b07]:9001 # titania1
-51.15.205.214:9031 orport=9002 id=5E363D72488276160D062DDD2DFA25CFEBAF5EA9 ipv6=[2001:bc8:4400:2500::5:b07]:9002 # titania2
-
-# Email sent directly to teor
-185.129.249.124:9030 orport=9001 id=1FA8F638298645BE58AC905276680889CB795A94 # treadstone
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/014000.html
-24.117.231.229:34175 orport=45117 id=CE24412AD69444954B4015E293AE53DDDAFEA3D6 # Anosognosia
-
-# https://lists.torproject.org/pipermail/tor-relays/2018-January/014012.html
-128.31.0.13:80 orport=443 id=A53C46F5B157DD83366D45A8E99A244934A14C46 # csailmitexit
-
-# Email sent directly to teor
-82.247.103.117:110 orport=995 id=C9B3C1661A9577BA24C1C2C6123918921A495509 # Casper01
-109.238.2.79:110 orport=995 id=7520892E3DD133D0B0464D01A158B54B8E2A8B75 # Casper02
-51.15.179.153:110 orport=995 id=BB60F5BA113A0B8B44B7B37DE3567FE561E92F78 # Casper04
-
-# Email sent directly to teor
-80.127.107.179:80 orport=443 id=BC6B2E2F62ACC5EDECBABE64DA1E48F84DD98B78 ipv6=[2001:981:4a22:c::6]:443 # TVISION02
-
-# https://lists.torproject.org/pipermail/tor-relays/2018-January/014020.html
-37.120.174.249:80 orport=443 id=11DF0017A43AF1F08825CD5D973297F81AB00FF3 ipv6=[2a03:4000:6:724c:df98:15f9:b34d:443]:443 # gGDHjdcC6zAlM8k08lX
-
-# These fallbacks opted-in in previous releases, then changed their details,
-# and so we blacklisted them. Now we want to whitelist changes.
-# Assume details update is permanent
-85.230.184.93:9030 orport=443 id=855BC2DABE24C861CD887DB9B2E950424B49FC34 # Logforme
-176.31.180.157:143 orport=22 id=E781F4EC69671B3F1864AE2753E0890351506329 ipv6=[2001:41d0:8:eb9d::1]:22 # armbrust
-
-# https://lists.torproject.org/pipermail/tor-relays/2018-January/014024.html
-82.161.212.209:9030 orport=9001 id=4E8CE6F5651E7342C1E7E5ED031E82078134FB0D ipv6=[2001:980:d7ed:1:ff:b0ff:fe00:d0b]:9001 # ymkeo

scripts/maint/format_changelog.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright (c) 2014-2017, The Tor Project, Inc.
+# Copyright (c) 2014-2015, The Tor Project, Inc.
 # See LICENSE for licensing information
 #
 # This script reformats a section of the changelog to wrap everything to
@@ -205,8 +205,6 @@ def head_score(s):
         score = -300
     elif lw.startswith("deprecated version"):
         score = -200
-    elif lw.startswith("directory auth"):
-        score = -150
     elif (('new' in lw and 'requirement' in lw) or
           ('new' in lw and 'dependenc' in lw) or
           ('build' in lw and 'requirement' in lw) or
@@ -400,24 +398,16 @@ class ChangeLog(object):
         self.dumpEndOfSections()
         self.dumpEndOfChangelog()
 
-# Let's turn bugs to html.
-BUG_PAT = re.compile('(bug|ticket|feature)\s+(\d{4,5})', re.I)
-def bug_html(m):
-    return "%s <a href='https://bugs.torproject.org/%s'>%s</a>" % (m.group(1), m.group(2), m.group(2))
-
 class HTMLChangeLog(ChangeLog):
     def __init__(self, *args, **kwargs):
         ChangeLog.__init__(self, *args, **kwargs)
 
     def htmlText(self, graf):
-        output = []
         for line in graf:
             line = line.rstrip().replace("&","&amp;")
             line = line.rstrip().replace("<","&lt;").replace(">","&gt;")
-            output.append(line.strip())
-        output = " ".join(output)
-        output = BUG_PAT.sub(bug_html, output)
-        sys.stdout.write(output)
+            sys.stdout.write(line.strip())
+            sys.stdout.write(" ")
 
     def htmlPar(self, graf):
         sys.stdout.write("<p>")

scripts/maint/generateFallbackDirLine.py

@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-
-# Generate a fallback directory whitelist/blacklist line for every fingerprint
-# passed as an argument.
-#
-# Usage:
-# generateFallbackDirLine.py fingerprint ...
-
-import sys
-import urllib2
-
-import stem.descriptor.remote
-import stem.util.tor_tools
-
-if len(sys.argv) <= 1:
-  print('Usage: %s fingerprint ...' % sys.argv[0])
-  sys.exit(1)
-
-for fingerprint in sys.argv[1:]:
-  if not stem.util.tor_tools.is_valid_fingerprint(fingerprint):
-    print("'%s' isn't a valid relay fingerprint" % fingerprint)
-    sys.exit(1)
-
-  try:
-    desc = stem.descriptor.remote.get_server_descriptors(fingerprint).run()[0]
-  except urllib2.HTTPError as exc:
-    if exc.code == 404:
-      print('# %s not found in recent descriptors' % fingerprint)
-      continue
-    else:
-      raise
-
-  if not desc.dir_port:
-    print("# %s needs a DirPort" % fingerprint)
-  else:
-    ipv6_addresses = [(address, port) for address, port, is_ipv6 in desc.or_addresses if is_ipv6]
-    ipv6_field = ' ipv6=[%s]:%s' % ipv6_addresses[0] if ipv6_addresses else ''
-    print('%s:%s orport=%s id=%s%s # %s' % (desc.address, desc.dir_port, desc.or_port, fingerprint, ipv6_field, desc.nickname))

scripts/maint/generate_callgraph.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+C_FILES=`echo src/common/*.c src/or/*.c src/tools/*.c`
+CFLAGS="-Isrc/ext/trunnel -Isrc/trunnel -I. -Isrc/ext -Isrc/common -DLOCALSTATEDIR=\"\" -DSHARE_DATADIR=\"\" -Dinline="
+
+mkdir -p callgraph/src/common
+mkdir -p callgraph/src/or
+mkdir -p callgraph/src/tools
+
+for fn in $C_FILES; do
+  echo $fn
+  clang $CFLAGS  -S -emit-llvm -fno-inline -o - $fn  | \
+    opt -analyze -print-callgraph >/dev/null 2> "callgraph/${fn}allgraph"
+done

scripts/maint/lintChanges.py

@@ -7,7 +7,7 @@ import re
 import os
 
 
-KNOWN_GROUPS = set([
+KNOWN_GROUPS=set([
     "Minor bugfix",
     "Minor bugfixes",
     "Major bugfix",
@@ -20,20 +20,7 @@ KNOWN_GROUPS = set([
     "Testing",
     "Documentation",
     "Code simplification and refactoring",
-    "Removed features",
-    "Deprecated features",
-    "Directory authority changes"])
-
-NEEDS_SUBCATEGORIES = set([
-    "Minor bugfix",
-    "Minor bugfixes",
-    "Major bugfix",
-    "Major bugfixes",
-    "Minor feature",
-    "Minor features",
-    "Major feature",
-    "Major features",
-    ])
+    "Removed features"])
 
 def lintfile(fname):
     have_warned = []
@@ -56,62 +43,40 @@ def lintfile(fname):
     if bugnum and bugnum not in contents:
         warn("bug number {} does not appear".format(bugnum))
 
+    lines = contents.split("\n")
+    isBug = ("bug" in lines[0] or "fix" in lines[0])
+
     m = re.match(r'^[ ]{2}o ([^\(:]*)([^:]*):', contents)
     if not m:
-        warn("Header not in format expected. ('  o Foo:' or '  o Foo (Bar):')")
+        warn("header not in format expected")
     elif m.group(1).strip() not in KNOWN_GROUPS:
-        warn("Unrecognized header: %r" % m.group(1))
-    elif (m.group(1) in NEEDS_SUBCATEGORIES and '(' not in m.group(2)):
-        warn("Missing subcategory on %r" % m.group(1))
+        warn("Weird header: %r"%m.group(1))
+    elif ( ("bugfix" in m.group(1) or "feature" in m.group(1)) and
+           ("Removed" not in m.group(1)) and
+           '(' not in m.group(2)):
+        warn("Missing subcategory on %s"%m.group(1))
 
-    if m:
-        isBug = ("bug" in m.group(1).lower() or "fix" in m.group(1).lower())
-    else:
-        isBug = False
 
     contents = " ".join(contents.split())
 
     if re.search(r'\#\d{2,}', contents):
-        warn("Don't use a # before ticket numbers. ('bug 1234' not '#1234')")
+        warn("don't use a # before ticket numbers")
 
     if isBug and not re.search(r'(\d+)', contents):
-        warn("Ticket marked as bugfix, but does not mention a number.")
-    elif isBug and not re.search(r'Fixes ([a-z ]*)bugs? (\d+)', contents):
-        warn("Ticket marked as bugfix, but does not say 'Fixes bug XXX'")
+        warn("bugfix does not mention a number")
+    elif isBug and not re.search(r'Fixes ([a-z ]*)bug (\d+)', contents):
+        warn("bugfix does not say 'Fixes bug XXX'")
 
     if re.search(r'[bB]ug (\d+)', contents):
         if not re.search(r'[Bb]ugfix on ', contents):
-            warn("Bugfix does not say 'bugfix on X.Y.Z'")
-        elif not re.search('[fF]ixes ([a-z ]*)bugs? (\d+)((, \d+)* and \d+)?; bugfix on ',
+            warn("bugfix does not say 'bugfix on X.Y.Z'")
+        elif not re.search('[fF]ixes ([a-z ]*)bug (\d+); bugfix on ',
                            contents):
-            warn("Bugfix does not say 'Fixes bug X; bugfix on Y'")
-        elif re.search('tor-([0-9]+)', contents):
-            warn("Do not prefix versions with 'tor-'. ('0.1.2', not 'tor-0.1.2'.)")
+            warn("bugfix incant is not semicoloned")
 
-    return have_warned != []
-
-def files(args):
-    """Walk through the arguments: for directories, yield their contents;
-       for files, just yield the files. Only search one level deep, because
-       that's how the changes directory is laid out."""
-    for f in args:
-        if os.path.isdir(f):
-            for item in os.listdir(f):
-                if item.startswith("."): #ignore dotfiles
-                    continue
-                yield os.path.join(f, item)
-        else:
-            yield f
 
 if __name__ == '__main__':
-    problems = 0
-    for fname in files(sys.argv[1:]):
+    for fname in sys.argv[1:]:
         if fname.endswith("~"):
             continue
-        if lintfile(fname):
-            problems += 1
-
-    if problems:
-        sys.exit(1)
-    else:
-        sys.exit(0)
+        lintfile(fname)

scripts/maint/lookupFallbackDirContact.py

@@ -1,28 +0,0 @@
-#!/usr/bin/env python
-
-# Lookup fallback directory contact lines for every fingerprint passed as an
-# argument.
-#
-# Usage:
-# lookupFallbackDirContact.py fingerprint ...
-
-import sys
-
-import stem.descriptor.remote as remote
-
-if len(sys.argv) <= 1:
-  print "Usage: {} fingerprint ...".format(sys.argv[0])
-  sys.exit(-1)
-
-# we need descriptors, because the consensus does not have contact infos
-descriptor_list = remote.get_server_descriptors(fingerprints=sys.argv[1:]).run()
-
-descriptor_list_fingerprints = []
-for d in descriptor_list:
-  assert d.fingerprint in sys.argv[1:]
-  descriptor_list_fingerprints.append(d.fingerprint)
-  print "{} {}".format(d.fingerprint, d.contact)
-
-for fingerprint in sys.argv[1:]:
-  if fingerprint not in descriptor_list_fingerprints:
-    print "{} not found in current descriptors".format(fingerprint)

scripts/maint/redox.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python
 #
-#  Copyright (c) 2008-2017, The Tor Project, Inc.
+#  Copyright (c) 2008-2015, The Tor Project, Inc.
 #  See LICENSE for licensing information.
 #
 # Hi!
@@ -33,6 +33,8 @@
 # files that we've snarfed in from somebody else, whose C we do no intend
 # to document for them.
 SKIP_FILES = [ "OpenBSD_malloc_Linux.c",
+               "eventdns.c",
+               "eventdns.h",
                "strlcat.c",
                "strlcpy.c",
                "sha256.c",
@@ -101,7 +103,7 @@ def read():
 
 def findline(lines, lineno, ident):
     """Given a list of all the lines in the file (adjusted so 1-indexing works),
-       a line number that ident is allegedly on, and ident, I figure out
+       a line number that ident is alledgedly on, and ident, I figure out
        the line where ident was really declared."""
     lno = lineno
     for lineno in xrange(lineno, 0, -1):

scripts/maint/run_calltool.sh

@@ -1,29 +0,0 @@
-#!/bin/sh
-
-# You can find calltool at https://gitweb.torproject.org/user/nickm/calltool.git
-
-set -e
-
-if test "x$CALLTOOL_PATH" != "x"; then
-  PYTHONPATH="${CALLTOOL_PATH}:${PYTHONPATH}"
-  export PYTHONPATH
-fi
-
-mkdir -p callgraph
-
-SUBITEMS="fn_graph fn_invgraph fn_scc fn_scc_weaklinks module_graph module_invgraph module_scc module_scc_weaklinks"
-
-for calculation in $SUBITEMS; do
-  echo "======== $calculation"
-  python -m calltool $calculation > callgraph/$calculation
-done
-
-echo <<EOF > callgraph/README
-This directory holds output from calltool, as run on Tor.  For more
-information about each of these files, see the NOTES and README files in
-the calltool distribution.
-
-You can find calltool at
-    https://gitweb.torproject.org/user/nickm/calltool.git
-EOF
-

scripts/maint/sortChanges.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright (c) 2014-2017, The Tor Project, Inc.
+# Copyright (c) 2014-2015, The Tor Project, Inc.
 # See LICENSE for licensing information
 
 """This script sorts a bunch of changes files listed on its command

scripts/maint/updateCopyright.pl

@@ -1,7 +1,7 @@
 #!/usr/bin/perl -i -w -p
 
-$NEWYEAR=2017;
+$NEWYEAR=2016;
 
-s/Copyright(.*) (201[^7]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/;
+s/Copyright(.*) (201[^6]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/;
 
 s/Copyright(.*)-(20..), The Tor Project/Copyright$1-${NEWYEAR}, The Tor Project/;

scripts/maint/updateRustDependencies.sh

@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018 The Tor Project, Inc.
-# Copyright (c) 2018 isis agora lovecruft
-# See LICENSE for license information
-#
-# updateRustDependencies.sh
-# -------------------------
-# Update our vendored Rust dependencies, either adding/removing
-# dependencies and/or upgrading current dependencies to newer
-# versions.
-#
-# To use this script, first add your dependencies, exactly specifying
-# their versions, into the appropriate *crate-level* Cargo.toml in
-# src/rust/ (i.e. *not* /src/rust/Cargo.toml, but instead the one for
-# your crate).
-#
-# Next, run this script.  Then, go into src/ext/rust and commit the
-# changes to the tor-rust-dependencies repo.
-
-set -e
-
-HERE=`dirname $(realpath $0)`
-TOPLEVEL=`dirname $(dirname $HERE)`
-TOML="$TOPLEVEL/src/rust/Cargo.toml"
-VENDORED="$TOPLEVEL/src/ext/rust/crates"
-CARGO=`which cargo`
-
-if ! test -f "$TOML"  ; then
-    printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\n" "$TOML"
-fi
-
-if ! test -d "$VENDORED" ; then
-    printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\n" "$VENDORED"
-fi
-
-if test -z "$CARGO" ; then
-    printf "Error: cargo must be installed and in your \$PATH\n"
-fi
-
-if test -z `cargo --list | grep vendor` ; then
-    printf "Error: cargo-vendor not installed\n"
-fi
-
-$CARGO vendor -v --locked --explicit-version --no-delete --sync $TOML $VENDORED

scripts/test/appveyor-irc-notify.py

@@ -1,192 +0,0 @@
-# coding=utf8
-# Copyright (C) 2015-2016 Christopher R. Wood
-# Copyright (c) 2018 The Tor Project
-# Copyright (c) 2018 isis agora lovecruft
-#
-# From: https://raw.githubusercontent.com/gridsync/gridsync/def54f8166089b733d166665fdabcad4cdc526d8/misc/irc-notify.py
-# and: https://github.com/gridsync/gridsync
-#
-# Modified by nexB on October 2016:
-#  - rework the handling of environment variables.
-#  - made the script use functions
-#  - support only Appveyor loading its environment variable to craft IRC notices.
-#
-# Modified by isis agora lovecruft <isis@torproject.org> in 2018:
-#  - Make IRC server configurable.
-#  - Make bot IRC nick deterministic.
-#  - Make bot join the channel rather than sending NOTICE messages externally.
-#  - Fix a bug which always caused sys.exit() to be logged as a traceback.
-#  - Actually reset the IRC colour codes after printing.
-#
-# Modified by Marcin Cieślak in 2018:
-#  - Accept UTF-8
-#  - only guess github URLs
-#  - stop using ANSI colors
-
-# This program is free software; you can redistribute it and/or modify it under the
-# terms of the GNU General Public License as published by the Free Software Foundation;
-# either version 2 of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT ANY
-# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
-# PARTICULAR PURPOSE. See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with this
-# program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street,
-# Fifth Floor, Boston, MA 02110-1301 USA.
-
-"""Simple AppVeyor IRC notification script.
-
-The first argument is an IRC server and port; the second is the channel. Other
-arguments passed to the script will be sent as notice messages content and any
-{var}-formatted environment variables will be expanded automatically, replaced
-with a corresponding Appveyor environment variable value. Use commas to
-delineate multiple messages.
-
-
-Example:
-export APPVEYOR_URL=https://ci.appveyor.com
-export APPVEYOR_PROJECT_NAME=tor
-export APPVEYOR_REPO_COMMIT_AUTHOR=isislovecruft
-export APPVEYOR_REPO_COMMIT_TIMESTAMP=2018-04-23
-export APPVEYOR_REPO_PROVIDER=gihub
-export APPVEYOR_REPO_BRANCH=repo_branch
-export APPVEYOR_PULL_REQUEST_TITLE=pull_request_title
-export APPVEYOR_BUILD_VERSION=1
-export APPVEYOR_REPO_COMMIT=22c95b72e29248dc4de9b85e590ee18f6f587de8
-export APPVEYOR_REPO_COMMIT_MESSAGE="some IRC test"
-export APPVEYOR_ACCOUNT_NAME=isislovecruft
-export APPVEYOR_PULL_REQUEST_NUMBER=pull_request_number
-export APPVEYOR_REPO_NAME=isislovecruft/tor
-python ./appveyor-irc-notify.py irc.oftc.net:6697 tor-ci '{repo_name} {repo_branch} {short_commit} - {repo_commit_author}: {repo_commit_message}','Build #{build_version} passed. Details: {build_url} |  Commit: {commit_url}
-
-See also https://github.com/gridsync/gridsync/blob/master/appveyor.yml for examples
-in Appveyor's YAML:
-
-    on_success:
-      - "python scripts/test/appveyor-irc-notify.py irc.oftc.net:6697 tor-ci success
-    on_failure:
-      - "python scripts/test/appveyor-irc-notify.py irc.oftc.net:6697 tor-ci failure
-"""
-
-from __future__ import print_function
-from __future__ import absolute_import
-
-import os
-import random
-import socket
-import ssl
-import sys
-import time
-
-
-def appveyor_vars():
-    """
-    Return a dict of key value carfted from appveyor environment variables.
-    """
-
-    vars = dict([
-            (
-                v.replace('APPVEYOR_', '').lower(),
-                os.getenv(v, '').decode('utf-8')
-            ) for v in [
-                'APPVEYOR_URL',
-                'APPVEYOR_REPO_COMMIT_MESSAGE_EXTENDED',
-                'APPVEYOR_REPO_BRANCH',
-                'APPVEYOR_REPO_COMMIT_AUTHOR',
-                'APPVEYOR_REPO_COMMIT_AUTHOR_EMAIL',
-                'APPVEYOR_REPO_COMMIT_TIMESTAMP',
-                'APPVEYOR_REPO_PROVIDER',
-                'APPVEYOR_PROJECT_NAME',
-                'APPVEYOR_PULL_REQUEST_TITLE',
-                'APPVEYOR_BUILD_VERSION',
-                'APPVEYOR_REPO_COMMIT',
-                'APPVEYOR_REPO_COMMIT_MESSAGE',
-                'APPVEYOR_ACCOUNT_NAME',
-                'APPVEYOR_PULL_REQUEST_NUMBER',
-                'APPVEYOR_REPO_NAME'
-            ]
-    ])
-
-    BUILD_FMT = u'{url}/project/{account_name}/{project_name}/build/{build_version}'
-
-    if vars["repo_provider"] == 'github':
-        COMMIT_FMT = u'https://{repo_provider}.com/{repo_name}/commit/{repo_commit}'
-        vars.update(commit_url=COMMIT_FMT.format(**vars))
-
-    vars.update(
-        build_url=BUILD_FMT.format(**vars),
-        short_commit=vars["repo_commit"][:7],
-    )
-    return vars
-
-
-def notify():
-    """
-    Send IRC notification
-    """
-    apvy_vars = appveyor_vars()
-
-    server, port = sys.argv[1].rsplit(":", 1)
-    channel = sys.argv[2]
-    success = sys.argv[3] == "success"
-    failure = sys.argv[3] == "failure"
-
-    if success or failure:
-        messages = []
-        messages.append(u"{repo_name} {repo_branch} {short_commit} - {repo_commit_author}: {repo_commit_message}")
-
-        if success:
-            m = u"Build #{build_version} passed. Details: {build_url}"
-        if failure:
-            m = u"Build #{build_version} failed. Details: {build_url}"
-
-        if "commit_url" in apvy_vars:
-            m += " Commit: {commit_url}"
-     
-        messages.append(m)
-    else:
-        messages = sys.argv[3:]
-        messages = ' '.join(messages)
-        messages = messages.decode("utf-8").split(',')
-
-    print(repr(apvy_vars))
-    messages = [msg.format(**apvy_vars).strip() for msg in messages]
-
-    irc_username = 'appveyor-ci'
-    irc_nick = irc_username
-
-    # establish connection
-    irc_sock = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM))
-    irc_sock.connect((socket.gethostbyname(server), int(port)))
-    irc_sock.send('NICK {0}\r\nUSER {0} * 0 :{0}\r\n'.format(irc_username).encode())
-    irc_sock.send('JOIN #{0}\r\n'.format(channel).encode())
-    irc_file = irc_sock.makefile()
-
-    while irc_file:
-        line = irc_file.readline()
-        print(line.rstrip())
-        response = line.split()
-
-        if response[0] == 'PING':
-            irc_file.send('PONG {}\r\n'.format(response[1]).encode())
-
-        elif response[1] == '433':
-            irc_sock.send('NICK {}\r\n'.format(irc_nick).encode())
-
-        elif response[1] == '001':
-            time.sleep(5)
-            # send notification
-            for msg in messages:
-                print(u'PRIVMSG #{} :{}'.format(channel, msg).encode("utf-8"))
-                irc_sock.send(u'PRIVMSG #{} :{}\r\n'.format(channel, msg).encode("utf-8"))
-            time.sleep(5)
-            return
-
-
-if __name__ == '__main__':
-    try:
-        notify()
-    except:
-        import traceback
-        print('ERROR: Failed to send notification: \n' + traceback.format_exc())

scripts/test/cov-diff

@@ -7,15 +7,11 @@
 DIRA="$1"
 DIRB="$2"
 
-for B in $DIRB/*; do
-  A=$DIRA/`basename $B`
-  if [ -f $A ]; then
-      perl -pe 's/^\s*\!*\d+(\*?):/        1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp"
-  else
-      cat /dev/null > "$A.tmp"
-  fi
-  perl -pe 's/^\s*\!*\d+(\*?):/        1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp"
-  diff -u "$A.tmp" "$B.tmp" |perl -pe 's/^((?:\+\+\+|---)(?:.*tmp))\s+.*/$1/;'
+for A in $DIRA/*; do
+  B=$DIRB/`basename $A`
+  perl -pe 's/^\s*\d+:/        1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp"
+  perl -pe 's/^\s*\d+:/        1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp"
+  diff -u "$A.tmp" "$B.tmp"
   rm "$A.tmp" "$B.tmp"
 done
 

scripts/test/cov-exclude

@@ -1,34 +0,0 @@
-#!/usr/bin/perl -p -i
-
-use warnings;
-use strict;
-our $excluding;
-
-# This script is meant to post-process a .gcov file for an input source
-# that was annotated with LCOV_EXCL_START, LCOV_EXCL_STOP, and LCOV_EXCL_LINE
-# entries.  It doesn't understand the LCOV_EXCL_BR* variations.
-#
-# It replaces unreached reached lines with x:, and reached excluded lines
-# with !!!num:.
-
-BEGIN { our $excluding = 0; }
-
-if (m/LCOV_EXCL_START/) {
-  $excluding = 1;
-}
-if ($excluding and m/LCOV_EXCL_STOP/) {
-  $excluding = 0;
-}
-
-my $exclude_this = (m/LCOV_EXCL_LINE/);
-
-if ($excluding or $exclude_this) {
-  s{^\s*\#\#+:}{        x:};
-  s{^   (\s*)(\d+):}{$1!!!$2:};
-}
-
-if (eof and $excluding) {
-    warn "Runaway LCOV_EXCL_START in $ARGV";
-    $excluding = 0;
-}
-

scripts/test/coverage

@@ -29,7 +29,7 @@ for fn in src/or/*.c src/common/*.c; do
           gcov -o $on $fn
           if [ -e $GC ]
           then
-            if [ -d "$dst" ]
+            if [ -n $dst ]
             then
               mv $GC $dst/$GC
             fi

scripts/test/scan-build.sh

@@ -5,76 +5,37 @@
 # This script is used for running a bunch of clang scan-build checkers
 # on Tor.
 
-# These don't seem to cause false positives in our code, so let's turn
-# them on.
 CHECKERS="\
-    -enable-checker alpha.core.CallAndMessageUnInitRefArg \
+    -disable-checker deadcode.DeadStores \
+    -enable-checker alpha.core.CastSize \
     -enable-checker alpha.core.CastToStruct \
-    -enable-checker alpha.core.Conversion \
-    -enable-checker alpha.core.FixedAddr \
     -enable-checker alpha.core.IdenticalExpr \
-    -enable-checker alpha.core.PointerArithm \
     -enable-checker alpha.core.SizeofPtr \
-    -enable-checker alpha.core.TestAfterDivZero \
+    -enable-checker alpha.security.ArrayBoundV2 \
     -enable-checker alpha.security.MallocOverflow \
     -enable-checker alpha.security.ReturnPtrRange \
-    -enable-checker alpha.unix.BlockInCriticalSection \
-    -enable-checker alpha.unix.Chroot \
-    -enable-checker alpha.unix.PthreadLock \
-    -enable-checker alpha.unix.PthreadLock \
-    -enable-checker alpha.unix.SimpleStream \
-    -enable-checker alpha.unix.Stream \
+    -enable-checker alpha.unix.SimpleStream
     -enable-checker alpha.unix.cstring.BufferOverlap \
     -enable-checker alpha.unix.cstring.NotNullTerminated \
-    -enable-checker valist.CopyToSelf \
-    -enable-checker valist.Uninitialized \
-    -enable-checker valist.Unterminated \
-    -enable-checker security.FloatLoopCounter \
-    -enable-checker security.insecureAPI.strcpy \
-"
-
-# These have high false-positive rates.
-EXTRA_CHECKERS="\
-    -enable-checker alpha.security.ArrayBoundV2 \
     -enable-checker alpha.unix.cstring.OutOfBounds \
-    -enable-checker alpha.core.CastSize \
+    -enable-checker alpha.core.FixedAddr \
+    -enable-checker security.insecureAPI.strcpy \
+    -enable-checker alpha.unix.PthreadLock \
+    -enable-checker alpha.core.PointerArithm \
+    -enable-checker alpha.core.TestAfterDivZero \
 "
 
-# These don't seem to generate anything useful
-NOISY_CHECKERS="\
-    -enable-checker alpha.clone.CloneChecker \
-    -enable-checker alpha.deadcode.UnreachableCode \
-"
-
-if test "x$SCAN_BUILD_OUTPUT" != "x"; then
-   OUTPUTARG="-o $SCAN_BUILD_OUTPUT"
-else
-   OUTPUTARG=""
-fi
-
 scan-build \
     $CHECKERS \
     ./configure
 
-scan-build \
-    make clean
-
-# Make this not get scanned for dead assignments, since it has lots of
-# dead assignments we don't care about.
 scan-build \
     $CHECKERS \
-    -disable-checker deadcode.DeadStores \
-    make -j5 -k ./src/ext/ed25519/ref10/libed25519_ref10.a
+    make -j2 -k
 
-scan-build \
-    $CHECKERS $OUTPUTARG \
-    make -j5 -k
-
-CHECKERS="\
-"
 
 # This one gives a false positive on every strcmp.
 #    -enable-checker alpha.core.PointerSub
 
 # Needs work
-#    -enable-checker alpha.unix.MallocWithAnnotations
+#  alpha.unix.MallocWithAnnotations ??

src/common/Makefile.nmake

@@ -7,8 +7,8 @@ LIBOR_OBJECTS = address.obj backtrace.obj compat.obj container.obj di_ops.obj \
 	log.obj memarea.obj mempool.obj procmon.obj sandbox.obj util.obj \
 	util_codedigest.obj
 
-LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj compress.obj compress_zlib.obj \
-	tortls.obj crypto_curve25519.obj curve25519-donna.obj
+LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj torgzip.obj tortls.obj \
+	crypto_curve25519.obj curve25519-donna.obj
 
 LIBOR_EVENT_OBJECTS = compat_libevent.obj
 

src/common/address.c

@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2017, The Tor Project, Inc. */
+ * Copyright (c) 2007-2016, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -33,7 +33,7 @@
 #include <process.h>
 #include <windows.h>
 #include <iphlpapi.h>
-#endif /* defined(_WIN32) */
+#endif
 
 #include "compat.h"
 #include "util.h"
@@ -131,8 +131,7 @@ tor_addr_to_sockaddr(const tor_addr_t *a,
 #endif
     sin6->sin6_family = AF_INET6;
     sin6->sin6_port = htons(port);
-    memcpy(&sin6->sin6_addr, tor_addr_to_in6_assert(a),
-           sizeof(struct in6_addr));
+    memcpy(&sin6->sin6_addr, tor_addr_to_in6(a), sizeof(struct in6_addr));
     return sizeof(struct sockaddr_in6);
   } else {
     return 0;
@@ -159,8 +158,6 @@ tor_addr_from_sockaddr(tor_addr_t *a, const struct sockaddr *sa,
   tor_assert(a);
   tor_assert(sa);
 
-  /* This memset is redundant; leaving it in to avoid any future accidents,
-     however. */
   memset(a, 0, sizeof(*a));
 
   if (sa->sa_family == AF_INET) {
@@ -198,7 +195,7 @@ tor_sockaddr_to_str(const struct sockaddr *sa)
     tor_asprintf(&result, "unix:%s", s_un->sun_path);
     return result;
   }
-#endif /* defined(HAVE_SYS_UN_H) */
+#endif
   if (sa->sa_family == AF_UNSPEC)
     return tor_strdup("unspec");
 
@@ -237,8 +234,8 @@ tor_addr_make_null(tor_addr_t *a, sa_family_t family)
  *
  * Return 0 on success, -1 on failure; 1 on transient failure.
  */
-MOCK_IMPL(int,
-tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr))
+int
+tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr)
 {
   /* Perhaps eventually this should be replaced by a tor_getaddrinfo or
    * something.
@@ -305,7 +302,7 @@ tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr))
       return result;
     }
     return (err == EAI_AGAIN) ? 1 : -1;
-#else /* !(defined(HAVE_GETADDRINFO)) */
+#else
     struct hostent *ent;
     int err;
 #ifdef HAVE_GETHOSTBYNAME_R_6_ARG
@@ -330,14 +327,14 @@ tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr))
 #else
     err = h_errno;
 #endif
-#endif /* defined(HAVE_GETHOSTBYNAME_R_6_ARG) || ... */
+#endif /* endif HAVE_GETHOSTBYNAME_R_6_ARG. */
     if (ent) {
       if (ent->h_addrtype == AF_INET) {
         tor_addr_from_in(addr, (struct in_addr*) ent->h_addr);
       } else if (ent->h_addrtype == AF_INET6) {
         tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr);
       } else {
-        tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type
+        tor_assert(0); /* gethostbyname() returned a bizarre addrtype */
       }
       return 0;
     }
@@ -346,7 +343,7 @@ tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr))
 #else
     return (err == TRY_AGAIN) ? 1 : -1;
 #endif
-#endif /* defined(HAVE_GETADDRINFO) */
+#endif
   }
 }
 
@@ -564,8 +561,8 @@ tor_addr_parse_PTR_name(tor_addr_t *result, const char *address,
 
 /** Convert <b>addr</b> to an in-addr.arpa name or a .ip6.arpa name,
  * and store the result in the <b>outlen</b>-byte buffer at
- * <b>out</b>.  Returns a non-negative integer on success.
- * Returns -1 on failure. */
+ * <b>out</b>.  Return the number of chars written to <b>out</b>, not
+ * including the trailing \0, on success. Returns -1 on failure. */
 int
 tor_addr_to_PTR_name(char *out, size_t outlen,
                      const tor_addr_t *addr)
@@ -907,11 +904,9 @@ tor_addr_is_loopback(const tor_addr_t *addr)
       return (tor_addr_to_ipv4h(addr) & 0xff000000) == 0x7f000000;
     case AF_UNSPEC:
       return 0;
-      /* LCOV_EXCL_START */
     default:
       tor_fragile_assert();
       return 0;
-      /* LCOV_EXCL_STOP */
   }
 }
 
@@ -1031,10 +1026,8 @@ tor_addr_copy_tight(tor_addr_t *dest, const tor_addr_t *src)
       memcpy(dest->addr.in6_addr.s6_addr, src->addr.in6_addr.s6_addr, 16);
     case AF_UNSPEC:
       break;
-      // LCOV_EXCL_START
     default:
       tor_fragile_assert();
-      // LCOV_EXCL_STOP
     }
 }
 
@@ -1045,10 +1038,6 @@ tor_addr_copy_tight(tor_addr_t *dest, const tor_addr_t *src)
  * Different address families (IPv4 vs IPv6) are always considered unequal if
  * <b>how</b> is CMP_EXACT; otherwise, IPv6-mapped IPv4 addresses are
  * considered equivalent to their IPv4 equivalents.
- *
- * As a special case, all pointer-wise distinct AF_UNIX addresses are always
- * considered unequal since tor_addr_t currently does not contain the
- * information required to make the comparison.
  */
 int
 tor_addr_compare(const tor_addr_t *addr1, const tor_addr_t *addr2,
@@ -1107,7 +1096,6 @@ tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2,
       case AF_INET6: {
         if (mbits > 128)
           mbits = 128;
-
         const uint8_t *a1 = tor_addr_to_in6_addr8(addr1);
         const uint8_t *a2 = tor_addr_to_in6_addr8(addr2);
         const int bytes = mbits >> 3;
@@ -1122,29 +1110,9 @@ tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2,
           return 0;
         }
       }
-      case AF_UNIX:
-        /* HACKHACKHACKHACKHACK:
-         * tor_addr_t doesn't contain a copy of sun_path, so it's not
-         * possible to compare this at all.
-         *
-         * Since the only time we currently actually should be comparing
-         * 2 AF_UNIX addresses is when dealing with ISO_CLIENTADDR (which
-         * is disabled for AF_UNIX SocksPorts anyway), this just does
-         * a pointer comparison.
-         *
-         * See: #20261.
-         */
-        if (addr1 < addr2)
-          return -1;
-        else if (addr1 == addr2)
-          return 0;
-        else
-          return 1;
-        /* LCOV_EXCL_START */
       default:
         tor_fragile_assert();
         return 0;
-        /* LCOV_EXCL_STOP */
     }
   } else if (how == CMP_EXACT) {
     /* Unequal families and an exact comparison?  Stop now! */
@@ -1185,9 +1153,6 @@ tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2,
   }
 }
 
-/** Input for siphash, to produce some output for an unspec value. */
-static const uint32_t unspec_hash_input[] = { 0x4e4df09f, 0x92985342 };
-
 /** Return a hash code based on the address addr. DOCDOC extra */
 uint64_t
 tor_addr_hash(const tor_addr_t *addr)
@@ -1196,42 +1161,18 @@ tor_addr_hash(const tor_addr_t *addr)
   case AF_INET:
     return siphash24g(&addr->addr.in_addr.s_addr, 4);
   case AF_UNSPEC:
-    return siphash24g(unspec_hash_input, sizeof(unspec_hash_input));
+    return 0x4e4d5342;
   case AF_INET6:
     return siphash24g(&addr->addr.in6_addr.s6_addr, 16);
-    /* LCOV_EXCL_START */
   default:
     tor_fragile_assert();
     return 0;
-    /* LCOV_EXCL_STOP */
-  }
-}
-
-/** As tor_addr_hash, but use a particular siphash key. */
-uint64_t
-tor_addr_keyed_hash(const struct sipkey *key, const tor_addr_t *addr)
-{
-  /* This is duplicate code with tor_addr_hash, since this function needs to
-   * be backportable all the way to 0.2.9. */
-
-  switch (tor_addr_family(addr)) {
-  case AF_INET:
-    return siphash24(&addr->addr.in_addr.s_addr, 4, key);
-  case AF_UNSPEC:
-    return siphash24(unspec_hash_input, sizeof(unspec_hash_input), key);
-  case AF_INET6:
-    return siphash24(&addr->addr.in6_addr.s6_addr, 16, key);
-  default:
-    /* LCOV_EXCL_START */
-    tor_fragile_assert();
-    return 0;
-    /* LCOV_EXCL_STOP */
   }
 }
 
 /** Return a newly allocated string with a representation of <b>addr</b>. */
 char *
-tor_addr_to_str_dup(const tor_addr_t *addr)
+tor_dup_addr(const tor_addr_t *addr)
 {
   char buf[TOR_ADDR_BUF_LEN];
   if (tor_addr_to_str(buf, addr, sizeof(buf), 0)) {
@@ -1436,7 +1377,7 @@ get_interface_addresses_ifaddrs(int severity, sa_family_t family)
 
   return result;
 }
-#endif /* defined(HAVE_IFADDRS_TO_SMARTLIST) */
+#endif
 
 #ifdef HAVE_IP_ADAPTER_TO_SMARTLIST
 
@@ -1527,7 +1468,7 @@ get_interface_addresses_win32(int severity, sa_family_t family)
   return result;
 }
 
-#endif /* defined(HAVE_IP_ADAPTER_TO_SMARTLIST) */
+#endif
 
 #ifdef HAVE_IFCONF_TO_SMARTLIST
 
@@ -1540,18 +1481,6 @@ get_interface_addresses_win32(int severity, sa_family_t family)
 #define _SIZEOF_ADDR_IFREQ sizeof
 #endif
 
-/* Free ifc->ifc_buf safely. */
-static void
-ifconf_free_ifc_buf(struct ifconf *ifc)
-{
-  /* On macOS, tor_free() takes the address of ifc.ifc_buf, which leads to
-   * undefined behaviour, because pointer-to-pointers are expected to be
-   * aligned at 8-bytes, but the ifconf structure is packed.  So we use
-   * raw_free() instead. */
-  raw_free(ifc->ifc_buf);
-  ifc->ifc_buf = NULL;
-}
-
 /** Convert <b>*buf</b>, an ifreq structure array of size <b>buflen</b>,
  * into smartlist of <b>tor_addr_t</b> structures.
  */
@@ -1638,10 +1567,10 @@ get_interface_addresses_ioctl(int severity, sa_family_t family)
  done:
   if (fd >= 0)
     close(fd);
-  ifconf_free_ifc_buf(&ifc);
+  tor_free(ifc.ifc_buf);
   return result;
 }
-#endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */
+#endif
 
 /** Try to ask our network interfaces what addresses they are bound to.
  * Return a new smartlist of tor_addr_t on success, and NULL on failure.
@@ -1666,7 +1595,6 @@ get_interface_addresses_raw,(int severity, sa_family_t family))
     return result;
 #endif
   (void) severity;
-  (void) result;
   return NULL;
 }
 
@@ -1697,7 +1625,7 @@ get_interface_address6_via_udp_socket_hack,(int severity,
                                             sa_family_t family,
                                             tor_addr_t *addr))
 {
-  struct sockaddr_storage target_addr;
+  struct sockaddr_storage my_addr, target_addr;
   int sock=-1, r=-1;
   socklen_t addr_len;
 
@@ -1740,19 +1668,21 @@ get_interface_address6_via_udp_socket_hack,(int severity,
     goto err;
   }
 
-  if (tor_addr_from_getsockname(addr, sock) < 0) {
+  if (tor_getsockname(sock,(struct sockaddr*)&my_addr, &addr_len)) {
     int e = tor_socket_errno(sock);
     log_fn(severity, LD_NET, "getsockname() to determine interface failed: %s",
            tor_socket_strerror(e));
     goto err;
   }
 
-  if (tor_addr_is_loopback(addr) || tor_addr_is_multicast(addr)) {
-    log_fn(severity, LD_NET, "Address that we determined via UDP socket"
-           " magic is unsuitable for public comms.");
-  } else {
-    r=0;
-  }
+ if (tor_addr_from_sockaddr(addr, (struct sockaddr*)&my_addr, NULL) == 0) {
+    if (tor_addr_is_loopback(addr) || tor_addr_is_multicast(addr)) {
+      log_fn(severity, LD_NET, "Address that we determined via UDP socket"
+                               " magic is unsuitable for public comms.");
+    } else {
+      r=0;
+    }
+ }
 
  err:
   if (sock >= 0)
@@ -1794,14 +1724,14 @@ get_interface_address6,(int severity, sa_family_t family, tor_addr_t *addr))
       break;
   } SMARTLIST_FOREACH_END(a);
 
-  interface_address6_list_free(addrs);
+  free_interface_address6_list(addrs);
   return rv;
 }
 
 /** Free a smartlist of IP addresses returned by get_interface_address6_list.
  */
 void
-interface_address6_list_free_(smartlist_t *addrs)
+free_interface_address6_list(smartlist_t *addrs)
 {
   if (addrs != NULL) {
     SMARTLIST_FOREACH(addrs, tor_addr_t *, a, tor_free(a));
@@ -1816,12 +1746,11 @@ interface_address6_list_free_(smartlist_t *addrs)
  * An empty smartlist means that there are no addresses of the selected type
  * matching these criteria.
  * Returns NULL on failure.
- * Use interface_address6_list_free to free the returned list.
+ * Use free_interface_address6_list to free the returned list.
  */
-MOCK_IMPL(smartlist_t *,
-get_interface_address6_list,(int severity,
-                             sa_family_t family,
-                             int include_internal))
+MOCK_IMPL(smartlist_t *,get_interface_address6_list,(int severity,
+                                                     sa_family_t family,
+                                                     int include_internal))
 {
   smartlist_t *addrs;
   tor_addr_t addr;
@@ -1832,13 +1761,13 @@ get_interface_address6_list,(int severity,
     {
       if (tor_addr_is_loopback(a) ||
           tor_addr_is_multicast(a)) {
-        SMARTLIST_DEL_CURRENT_KEEPORDER(addrs, a);
+        SMARTLIST_DEL_CURRENT(addrs, a);
         tor_free(a);
         continue;
       }
 
       if (!include_internal && tor_addr_is_internal(a, 0)) {
-        SMARTLIST_DEL_CURRENT_KEEPORDER(addrs, a);
+        SMARTLIST_DEL_CURRENT(addrs, a);
         tor_free(a);
         continue;
       }
@@ -1880,7 +1809,7 @@ get_interface_address6_list,(int severity,
 
 /* ======
  * IPv4 helpers
- * XXXX IPv6 deprecate some of these.
+ * XXXX024 IPv6 deprecate some of these.
  */
 
 /** Given an address of the form "ip:port", try to divide it into its
@@ -2089,8 +2018,7 @@ parse_port_range(const char *port, uint16_t *port_min_out,
 
 /** Given an IPv4 in_addr struct *<b>in</b> (in network order, as usual),
  *  write it as a string into the <b>buf_len</b>-byte buffer in
- *  <b>buf</b>. Returns a non-negative integer on success.
- *  Returns -1 on failure.
+ *  <b>buf</b>.
  */
 int
 tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len)
@@ -2141,8 +2069,7 @@ get_interface_address,(int severity, uint32_t *addr))
 }
 
 /** Return true if we can tell that <b>name</b> is a canonical name for the
- * loopback address.  Return true also for *.local hostnames, which are
- * multicast DNS names for hosts on the local network. */
+ * loopback address. */
 int
 tor_addr_hostname_is_local(const char *name)
 {
@@ -2163,11 +2090,3 @@ tor_addr_port_new(const tor_addr_t *addr, uint16_t port)
   return ap;
 }
 
-/** Return true iff <a>a</b> and <b>b</b> are the same address and port */
-int
-tor_addr_port_eq(const tor_addr_port_t *a,
-                 const tor_addr_port_t *b)
-{
-  return tor_addr_eq(&a->addr, &b->addr) && a->port == b->port;
-}
-

src/common/address.h

@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2017, The Tor Project, Inc. */
+ * Copyright (c) 2007-2016, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -44,7 +44,7 @@
 #endif
 
 // TODO win32 specific includes
-#endif /* defined(ADDRESS_PRIVATE) */
+#endif // ADDRESS_PRIVATE
 
 /** The number of bits from an address to consider while doing a masked
  * comparison. */
@@ -74,8 +74,6 @@ typedef struct tor_addr_port_t
 #define TOR_ADDR_NULL {AF_UNSPEC, {0}}
 
 static inline const struct in6_addr *tor_addr_to_in6(const tor_addr_t *a);
-static inline const struct in6_addr *tor_addr_to_in6_assert(
-    const tor_addr_t *a);
 static inline uint32_t tor_addr_to_ipv4n(const tor_addr_t *a);
 static inline uint32_t tor_addr_to_ipv4h(const tor_addr_t *a);
 static inline uint32_t tor_addr_to_mapped_ipv4h(const tor_addr_t *a);
@@ -99,31 +97,21 @@ tor_addr_to_in6(const tor_addr_t *a)
   return a->family == AF_INET6 ? &a->addr.in6_addr : NULL;
 }
 
-/** As tor_addr_to_in6, but assert that the address truly is an IPv6
- * address. */
-static inline const struct in6_addr *
-tor_addr_to_in6_assert(const tor_addr_t *a)
-{
-  tor_assert(a->family == AF_INET6);
-  return &a->addr.in6_addr;
-}
-
 /** Given an IPv6 address <b>x</b>, yield it as an array of uint8_t.
  *
  * Requires that <b>x</b> is actually an IPv6 address.
  */
-#define tor_addr_to_in6_addr8(x) tor_addr_to_in6_assert(x)->s6_addr
-
+#define tor_addr_to_in6_addr8(x) tor_addr_to_in6(x)->s6_addr
 /** Given an IPv6 address <b>x</b>, yield it as an array of uint16_t.
  *
  * Requires that <b>x</b> is actually an IPv6 address.
  */
-#define tor_addr_to_in6_addr16(x) S6_ADDR16(*tor_addr_to_in6_assert(x))
+#define tor_addr_to_in6_addr16(x) S6_ADDR16(*tor_addr_to_in6(x))
 /** Given an IPv6 address <b>x</b>, yield it as an array of uint32_t.
  *
  * Requires that <b>x</b> is actually an IPv6 address.
  */
-#define tor_addr_to_in6_addr32(x) S6_ADDR32(*tor_addr_to_in6_assert(x))
+#define tor_addr_to_in6_addr32(x) S6_ADDR32(*tor_addr_to_in6(x))
 
 /** Return an IPv4 address in network order for <b>a</b>, or 0 if
  * <b>a</b> is not an IPv4 address. */
@@ -190,9 +178,8 @@ tor_addr_eq_ipv4h(const tor_addr_t *a, uint32_t u)
  */
 #define TOR_ADDR_BUF_LEN 48
 
-MOCK_DECL(int, tor_addr_lookup,(const char *name, uint16_t family,
-                                tor_addr_t *addr_out));
-char *tor_addr_to_str_dup(const tor_addr_t *addr) ATTR_MALLOC;
+int tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr_out);
+char *tor_dup_addr(const tor_addr_t *addr) ATTR_MALLOC;
 
 /** Wrapper function of fmt_addr_impl(). It does not decorate IPv6
  *  addresses. */
@@ -206,9 +193,7 @@ const char * fmt_addr32(uint32_t addr);
 
 MOCK_DECL(int,get_interface_address6,(int severity, sa_family_t family,
 tor_addr_t *addr));
-void interface_address6_list_free_(smartlist_t * addrs);// XXXX
-#define interface_address6_list_free(addrs) \
-  FREE_AND_NULL(smartlist_t, interface_address6_list_free_, (addrs))
+void free_interface_address6_list(smartlist_t * addrs);
 MOCK_DECL(smartlist_t *,get_interface_address6_list,(int severity,
                                                      sa_family_t family,
                                                      int include_internal));
@@ -231,8 +216,6 @@ int tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2,
 #define tor_addr_eq(a,b) (0==tor_addr_compare((a),(b),CMP_EXACT))
 
 uint64_t tor_addr_hash(const tor_addr_t *addr);
-struct sipkey;
-uint64_t tor_addr_keyed_hash(const struct sipkey *key, const tor_addr_t *addr);
 int tor_addr_is_v4(const tor_addr_t *addr);
 int tor_addr_is_internal_(const tor_addr_t *ip, int for_listening,
                           const char *filename, int lineno);
@@ -325,8 +308,13 @@ int addr_mask_get_bits(uint32_t mask);
 int tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len);
 char *tor_dup_ip(uint32_t addr) ATTR_MALLOC;
 MOCK_DECL(int,get_interface_address,(int severity, uint32_t *addr));
-#define interface_address_list_free(lst)\
-  interface_address6_list_free(lst)
+/** Free a smartlist of IP addresses returned by get_interface_address_list.
+ */
+static inline void
+free_interface_address_list(smartlist_t *addrs)
+{
+  free_interface_address6_list(addrs);
+}
 /** Return a smartlist of the IPv4 addresses of all interfaces on the server.
  * Excludes loopback and multicast addresses. Only includes internal addresses
  * if include_internal is true. (Note that a relay behind NAT may use an
@@ -342,8 +330,6 @@ get_interface_address_list(int severity, int include_internal)
 }
 
 tor_addr_port_t *tor_addr_port_new(const tor_addr_t *addr, uint16_t port);
-int tor_addr_port_eq(const tor_addr_port_t *a,
-                     const tor_addr_port_t *b);
 
 #ifdef ADDRESS_PRIVATE
 MOCK_DECL(smartlist_t *,get_interface_addresses_raw,(int severity,
@@ -357,23 +343,23 @@ STATIC smartlist_t *ifaddrs_to_smartlist(const struct ifaddrs *ifa,
                                          sa_family_t family);
 STATIC smartlist_t *get_interface_addresses_ifaddrs(int severity,
                                                     sa_family_t family);
-#endif /* defined(HAVE_IFADDRS_TO_SMARTLIST) */
+#endif
 
 #ifdef HAVE_IP_ADAPTER_TO_SMARTLIST
 STATIC smartlist_t *ip_adapter_addresses_to_smartlist(
                                         const IP_ADAPTER_ADDRESSES *addresses);
 STATIC smartlist_t *get_interface_addresses_win32(int severity,
                                                   sa_family_t family);
-#endif /* defined(HAVE_IP_ADAPTER_TO_SMARTLIST) */
+#endif
 
 #ifdef HAVE_IFCONF_TO_SMARTLIST
 STATIC smartlist_t *ifreq_to_smartlist(char *ifr,
                                        size_t buflen);
 STATIC smartlist_t *get_interface_addresses_ioctl(int severity,
                                                   sa_family_t family);
-#endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */
+#endif
 
-#endif /* defined(ADDRESS_PRIVATE) */
+#endif // ADDRESS_PRIVATE
 
-#endif /* !defined(TOR_ADDRESS_H) */
+#endif
 

src/common/address_set.c

@@ -1,129 +0,0 @@
-/* Copyright (c) 2018, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file address_set.c
- * \brief Implementation for a set of addresses.
- *
- * This module was first written on a semi-emergency basis to improve the
- * robustness of the anti-DoS module.  As such, it's written in a pretty
- * conservative way, and should be susceptible to improvement later on.
- **/
-
-#include "orconfig.h"
-#include "address_set.h"
-#include "address.h"
-#include "compat.h"
-#include "container.h"
-#include "crypto_rand.h"
-#include "util.h"
-#include "siphash.h"
-
-/** How many 64-bit siphash values to extract per address */
-#define N_HASHES 2
-/** How many bloom-filter bits we set per address. This is twice the N_HASHES
- * value, since we split the siphash output into two 32-bit values. */
-#define N_BITS_PER_ITEM (N_HASHES * 2)
-
-/* XXXX This code is largely duplicated with digestset_t.  We should merge
- * them together into a common bloom-filter implementation.  I'm keeping
- * them separate for now, though, since this module needs to be backported
- * all the way to 0.2.9.
- *
- * The main difference between digestset_t and this code is that we use
- * independent siphashes rather than messing around with bit-shifts.  The
- * approach here is probably more sound, and we should prefer it if&when we
- * unify the implementations.
- */
-
-struct address_set_t {
-  /** siphash keys to make N_HASHES independent hashes for each address. */
-  struct sipkey key[N_HASHES];
-  int mask; /**< One less than the number of bits in <b>ba</b>; always one less
-             * than a power of two. */
-  bitarray_t *ba; /**< A bit array to implement the Bloom filter. */
-};
-
-/**
- * Allocate and return an address_set, suitable for holding up to
- * <b>max_address_guess</b> distinct values.
- */
-address_set_t *
-address_set_new(int max_addresses_guess)
-{
-  /* See digestset_new() for rationale on this equation. */
-  int n_bits = 1u << (tor_log2(max_addresses_guess)+5);
-
-  address_set_t *set = tor_malloc_zero(sizeof(address_set_t));
-  set->mask = n_bits - 1;
-  set->ba = bitarray_init_zero(n_bits);
-  crypto_rand((char*) set->key, sizeof(set->key));
-
-  return set;
-}
-
-/**
- * Release all storage associated with <b>set</b>.
- */
-void
-address_set_free(address_set_t *set)
-{
-  if (! set)
-    return;
-
-  bitarray_free(set->ba);
-  tor_free(set);
-}
-
-/** Yield the bit index corresponding to 'val' for set. */
-#define BIT(set, val) ((val) & (set)->mask)
-
-/**
- * Add <b>addr</b> to <b>set</b>.
- *
- * All future queries for <b>addr</b> in set will return true. Removing
- * items is not possible.
- */
-void
-address_set_add(address_set_t *set, const struct tor_addr_t *addr)
-{
-  int i;
-  for (i = 0; i < N_HASHES; ++i) {
-    uint64_t h = tor_addr_keyed_hash(&set->key[i], addr);
-    uint32_t high_bits = (uint32_t)(h >> 32);
-    uint32_t low_bits = (uint32_t)(h);
-    bitarray_set(set->ba, BIT(set, high_bits));
-    bitarray_set(set->ba, BIT(set, low_bits));
-  }
-}
-
-/** As address_set_add(), but take an ipv4 address in host order. */
-void
-address_set_add_ipv4h(address_set_t *set, uint32_t addr)
-{
-  tor_addr_t a;
-  tor_addr_from_ipv4h(&a, addr);
-  address_set_add(set, &a);
-}
-
-/**
- * Return true if <b>addr</b> is a member of <b>set</b>.  (And probably,
- * return false if <b>addr</b> is not a member of set.)
- */
-int
-address_set_probably_contains(address_set_t *set,
-                              const struct tor_addr_t *addr)
-{
-  int i, matches = 0;
-  for (i = 0; i < N_HASHES; ++i) {
-    uint64_t h = tor_addr_keyed_hash(&set->key[i], addr);
-    uint32_t high_bits = (uint32_t)(h >> 32);
-    uint32_t low_bits = (uint32_t)(h);
-    // Note that !! is necessary here, since bitarray_is_set does not
-    // necessarily return 1 on true.
-    matches += !! bitarray_is_set(set->ba, BIT(set, high_bits));
-    matches += !! bitarray_is_set(set->ba, BIT(set, low_bits));
-  }
-  return matches == N_BITS_PER_ITEM;
-}
-

src/common/address_set.h

@@ -1,35 +0,0 @@
-/* Copyright (c) 2018, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file address_set.h
- * \brief Types to handle sets of addresses.
- *
- * This module was first written on a semi-emergency basis to improve the
- * robustness of the anti-DoS module.  As such, it's written in a pretty
- * conservative way, and should be susceptible to improvement later on.
- **/
-
-#ifndef TOR_ADDRESS_SET_H
-#define TOR_ADDRESS_SET_H
-
-#include "orconfig.h"
-#include "torint.h"
-
-/**
- * An address_set_t represents a set of tor_addr_t values. The implementation
- * is probabilistic: false negatives cannot occur but false positives are
- * possible.
- */
-typedef struct address_set_t address_set_t;
-struct tor_addr_t;
-
-address_set_t *address_set_new(int max_addresses_guess);
-void address_set_free(address_set_t *set);
-void address_set_add(address_set_t *set, const struct tor_addr_t *addr);
-void address_set_add_ipv4h(address_set_t *set, uint32_t addr);
-int address_set_probably_contains(address_set_t *set,
-                                  const struct tor_addr_t *addr);
-
-#endif
-

src/common/aes.c

@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2017, The Tor Project, Inc. */
+ * Copyright (c) 2007-2016, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,15 +16,25 @@
   #include <ws2tcpip.h>
 #endif
 
-#include "compat_openssl.h"
 #include <openssl/opensslv.h>
-#include "crypto_openssl_mgt.h"
+#include "crypto.h"
 
 #if OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,0,0)
 #error "We require OpenSSL >= 1.0.0"
 #endif
 
-DISABLE_GCC_WARNING(redundant-decls)
+#ifdef __GNUC__
+#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+#endif
+
+#if __GNUC__ && GCC_VERSION >= 402
+#if GCC_VERSION >= 406
+#pragma GCC diagnostic push
+#endif
+/* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in
+ * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */
+#pragma GCC diagnostic ignored "-Wredundant-decls"
+#endif
 
 #include <assert.h>
 #include <stdlib.h>
@@ -34,7 +44,13 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/engine.h>
 #include <openssl/modes.h>
 
-ENABLE_GCC_WARNING(redundant-decls)
+#if __GNUC__ && GCC_VERSION >= 402
+#if GCC_VERSION >= 406
+#pragma GCC diagnostic pop
+#else
+#pragma GCC diagnostic warning "-Wredundant-decls"
+#endif
+#endif
 
 #include "compat.h"
 #include "aes.h"
@@ -49,7 +65,7 @@ ENABLE_GCC_WARNING(redundant-decls)
 
 /* We have five strategies for implementing AES counter mode.
  *
- * Best with x86 and x86_64: Use EVP_aes_*_ctr() and EVP_EncryptUpdate().
+ * Best with x86 and x86_64: Use EVP_aes_ctr128() and EVP_EncryptUpdate().
  * This is possible with OpenSSL 1.0.1, where the counter-mode implementation
  * can use bit-sliced or vectorized AES or AESNI as appropriate.
  *
@@ -67,11 +83,11 @@ ENABLE_GCC_WARNING(redundant-decls)
 #elif OPENSSL_VERSION_NUMBER >= OPENSSL_V_NOPATCH(1,0,1) &&               \
   (defined(__i386) || defined(__i386__) || defined(_M_IX86) ||          \
    defined(__x86_64) || defined(__x86_64__) ||                          \
-   defined(_M_AMD64) || defined(_M_X64) || defined(__INTEL__))
+   defined(_M_AMD64) || defined(_M_X64) || defined(__INTEL__))          \
 
 #define USE_EVP_AES_CTR
 
-#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_NOPATCH(1,1,0) || ... */
+#endif
 
 /* We have 2 strategies for getting the AES block cipher: Via OpenSSL's
  * AES_encrypt function, or via OpenSSL's EVP_EncryptUpdate function.
@@ -97,30 +113,20 @@ ENABLE_GCC_WARNING(redundant-decls)
 /* We don't actually define the struct here. */
 
 aes_cnt_cipher_t *
-aes_new_cipher(const uint8_t *key, const uint8_t *iv, int key_bits)
+aes_new_cipher(const char *key, const char *iv)
 {
   EVP_CIPHER_CTX *cipher = EVP_CIPHER_CTX_new();
-  const EVP_CIPHER *c;
-  switch (key_bits) {
-    case 128: c = EVP_aes_128_ctr(); break;
-    case 192: c = EVP_aes_192_ctr(); break;
-    case 256: c = EVP_aes_256_ctr(); break;
-    default: tor_assert(0); // LCOV_EXCL_LINE
-  }
-  EVP_EncryptInit(cipher, c, key, iv);
+  EVP_EncryptInit(cipher, EVP_aes_128_ctr(),
+                  (const unsigned char*)key, (const unsigned char *)iv);
   return (aes_cnt_cipher_t *) cipher;
 }
 void
-aes_cipher_free_(aes_cnt_cipher_t *cipher_)
+aes_cipher_free(aes_cnt_cipher_t *cipher_)
 {
   if (!cipher_)
     return;
   EVP_CIPHER_CTX *cipher = (EVP_CIPHER_CTX *) cipher_;
-#ifdef OPENSSL_1_1_API
-  EVP_CIPHER_CTX_reset(cipher);
-#else
   EVP_CIPHER_CTX_cleanup(cipher);
-#endif
   EVP_CIPHER_CTX_free(cipher);
 }
 void
@@ -147,7 +153,7 @@ evaluate_ctr_for_aes(void)
 {
   return 0;
 }
-#else /* !(defined(USE_EVP_AES_CTR)) */
+#else
 
 /*======================================================================*/
 /* Interface to AES code, and counter implementation */
@@ -168,7 +174,7 @@ struct aes_cnt_cipher {
   uint32_t counter2;
   uint32_t counter1;
   uint32_t counter0;
-#endif /* !defined(WORDS_BIGENDIAN) */
+#endif
 
   union {
     /** The counter, in big-endian order, as bytes. */
@@ -217,7 +223,7 @@ evaluate_evp_for_aes(int force_val)
     log_info(LD_CRYPTO, "No AES engine found; using AES_* functions.");
     should_use_EVP = 0;
   }
-#endif /* defined(DISABLE_ENGINES) */
+#endif
 
   return 0;
 }
@@ -256,11 +262,9 @@ evaluate_ctr_for_aes(void)
 
   if (fast_memneq(output, encrypt_zero, 16)) {
     /* Counter mode is buggy */
-    /* LCOV_EXCL_START */
     log_err(LD_CRYPTO, "This OpenSSL has a buggy version of counter mode; "
                   "quitting tor.");
-    exit(1); // exit ok: openssl is broken.
-    /* LCOV_EXCL_STOP */
+    exit(1);
   }
   return 0;
 }
@@ -271,20 +275,20 @@ evaluate_ctr_for_aes(void)
 #define COUNTER(c, n) ((c)->counter ## n)
 #endif
 
-static void aes_set_key(aes_cnt_cipher_t *cipher, const uint8_t *key,
+static void aes_set_key(aes_cnt_cipher_t *cipher, const char *key,
                         int key_bits);
-static void aes_set_iv(aes_cnt_cipher_t *cipher, const uint8_t *iv);
+static void aes_set_iv(aes_cnt_cipher_t *cipher, const char *iv);
 
 /**
  * Return a newly allocated counter-mode AES128 cipher implementation,
  * using the 128-bit key <b>key</b> and the 128-bit IV <b>iv</b>.
  */
 aes_cnt_cipher_t*
-aes_new_cipher(const uint8_t *key, const uint8_t *iv, int bits)
+aes_new_cipher(const char *key, const char *iv)
 {
   aes_cnt_cipher_t* result = tor_malloc_zero(sizeof(aes_cnt_cipher_t));
 
-  aes_set_key(result, key, bits);
+  aes_set_key(result, key, 128);
   aes_set_iv(result, iv);
 
   return result;
@@ -295,7 +299,7 @@ aes_new_cipher(const uint8_t *key, const uint8_t *iv, int bits)
  * the counter to 0.
  */
 static void
-aes_set_key(aes_cnt_cipher_t *cipher, const uint8_t *key, int key_bits)
+aes_set_key(aes_cnt_cipher_t *cipher, const char *key, int key_bits)
 {
   if (should_use_EVP) {
     const EVP_CIPHER *c = 0;
@@ -303,12 +307,12 @@ aes_set_key(aes_cnt_cipher_t *cipher, const uint8_t *key, int key_bits)
       case 128: c = EVP_aes_128_ecb(); break;
       case 192: c = EVP_aes_192_ecb(); break;
       case 256: c = EVP_aes_256_ecb(); break;
-      default: tor_assert(0); // LCOV_EXCL_LINE
+      default: tor_assert(0);
     }
-    EVP_EncryptInit(&cipher->key.evp, c, key, NULL);
+    EVP_EncryptInit(&cipher->key.evp, c, (const unsigned char*)key, NULL);
     cipher->using_evp = 1;
   } else {
-    AES_set_encrypt_key(key, key_bits,&cipher->key.aes);
+    AES_set_encrypt_key((const unsigned char *)key, key_bits,&cipher->key.aes);
     cipher->using_evp = 0;
   }
 
@@ -317,7 +321,7 @@ aes_set_key(aes_cnt_cipher_t *cipher, const uint8_t *key, int key_bits)
   cipher->counter1 = 0;
   cipher->counter2 = 0;
   cipher->counter3 = 0;
-#endif /* defined(USING_COUNTER_VARS) */
+#endif
 
   memset(cipher->ctr_buf.buf, 0, sizeof(cipher->ctr_buf.buf));
 
@@ -329,7 +333,7 @@ aes_set_key(aes_cnt_cipher_t *cipher, const uint8_t *key, int key_bits)
 /** Release storage held by <b>cipher</b>
  */
 void
-aes_cipher_free_(aes_cnt_cipher_t *cipher)
+aes_cipher_free(aes_cnt_cipher_t *cipher)
 {
   if (!cipher)
     return;
@@ -346,7 +350,7 @@ aes_cipher_free_(aes_cnt_cipher_t *cipher)
   STMT_END
 #else
 #define UPDATE_CTR_BUF(c, n)
-#endif /* defined(USING_COUNTER_VARS) */
+#endif
 
 /* Helper function to use EVP with openssl's counter-mode wrapper. */
 static void
@@ -366,8 +370,6 @@ evp_block128_fn(const uint8_t in[16],
 void
 aes_crypt_inplace(aes_cnt_cipher_t *cipher, char *data, size_t len)
 {
-  /* Note that the "128" below refers to the length of the counter,
-   * not the length of the AES key. */
   if (cipher->using_evp) {
     /* In openssl 1.0.0, there's an if'd out EVP_aes_128_ctr in evp.h.  If
      * it weren't disabled, it might be better just to use that.
@@ -394,17 +396,17 @@ aes_crypt_inplace(aes_cnt_cipher_t *cipher, char *data, size_t len)
 /** Reset the 128-bit counter of <b>cipher</b> to the 16-bit big-endian value
  * in <b>iv</b>. */
 static void
-aes_set_iv(aes_cnt_cipher_t *cipher, const uint8_t *iv)
+aes_set_iv(aes_cnt_cipher_t *cipher, const char *iv)
 {
 #ifdef USING_COUNTER_VARS
   cipher->counter3 = ntohl(get_uint32(iv));
   cipher->counter2 = ntohl(get_uint32(iv+4));
   cipher->counter1 = ntohl(get_uint32(iv+8));
   cipher->counter0 = ntohl(get_uint32(iv+12));
-#endif /* defined(USING_COUNTER_VARS) */
+#endif
   cipher->pos = 0;
   memcpy(cipher->ctr_buf.buf, iv, 16);
 }
 
-#endif /* defined(USE_EVP_AES_CTR) */
+#endif
 

src/common/aes.h

@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2017, The Tor Project, Inc. */
+ * Copyright (c) 2007-2016, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Implements a minimal interface to counter-mode AES. */
@@ -15,15 +15,12 @@
 
 typedef struct aes_cnt_cipher aes_cnt_cipher_t;
 
-aes_cnt_cipher_t* aes_new_cipher(const uint8_t *key, const uint8_t *iv,
-                                 int key_bits);
-void aes_cipher_free_(aes_cnt_cipher_t *cipher);
-#define aes_cipher_free(cipher) \
-  FREE_AND_NULL(aes_cnt_cipher_t, aes_cipher_free_, (cipher))
+aes_cnt_cipher_t* aes_new_cipher(const char *key, const char *iv);
+void aes_cipher_free(aes_cnt_cipher_t *cipher);
 void aes_crypt_inplace(aes_cnt_cipher_t *cipher, char *data, size_t len);
 
 int evaluate_evp_for_aes(int force_value);
 int evaluate_ctr_for_aes(void);
 
-#endif /* !defined(TOR_AES_H) */
+#endif
 

src/common/backtrace.c

@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2017, The Tor Project, Inc. */
+/* Copyright (c) 2013-2016, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,9 @@
  * detect crashes.
  */
 
+#define __USE_GNU
+#define _GNU_SOURCE 1
+
 #include "orconfig.h"
 #include "compat.h"
 #include "util.h"
@@ -37,7 +40,7 @@
 #include <sys/ucontext.h>
 #elif defined(HAVE_UCONTEXT_H)
 #include <ucontext.h>
-#endif /* defined(HAVE_CYGWIN_SIGNAL_H) || ... */
+#endif
 
 #define EXPOSE_CLEAN_BACKTRACE
 #include "backtrace.h"
@@ -76,21 +79,21 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx)
 #ifdef PC_FROM_UCONTEXT
 #if defined(__linux__)
   const size_t n = 1;
-#elif defined(__darwin__) || defined(__APPLE__) || defined(OpenBSD) \
+#elif defined(__darwin__) || defined(__APPLE__) || defined(__OpenBSD__) \
   || defined(__FreeBSD__)
   const size_t n = 2;
 #else
   const size_t n = 1;
-#endif /* defined(__linux__) || ... */
+#endif
   if (depth <= n)
     return;
 
   stack[n] = (void*) ctx->PC_FROM_UCONTEXT;
-#else /* !(defined(PC_FROM_UCONTEXT)) */
+#else
   (void) depth;
   (void) ctx;
   (void) stack;
-#endif /* defined(PC_FROM_UCONTEXT) */
+#endif
 }
 
 /** Log a message <b>msg</b> at <b>severity</b> in <b>domain</b>, and follow
@@ -109,15 +112,13 @@ log_backtrace(int severity, int domain, const char *msg)
 
   tor_log(severity, domain, "%s. Stack trace:", msg);
   if (!symbols) {
-    /* LCOV_EXCL_START -- we can't provoke this. */
     tor_log(severity, domain, "    Unable to generate backtrace.");
     goto done;
-    /* LCOV_EXCL_STOP */
   }
   for (i=0; i < depth; ++i) {
     tor_log(severity, domain, "    %s", symbols[i]);
   }
-  raw_free(symbols);
+  free(symbols);
 
  done:
   tor_mutex_release(&cb_buf_mutex);
@@ -175,10 +176,8 @@ install_bt_handler(void)
 
   for (i = 0; trap_signals[i] >= 0; ++i) {
     if (sigaction(trap_signals[i], &sa, NULL) == -1) {
-      /* LCOV_EXCL_START */
       log_warn(LD_BUG, "Sigaction failed: %s", strerror(errno));
       rv = -1;
-      /* LCOV_EXCL_STOP */
     }
   }
 
@@ -190,7 +189,7 @@ install_bt_handler(void)
     size_t depth = backtrace(cb_buf, MAX_DEPTH);
     symbols = backtrace_symbols(cb_buf, (int) depth);
     if (symbols)
-      raw_free(symbols);
+      free(symbols);
   }
 
   return rv;
@@ -202,7 +201,7 @@ remove_bt_handler(void)
 {
   tor_mutex_uninit(&cb_buf_mutex);
 }
-#endif /* defined(USE_BACKTRACE) */
+#endif
 
 #ifdef NO_BACKTRACE_IMPL
 void
@@ -221,7 +220,7 @@ static void
 remove_bt_handler(void)
 {
 }
-#endif /* defined(NO_BACKTRACE_IMPL) */
+#endif
 
 /** Set up code to handle generating error messages on crashes. */
 int

src/common/backtrace.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2017, The Tor Project, Inc. */
+/* Copyright (c) 2013-2016, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_BACKTRACE_H
@@ -15,7 +15,7 @@ void clean_up_backtrace_handler(void);
   defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION)
 void clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx);
 #endif
-#endif /* defined(EXPOSE_CLEAN_BACKTRACE) */
+#endif
 
-#endif /* !defined(TOR_BACKTRACE_H) */
+#endif
 

src/common/buffers.h

@@ -1,131 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2017, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file buffers.h
- * \brief Header file for buffers.c.
- **/
-
-#ifndef TOR_BUFFERS_H
-#define TOR_BUFFERS_H
-
-#include "compat.h"
-#include "torint.h"
-#include "testsupport.h"
-
-typedef struct buf_t buf_t;
-
-struct tor_compress_state_t;
-
-buf_t *buf_new(void);
-buf_t *buf_new_with_capacity(size_t size);
-size_t buf_get_default_chunk_size(const buf_t *buf);
-void buf_free_(buf_t *buf);
-#define buf_free(b) FREE_AND_NULL(buf_t, buf_free_, (b))
-void buf_clear(buf_t *buf);
-buf_t *buf_copy(const buf_t *buf);
-
-MOCK_DECL(size_t, buf_datalen, (const buf_t *buf));
-size_t buf_allocation(const buf_t *buf);
-size_t buf_slack(const buf_t *buf);
-
-uint32_t buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now);
-size_t buf_get_total_allocation(void);
-
-int buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
-                         int *reached_eof,
-                         int *socket_error);
-
-int buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
-                        size_t *buf_flushlen);
-
-int buf_add(buf_t *buf, const char *string, size_t string_len);
-void buf_add_string(buf_t *buf, const char *string);
-void buf_add_printf(buf_t *buf, const char *format, ...)
-  CHECK_PRINTF(2, 3);
-void buf_add_vprintf(buf_t *buf, const char *format, va_list args)
-  CHECK_PRINTF(2, 0);
-int buf_add_compress(buf_t *buf, struct tor_compress_state_t *state,
-                          const char *data, size_t data_len, int done);
-int buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen);
-void buf_move_all(buf_t *buf_out, buf_t *buf_in);
-void buf_peek(const buf_t *buf, char *string, size_t string_len);
-void buf_drain(buf_t *buf, size_t n);
-int buf_get_bytes(buf_t *buf, char *string, size_t string_len);
-int buf_get_line(buf_t *buf, char *data_out, size_t *data_len);
-
-#define PEEK_BUF_STARTSWITH_MAX 16
-int buf_peek_startswith(const buf_t *buf, const char *cmd);
-
-int buf_set_to_copy(buf_t **output,
-                    const buf_t *input);
-
-void buf_assert_ok(buf_t *buf);
-
-int buf_find_string_offset(const buf_t *buf, const char *s, size_t n);
-void buf_pullup(buf_t *buf, size_t bytes,
-                const char **head_out, size_t *len_out);
-char *buf_extract(buf_t *buf, size_t *sz_out);
-
-#ifdef BUFFERS_PRIVATE
-#ifdef TOR_UNIT_TESTS
-buf_t *buf_new_with_data(const char *cp, size_t sz);
-#endif
-size_t buf_preferred_chunk_size(size_t target);
-
-#define DEBUG_CHUNK_ALLOC
-/** A single chunk on a buffer. */
-typedef struct chunk_t {
-  struct chunk_t *next; /**< The next chunk on the buffer. */
-  size_t datalen; /**< The number of bytes stored in this chunk */
-  size_t memlen; /**< The number of usable bytes of storage in <b>mem</b>. */
-#ifdef DEBUG_CHUNK_ALLOC
-  size_t DBG_alloc;
-#endif
-  char *data; /**< A pointer to the first byte of data stored in <b>mem</b>. */
-  uint32_t inserted_time; /**< Timestamp when this chunk was inserted. */
-  char mem[FLEXIBLE_ARRAY_MEMBER]; /**< The actual memory used for storage in
-                * this chunk. */
-} chunk_t;
-
-/** Magic value for buf_t.magic, to catch pointer errors. */
-#define BUFFER_MAGIC 0xB0FFF312u
-/** A resizeable buffer, optimized for reading and writing. */
-struct buf_t {
-  uint32_t magic; /**< Magic cookie for debugging: Must be set to
-                   *   BUFFER_MAGIC. */
-  size_t datalen; /**< How many bytes is this buffer holding right now? */
-  size_t default_chunk_size; /**< Don't allocate any chunks smaller than
-                              * this for this buffer. */
-  chunk_t *head; /**< First chunk in the list, or NULL for none. */
-  chunk_t *tail; /**< Last chunk in the list, or NULL for none. */
-};
-
-chunk_t *buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped);
-/** If a read onto the end of a chunk would be smaller than this number, then
- * just start a new chunk. */
-#define MIN_READ_LEN 8
-
-/** Return the number of bytes that can be written onto <b>chunk</b> without
- * running out of space. */
-static inline size_t
-CHUNK_REMAINING_CAPACITY(const chunk_t *chunk)
-{
-  return (chunk->mem + chunk->memlen) - (chunk->data + chunk->datalen);
-}
-
-/** Return the next character in <b>chunk</b> onto which data can be appended.
- * If the chunk is full, this might be off the end of chunk->mem. */
-static inline char *
-CHUNK_WRITE_PTR(chunk_t *chunk)
-{
-  return chunk->data + chunk->datalen;
-}
-
-#endif /* defined(BUFFERS_PRIVATE) */
-
-#endif /* !defined(TOR_BUFFERS_H) */
-

src/common/buffers_tls.c

@@ -1,179 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2017, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#define BUFFERS_PRIVATE
-#include "orconfig.h"
-#include <stddef.h>
-#include "buffers.h"
-#include "buffers_tls.h"
-#include "compat.h"
-#include "compress.h"
-#include "util.h"
-#include "torint.h"
-#include "torlog.h"
-#include "tortls.h"
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-/** As read_to_chunk(), but return (negative) error code on error, blocking,
- * or TLS, and the number of bytes read otherwise. */
-static inline int
-read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls,
-                  size_t at_most)
-{
-  int read_result;
-
-  tor_assert(CHUNK_REMAINING_CAPACITY(chunk) >= at_most);
-  read_result = tor_tls_read(tls, CHUNK_WRITE_PTR(chunk), at_most);
-  if (read_result < 0)
-    return read_result;
-  buf->datalen += read_result;
-  chunk->datalen += read_result;
-  return read_result;
-}
-
-/** As read_to_buf, but reads from a TLS connection, and returns a TLS
- * status value rather than the number of bytes read.
- *
- * Using TLS on OR connections complicates matters in two ways.
- *
- * First, a TLS stream has its own read buffer independent of the
- * connection's read buffer.  (TLS needs to read an entire frame from
- * the network before it can decrypt any data.  Thus, trying to read 1
- * byte from TLS can require that several KB be read from the network
- * and decrypted.  The extra data is stored in TLS's decrypt buffer.)
- * Because the data hasn't been read by Tor (it's still inside the TLS),
- * this means that sometimes a connection "has stuff to read" even when
- * poll() didn't return POLLIN. The tor_tls_get_pending_bytes function is
- * used in connection.c to detect TLS objects with non-empty internal
- * buffers and read from them again.
- *
- * Second, the TLS stream's events do not correspond directly to network
- * events: sometimes, before a TLS stream can read, the network must be
- * ready to write -- or vice versa.
- */
-int
-buf_read_from_tls(buf_t *buf, tor_tls_t *tls, size_t at_most)
-{
-  int r = 0;
-  size_t total_read = 0;
-
-  check_no_tls_errors();
-
-  if (BUG(buf->datalen >= INT_MAX))
-    return -1;
-  if (BUG(buf->datalen >= INT_MAX - at_most))
-    return -1;
-
-  while (at_most > total_read) {
-    size_t readlen = at_most - total_read;
-    chunk_t *chunk;
-    if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) {
-      chunk = buf_add_chunk_with_capacity(buf, at_most, 1);
-      if (readlen > chunk->memlen)
-        readlen = chunk->memlen;
-    } else {
-      size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail);
-      chunk = buf->tail;
-      if (cap < readlen)
-        readlen = cap;
-    }
-
-    r = read_to_chunk_tls(buf, chunk, tls, readlen);
-    if (r < 0)
-      return r; /* Error */
-    tor_assert(total_read+r < INT_MAX);
-    total_read += r;
-    if ((size_t)r < readlen) /* eof, block, or no more to read. */
-      break;
-  }
-  return (int)total_read;
-}
-
-/** Helper for buf_flush_to_tls(): try to write <b>sz</b> bytes from chunk
- * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  (Tries to write
- * more if there is a forced pending write size.)  On success, deduct the
- * bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
- * written on success, and a TOR_TLS error code on failure or blocking.
- */
-static inline int
-flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk,
-                size_t sz, size_t *buf_flushlen)
-{
-  int r;
-  size_t forced;
-  char *data;
-
-  forced = tor_tls_get_forced_write_size(tls);
-  if (forced > sz)
-    sz = forced;
-  if (chunk) {
-    data = chunk->data;
-    tor_assert(sz <= chunk->datalen);
-  } else {
-    data = NULL;
-    tor_assert(sz == 0);
-  }
-  r = tor_tls_write(tls, data, sz);
-  if (r < 0)
-    return r;
-  if (*buf_flushlen > (size_t)r)
-    *buf_flushlen -= r;
-  else
-    *buf_flushlen = 0;
-  buf_drain(buf, r);
-  log_debug(LD_NET,"flushed %d bytes, %d ready to flush, %d remain.",
-            r,(int)*buf_flushlen,(int)buf->datalen);
-  return r;
-}
-
-/** As buf_flush_to_socket(), but writes data to a TLS connection.  Can write
- * more than <b>flushlen</b> bytes.
- */
-int
-buf_flush_to_tls(buf_t *buf, tor_tls_t *tls, size_t flushlen,
-              size_t *buf_flushlen)
-{
-  int r;
-  size_t flushed = 0;
-  ssize_t sz;
-  tor_assert(buf_flushlen);
-  if (BUG(*buf_flushlen > buf->datalen)) {
-    *buf_flushlen = buf->datalen;
-  }
-  if (BUG(flushlen > *buf_flushlen)) {
-    flushlen = *buf_flushlen;
-  }
-  sz = (ssize_t) flushlen;
-
-  /* we want to let tls write even if flushlen is zero, because it might
-   * have a partial record pending */
-  check_no_tls_errors();
-
-  do {
-    size_t flushlen0;
-    if (buf->head) {
-      if ((ssize_t)buf->head->datalen >= sz)
-        flushlen0 = sz;
-      else
-        flushlen0 = buf->head->datalen;
-    } else {
-      flushlen0 = 0;
-    }
-
-    r = flush_chunk_tls(tls, buf, buf->head, flushlen0, buf_flushlen);
-    if (r < 0)
-      return r;
-    flushed += r;
-    sz -= r;
-    if (r == 0) /* Can't flush any more now. */
-      break;
-  } while (sz > 0);
-  tor_assert(flushed < INT_MAX);
-  return (int)flushed;
-}
-

src/common/buffers_tls.h

@@ -1,19 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2017, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#ifndef TOR_BUFFERS_TLS_H
-#define TOR_BUFFERS_TLS_H
-
-struct buf_t;
-struct tor_tls_t;
-
-int buf_read_from_tls(struct buf_t *buf,
-                      struct tor_tls_t *tls, size_t at_most);
-int buf_flush_to_tls(struct buf_t *buf, struct tor_tls_t *tls,
-                     size_t sz, size_t *buf_flushlen);
-
-#endif /* !defined(TOR_BUFFERS_TLS_H) */
-

src/common/ciphers.inc

@@ -2,27 +2,8 @@
  * advertise.  Before including it, you should define the CIPHER and XCIPHER
  * macros.
  *
- * This file was automatically generated by get_mozilla_ciphers.py;
- * TLSv1.3 ciphers were added manually.
+ * This file was automatically generated by get_mozilla_ciphers.py.
  */
-
-/* Here are the TLS1.3 ciphers. Note that we don't have XCIPHER instances
- * here, since we don't want to ever fake them.
- */
-#ifdef TLS1_3_TXT_AES_128_GCM_SHA256
-   CIPHER(0x1301, TLS1_3_TXT_AES_128_GCM_SHA256)
-#endif
-#ifdef TLS1_3_TXT_AES_256_GCM_SHA384
-   CIPHER(0x1302, TLS1_3_TXT_AES_256_GCM_SHA384)
-#endif
-#ifdef TLS1_3_TXT_CHACHA20_POLY1305_SHA256
-   CIPHER(0x1303, TLS1_3_TXT_CHACHA20_POLY1305_SHA256)
-#endif
-#ifdef TLS1_3_TXT_AES_128_CCM_SHA256
-   CIPHER(0x1304, TLS1_3_TXT_AES_128_CCM_SHA256)
-#endif
-
-/* Here's the machine-generated list. */
 #ifdef TLS1_TXT_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
     CIPHER(0xc02b, TLS1_TXT_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256)
 #else
@@ -33,26 +14,6 @@
 #else
    XCIPHER(0xc02f, TLS1_TXT_ECDHE_RSA_WITH_AES_128_GCM_SHA256)
 #endif
-#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
-    CIPHER(0xcca9, TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305)
-#else
-   XCIPHER(0xcca9, TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305)
-#endif
-#ifdef TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305
-    CIPHER(0xcca8, TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305)
-#else
-   XCIPHER(0xcca8, TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305)
-#endif
-#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
-    CIPHER(0xc02c, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384)
-#else
-   XCIPHER(0xc02c, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384)
-#endif
-#ifdef TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384
-    CIPHER(0xc030, TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384)
-#else
-   XCIPHER(0xc030, TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384)
-#endif
 #ifdef TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_CBC_SHA
     CIPHER(0xc00a, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_CBC_SHA)
 #else
@@ -73,28 +34,88 @@
 #else
    XCIPHER(0xc014, TLS1_TXT_ECDHE_RSA_WITH_AES_256_CBC_SHA)
 #endif
+#ifdef TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA
+    CIPHER(0xc012, TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA)
+#else
+   XCIPHER(0xc012, TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA)
+#endif
+#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA
+    CIPHER(0xc007, TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA)
+#else
+   XCIPHER(0xc007, TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA)
+#endif
+#ifdef TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA
+    CIPHER(0xc011, TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA)
+#else
+   XCIPHER(0xc011, TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA)
+#endif
 #ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_SHA
     CIPHER(0x0033, TLS1_TXT_DHE_RSA_WITH_AES_128_SHA)
 #else
    XCIPHER(0x0033, TLS1_TXT_DHE_RSA_WITH_AES_128_SHA)
 #endif
+#ifdef TLS1_TXT_DHE_DSS_WITH_AES_128_SHA
+    CIPHER(0x0032, TLS1_TXT_DHE_DSS_WITH_AES_128_SHA)
+#else
+   XCIPHER(0x0032, TLS1_TXT_DHE_DSS_WITH_AES_128_SHA)
+#endif
+#ifdef TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA
+    CIPHER(0x0045, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA)
+#else
+   XCIPHER(0x0045, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA)
+#endif
 #ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_SHA
     CIPHER(0x0039, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA)
 #else
    XCIPHER(0x0039, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA)
 #endif
+#ifdef TLS1_TXT_DHE_DSS_WITH_AES_256_SHA
+    CIPHER(0x0038, TLS1_TXT_DHE_DSS_WITH_AES_256_SHA)
+#else
+   XCIPHER(0x0038, TLS1_TXT_DHE_DSS_WITH_AES_256_SHA)
+#endif
+#ifdef TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA
+    CIPHER(0x0088, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA)
+#else
+   XCIPHER(0x0088, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA)
+#endif
+#ifdef SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA
+    CIPHER(0x0016, SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA)
+#else
+   XCIPHER(0x0016, SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA)
+#endif
 #ifdef TLS1_TXT_RSA_WITH_AES_128_SHA
     CIPHER(0x002f, TLS1_TXT_RSA_WITH_AES_128_SHA)
 #else
    XCIPHER(0x002f, TLS1_TXT_RSA_WITH_AES_128_SHA)
 #endif
+#ifdef TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA
+    CIPHER(0x0041, TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA)
+#else
+   XCIPHER(0x0041, TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA)
+#endif
 #ifdef TLS1_TXT_RSA_WITH_AES_256_SHA
     CIPHER(0x0035, TLS1_TXT_RSA_WITH_AES_256_SHA)
 #else
    XCIPHER(0x0035, TLS1_TXT_RSA_WITH_AES_256_SHA)
 #endif
+#ifdef TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA
+    CIPHER(0x0084, TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA)
+#else
+   XCIPHER(0x0084, TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA)
+#endif
 #ifdef SSL3_TXT_RSA_DES_192_CBC3_SHA
     CIPHER(0x000a, SSL3_TXT_RSA_DES_192_CBC3_SHA)
 #else
    XCIPHER(0x000a, SSL3_TXT_RSA_DES_192_CBC3_SHA)
 #endif
+#ifdef SSL3_TXT_RSA_RC4_128_SHA
+    CIPHER(0x0005, SSL3_TXT_RSA_RC4_128_SHA)
+#else
+   XCIPHER(0x0005, SSL3_TXT_RSA_RC4_128_SHA)
+#endif
+#ifdef SSL3_TXT_RSA_RC4_128_MD5
+    CIPHER(0x0004, SSL3_TXT_RSA_RC4_128_MD5)
+#else
+   XCIPHER(0x0004, SSL3_TXT_RSA_RC4_128_MD5)
+#endif

src/common/compat.h

@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2017, The Tor Project, Inc. */
+ * Copyright (c) 2007-2016, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_COMPAT_H
@@ -10,9 +10,6 @@
 #ifdef _WIN32
 #include <winsock2.h>
 #include <ws2tcpip.h>
-#ifndef SIO_IDEAL_SEND_BACKLOG_QUERY
-#define SIO_IDEAL_SEND_BACKLOG_QUERY 0x4004747b
-#endif
 #endif
 #include "torint.h"
 #include "testsupport.h"
@@ -45,16 +42,14 @@
 #include <netinet6/in6.h>
 #endif
 
-#include "compat_time.h"
-
 #if defined(__has_feature)
 #  if __has_feature(address_sanitizer)
 /* Some of the fancy glibc strcmp() macros include references to memory that
  * clang rejects because it is off the end of a less-than-3. Clang hates this,
  * even though those references never actually happen. */
 #    undef strcmp
-#endif /* __has_feature(address_sanitizer) */
-#endif /* defined(__has_feature) */
+#  endif
+#endif
 
 #include <stdio.h>
 #include <errno.h>
@@ -79,52 +74,14 @@
    __attribute__ ((format(printf, formatIdx, firstArg)))
 #else
 #define CHECK_PRINTF(formatIdx, firstArg)
-#endif /* defined(__GNUC__) */
+#endif
 #ifdef __GNUC__
 #define CHECK_SCANF(formatIdx, firstArg) \
    __attribute__ ((format(scanf, formatIdx, firstArg)))
 #else
 #define CHECK_SCANF(formatIdx, firstArg)
-#endif /* defined(__GNUC__) */
-
-/* What GCC do we have? */
-#ifdef __GNUC__
-#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#else
-#define GCC_VERSION 0
 #endif
 
-/* Temporarily enable and disable warnings. */
-#ifdef __GNUC__
-#  define PRAGMA_STRINGIFY_(s) #s
-#  define PRAGMA_JOIN_STRINGIFY_(a,b) PRAGMA_STRINGIFY_(a ## b)
-/* Support for macro-generated pragmas (c99) */
-#  define PRAGMA_(x) _Pragma (#x)
-#  ifdef __clang__
-#    define PRAGMA_DIAGNOSTIC_(x) PRAGMA_(clang diagnostic x)
-#  else
-#    define PRAGMA_DIAGNOSTIC_(x) PRAGMA_(GCC diagnostic x)
-#  endif
-#  if defined(__clang__) || GCC_VERSION >= 406
-/* we have push/pop support */
-#    define DISABLE_GCC_WARNING(warningopt) \
-          PRAGMA_DIAGNOSTIC_(push) \
-          PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
-#    define ENABLE_GCC_WARNING(warningopt) \
-          PRAGMA_DIAGNOSTIC_(pop)
-#else /* !(defined(__clang__) || GCC_VERSION >= 406) */
-/* older version of gcc: no push/pop support. */
-#    define DISABLE_GCC_WARNING(warningopt) \
-         PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
-#    define ENABLE_GCC_WARNING(warningopt) \
-         PRAGMA_DIAGNOSTIC_(warning PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
-#endif /* defined(__clang__) || GCC_VERSION >= 406 */
-#else /* !(defined(__GNUC__)) */
-/* not gcc at all */
-# define DISABLE_GCC_WARNING(warning)
-# define ENABLE_GCC_WARNING(warning)
-#endif /* defined(__GNUC__) */
-
 /* inline is __inline on windows. */
 #ifdef _WIN32
 #define inline __inline
@@ -145,9 +102,9 @@
 #define __func__ __FUNC__
 #else
 #define __func__ "???"
-#endif /* defined(HAVE_MACRO__FUNCTION__) || ... */
-#endif /* !defined(HAVE_MACRO__func__) */
-#endif /* defined(_MSC_VER) */
+#endif
+#endif /* ifndef MAVE_MACRO__func__ */
+#endif /* if not windows */
 
 #define U64_TO_DBL(x) ((double) (x))
 #define DBL_TO_U64(x) ((uint64_t) (x))
@@ -160,7 +117,7 @@
  * problems), but if enumerated types are unsigned, we must use unsigned,
  * so that the loss of precision doesn't make large values negative. */
 #define ENUM_BF(t) t
-#endif /* defined(ENUM_VALS_ARE_SIGNED) */
+#endif
 
 /* GCC has several useful attributes. */
 #if defined(__GNUC__) && __GNUC__ >= 3
@@ -197,7 +154,7 @@
  * taken.  This can generate slightly better code with some CPUs.
  */
 #define PREDICT_UNLIKELY(exp) __builtin_expect(!!(exp), 0)
-#else /* !(defined(__GNUC__) && __GNUC__ >= 3) */
+#else
 #define ATTR_NORETURN
 #define ATTR_CONST
 #define ATTR_MALLOC
@@ -207,7 +164,7 @@
 #define ATTR_WUR
 #define PREDICT_LIKELY(exp) (exp)
 #define PREDICT_UNLIKELY(exp) (exp)
-#endif /* defined(__GNUC__) && __GNUC__ >= 3 */
+#endif
 
 /** Expands to a syntactically valid empty statement.  */
 #define STMT_NIL (void)0
@@ -227,7 +184,7 @@
 #else
 #define STMT_BEGIN do {
 #define STMT_END } while (0)
-#endif /* defined(__GNUC__) || ... */
+#endif
 
 /* Some tools (like coccinelle) don't like to see operators as macro
  * arguments. */
@@ -254,7 +211,7 @@
  */
 #undef strlcat
 #undef strlcpy
-#endif /* defined __APPLE__ */
+#endif
 
 #ifndef HAVE_STRLCAT
 size_t strlcat(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2));
@@ -275,28 +232,24 @@ size_t strlcpy(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2));
 #define I64_PRINTF_ARG(a) (a)
 #define I64_SCANF_ARG(a) (a)
 #define I64_LITERAL(n) (n ## i64)
-#else /* !(defined(_MSC_VER)) */
+#else
 #define U64_PRINTF_ARG(a) ((long long unsigned int)(a))
 #define U64_SCANF_ARG(a) ((long long unsigned int*)(a))
 #define U64_LITERAL(n) (n ## llu)
 #define I64_PRINTF_ARG(a) ((long long signed int)(a))
 #define I64_SCANF_ARG(a) ((long long signed int*)(a))
 #define I64_LITERAL(n) (n ## ll)
-#endif /* defined(_MSC_VER) */
-
-#if defined(__MINGW32__) || defined(__MINGW64__)
-#define MINGW_ANY
 #endif
 
-#if defined(_MSC_VER) || defined(MINGW_ANY)
+#if defined(_MSC_VER) || defined(__MINGW32__) || defined(__MINGW64__)
 /** The formatting string used to put a uint64_t value in a printf() or
  * scanf() function.  See also U64_PRINTF_ARG and U64_SCANF_ARG. */
 #define U64_FORMAT "%I64u"
 #define I64_FORMAT "%I64d"
-#else /* !(defined(_MSC_VER) || defined(MINGW_ANY)) */
+#else
 #define U64_FORMAT "%llu"
 #define I64_FORMAT "%lld"
-#endif /* defined(_MSC_VER) || defined(MINGW_ANY) */
+#endif
 
 #if (SIZEOF_INTPTR_T == SIZEOF_INT)
 #define INTPTR_T_FORMAT "%d"
@@ -309,7 +262,7 @@ size_t strlcpy(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2));
 #define INTPTR_PRINTF_ARG(x) I64_PRINTF_ARG(x)
 #else
 #error Unknown: SIZEOF_INTPTR_T
-#endif /* (SIZEOF_INTPTR_T == SIZEOF_INT) || ... */
+#endif
 
 /** Represents an mmaped file. Allocated via tor_mmap_file; freed with
  * tor_munmap_file. */
@@ -318,12 +271,12 @@ typedef struct tor_mmap_t {
   size_t size; /**< Size of the file. */
 
   /* None of the fields below should be accessed from outside compat.c */
-#ifdef HAVE_MMAP
+#ifdef HAVE_SYS_MMAN_H
   size_t mapping_size; /**< Size of the actual mapping. (This is this file
                         * size, rounded up to the nearest page.) */
 #elif defined _WIN32
   HANDLE mmap_handle;
-#endif /* defined(HAVE_MMAP) || ... */
+#endif
 
 } tor_mmap_t;
 
@@ -367,8 +320,8 @@ DECLARE_CTYPE_FN(ISXDIGIT)
 DECLARE_CTYPE_FN(ISPRINT)
 DECLARE_CTYPE_FN(ISLOWER)
 DECLARE_CTYPE_FN(ISUPPER)
-extern const uint8_t TOR_TOUPPER_TABLE[];
-extern const uint8_t TOR_TOLOWER_TABLE[];
+extern const char TOR_TOUPPER_TABLE[];
+extern const char TOR_TOLOWER_TABLE[];
 #define TOR_TOLOWER(c) (TOR_TOLOWER_TABLE[(uint8_t)c])
 #define TOR_TOUPPER(c) (TOR_TOUPPER_TABLE[(uint8_t)c])
 
@@ -385,9 +338,18 @@ const char *tor_fix_source_file(const char *fname);
 #else
 #define SHORT_FILE__ (__FILE__)
 #define tor_fix_source_file(s) (s)
-#endif /* defined(_WIN32) */
+#endif
 
 /* ===== Time compatibility */
+#if !defined(HAVE_GETTIMEOFDAY) && !defined(HAVE_STRUCT_TIMEVAL_TV_SEC)
+/** Implementation of timeval for platforms that don't have it. */
+struct timeval {
+  time_t tv_sec;
+  unsigned int tv_usec;
+};
+#endif
+
+void tor_gettimeofday(struct timeval *timeval);
 
 struct tm *tor_localtime_r(const time_t *timep, struct tm *result);
 struct tm *tor_gmtime_r(const time_t *timep, struct tm *result);
@@ -404,7 +366,7 @@ struct tm *tor_gmtime_r(const time_t *timep, struct tm *result);
       (tvout)->tv_sec++;                                \
     }                                                   \
   } while (0)
-#endif /* !defined(timeradd) */
+#endif
 
 #ifndef timersub
 /** Replacement for timersub on platforms that do not have it: sets tvout to
@@ -418,13 +380,13 @@ struct tm *tor_gmtime_r(const time_t *timep, struct tm *result);
       (tvout)->tv_sec--;                                \
     }                                                   \
   } while (0)
-#endif /* !defined(timersub) */
+#endif
 
 #ifndef timercmp
-/** Replacement for timercmp on platforms that do not have it: returns true
+/** Replacement for timersub on platforms that do not have it: returns true
  * iff the relational operator "op" makes the expression tv1 op tv2 true.
  *
- * Note that while this definition should work for all boolean operators, some
+ * Note that while this definition should work for all boolean opeators, some
  * platforms' native timercmp definitions do not support >=, <=, or ==.  So
  * don't use those.
  */
@@ -432,7 +394,7 @@ struct tm *tor_gmtime_r(const time_t *timep, struct tm *result);
   (((tv1)->tv_sec == (tv2)->tv_sec) ?           \
    ((tv1)->tv_usec op (tv2)->tv_usec) :         \
    ((tv1)->tv_sec op (tv2)->tv_sec))
-#endif /* !defined(timercmp) */
+#endif
 
 /* ===== File compatibility */
 int tor_open_cloexec(const char *path, int flags, unsigned mode);
@@ -468,13 +430,13 @@ typedef int socklen_t;
 
 #ifdef _WIN32
 /* XXX Actually, this should arguably be SOCKET; we use intptr_t here so that
- * any inadvertent checks for the socket being <= 0 or > 0 will probably
+ * any inadvertant checks for the socket being <= 0 or > 0 will probably
  * still work. */
 #define tor_socket_t intptr_t
 #define TOR_SOCKET_T_FORMAT INTPTR_T_FORMAT
 #define SOCKET_OK(s) ((SOCKET)(s) != INVALID_SOCKET)
 #define TOR_INVALID_SOCKET INVALID_SOCKET
-#else /* !(defined(_WIN32)) */
+#else
 /** Type used for a network socket. */
 #define tor_socket_t int
 #define TOR_SOCKET_T_FORMAT "%d"
@@ -482,11 +444,10 @@ typedef int socklen_t;
 #define SOCKET_OK(s) ((s) >= 0)
 /** Error/uninitialized value for a tor_socket_t. */
 #define TOR_INVALID_SOCKET (-1)
-#endif /* defined(_WIN32) */
+#endif
 
 int tor_close_socket_simple(tor_socket_t s);
-MOCK_DECL(int, tor_close_socket, (tor_socket_t s));
-void tor_take_socket_ownership(tor_socket_t s);
+int tor_close_socket(tor_socket_t s);
 tor_socket_t tor_open_socket_with_extensions(
                                            int domain, int type, int protocol,
                                            int cloexec, int nonblock);
@@ -510,8 +471,6 @@ int get_n_open_sockets(void);
 MOCK_DECL(int,
 tor_getsockname,(tor_socket_t socket, struct sockaddr *address,
                  socklen_t *address_len));
-struct tor_addr_t;
-int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock);
 
 #define tor_socket_send(s, buf, len, flags) send(s, buf, len, flags)
 #define tor_socket_recv(s, buf, len, flags) recv(s, buf, len, flags)
@@ -532,19 +491,19 @@ struct in6_addr
 #define s6_addr16 in6_u.u6_addr16
 #define s6_addr32 in6_u.u6_addr32
 };
-#endif /* !defined(HAVE_STRUCT_IN6_ADDR) */
+#endif
 
 /** @{ */
 /** Many BSD variants seem not to define these. */
-#if defined(__APPLE__) || defined(__darwin__) || \
-  defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)
+#if defined(__APPLE__) || defined(__darwin__) || defined(__FreeBSD__) \
+    || defined(__NetBSD__) || defined(__OpenBSD__)
 #ifndef s6_addr16
 #define s6_addr16 __u6_addr.__u6_addr16
 #endif
 #ifndef s6_addr32
 #define s6_addr32 __u6_addr.__u6_addr32
 #endif
-#endif /* defined(__APPLE__) || defined(__darwin__) || ... */
+#endif
 /** @} */
 
 #ifndef HAVE_SA_FAMILY_T
@@ -576,7 +535,7 @@ struct sockaddr_in6 {
   struct in6_addr sin6_addr;
   // uint32_t sin6_scope_id;
 };
-#endif /* !defined(HAVE_STRUCT_SOCKADDR_IN6) */
+#endif
 
 MOCK_DECL(int,tor_gethostname,(char *name, size_t namelen));
 int tor_inet_aton(const char *cp, struct in_addr *addr) ATTR_NONNULL((1,2));
@@ -617,14 +576,14 @@ int network_init(void);
 #define ERRNO_IS_EINTR(e)            ((e) == WSAEINTR || 0)
 int tor_socket_errno(tor_socket_t sock);
 const char *tor_socket_strerror(int e);
-#else /* !(defined(_WIN32)) */
+#else
 #define SOCK_ERRNO(e) e
 #if EAGAIN == EWOULDBLOCK
 /* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */
 #define ERRNO_IS_EAGAIN(e)           ((e) == EAGAIN || 0)
 #else
 #define ERRNO_IS_EAGAIN(e)           ((e) == EAGAIN || (e) == EWOULDBLOCK)
-#endif /* EAGAIN == EWOULDBLOCK */
+#endif
 #define ERRNO_IS_EINTR(e)            ((e) == EINTR || 0)
 #define ERRNO_IS_EINPROGRESS(e)      ((e) == EINPROGRESS || 0)
 #define ERRNO_IS_CONN_EINPROGRESS(e) ((e) == EINPROGRESS || 0)
@@ -635,7 +594,7 @@ const char *tor_socket_strerror(int e);
 #define ERRNO_IS_EADDRINUSE(e)       (((e) == EADDRINUSE) || 0)
 #define tor_socket_errno(sock)       (errno)
 #define tor_socket_strerror(e)       strerror(e)
-#endif /* defined(_WIN32) */
+#endif
 
 /** Specified SOCKS5 status codes. */
 typedef enum {
@@ -701,7 +660,7 @@ char *make_path_absolute(char *fname);
 
 char **get_environment(void);
 
-MOCK_DECL(int, get_total_system_memory, (size_t *mem_out));
+int get_total_system_memory(size_t *mem_out);
 
 int compute_num_cpus(void);
 
@@ -738,7 +697,11 @@ char *format_win32_error(DWORD err);
 #define VER_SUITE_SINGLEUSERTS 0x00000100
 #endif
 
-#endif /* defined(_WIN32) */
+#endif
+
+#ifdef TOR_UNIT_TESTS
+void tor_sleep_msec(int msec);
+#endif
 
 #ifdef COMPAT_PRIVATE
 #if !defined(HAVE_SOCKETPAIR) || defined(_WIN32) || defined(TOR_UNIT_TESTS)
@@ -746,12 +709,12 @@ char *format_win32_error(DWORD err);
 STATIC int tor_ersatz_socketpair(int family, int type, int protocol,
                                    tor_socket_t fd[2]);
 #endif
-#endif /* defined(COMPAT_PRIVATE) */
+#endif
 
 ssize_t tor_getpass(const char *prompt, char *output, size_t buflen);
 
 /* This needs some of the declarations above so we include it here. */
 #include "compat_threads.h"
 
-#endif /* !defined(TOR_COMPAT_H) */
+#endif