Merge branch 'maint-0.2.2' into release-0.2.2

changes/bug2230_clean_1

@@ -0,0 +1,4 @@
+  o Minor features
+    - Backport code from 0.2.3.x to allow directory authorities to clean
+      their microdescriptor caches.
+

changes/bug2230_part1

@@ -0,0 +1,7 @@
+  o Minor bugfixes
+    - When loading the microdesc journal, remember its current size.
+      In 0.2.2, this helps prevent the microdesc journal from growing
+      without limit on authorities (who are the only ones to use it in
+      0.2.2).  Fixes a part of bug 2230; bugfix on 0.2.2.6-alpha.
+      Fix posted by "cypherpunks."
+      

changes/bug2230_part2

@@ -0,0 +1,5 @@
+  o Minor bugfixes
+    - The microdesc journal is supposed to get rebuilt only if it is
+      at least _half_ the length of the store, not _twice_ the length
+      of the store.   Bugfix on 0.2.2.6-alpha; fixes part of bug 2230.
+

changes/bug2230_part4

@@ -0,0 +1,6 @@
+  o Minor bugfixes:
+    - Authorities now clean their microdesc cache periodically and when
+      reading from disk initially, not only when adding new descriptors.
+      This prevents a bug where we could lose microdescriptors.  Bugfix
+      on 0.2.2.6-alpha.
+

doc/HACKING

@@ -414,10 +414,47 @@ Here are the steps Roger takes when putting out a new Tor release:
 and as a directory authority. See if it has any obvious bugs, and
 resolve those.
 
+1.5) As applicable, merge the maint-X branch into the release-X branch.
+
 2) Gather the changes/* files into a changelog entry, rewriting many
 of them and reordering to focus on what users and funders would find
 interesting and understandable.
 
+   2.1) Make sure that everything that wants a bug number has one.
+   2.2) Concatenate them.
+   2.3) Sort them by section.  Within each section, try to make the
+      first entry or two and the last entry most interesting: they're
+      the ones that skimmers tend to read.
+
+   2.4) Clean them up
+
+   Standard idioms:
+     "Fixes bug 9999; Bugfix on 0.3.3.3-alpha."
+
+   One period after a space.
+
+   Make stuff very terse
+
+   Describe the user-visible problem right away
+
+   Mention relevant config options by name.  If they're rare or unusual,
+   remind people what they're for
+
+   Avoid starting lines with open-paren
+
+   Present and imperative tense: not past.
+
+   If a given changes stanza showed up in a different release (e.g.
+   maint-0.2.1), be sure to make the stanzas identical (so people can
+   distinguish if these are the same change).
+
+   2.5) Merge them in.
+
+   2.6) Clean everything one last time.
+
+   2.7) Run it through fmt to make it pretty.
+
+
 3) Compose a short release blurb to highlight the user-facing
 changes. Insert said release blurb into the ChangeLog stanza. If it's
 a stable release, add it to the ReleaseNotes file too. If we're adding
@@ -435,6 +472,9 @@ in their approved versions list.
 
 7) Sign and push the tarball to the website in the dist/ directory. Sign
 and push the git tag.
+   (That's either "git tag -u <keyid> tor-0.2.x.y-status", then
+    "git push origin tag tor-0.2.x.y-status".  To sign the
+    tarball, "gpg -ba <the_tarball>")
 
 8) Edit include/versions.wmi to note the new version. Rebuild and push
 the website.
@@ -453,3 +493,5 @@ the date in the ChangeLog.
 packages are up (for a stable release), and mail the release blurb and
 changelog to tor-talk or tor-announce.
 
+  (We might be moving to faster announcements, but don't announce until
+  the website is at least updated.)

src/or/config.c

@@ -3488,10 +3488,9 @@ options_validate(or_options_t *old_options, or_options_t *options,
   }
 
   if (options->CookieAuthFileGroupReadable && !options->CookieAuthFile) {
-    log_warn(LD_CONFIG, "You set the CookieAuthFileGroupReadable but did "
-             "not configure a the path for the cookie file via "
-             "CookieAuthFile. This means your cookie will not be group "
-             "readable.");
+    log_warn(LD_CONFIG, "CookieAuthFileGroupReadable is set, but will have "
+             "no effect: you must specify an explicit CookieAuthFile to "
+             "have it group-readable.");
   }
 
   if (options->UseEntryGuards && ! options->NumEntryGuards)

src/or/main.c

@@ -1075,6 +1075,8 @@ run_scheduled_events(time_t now)
     rep_history_clean(now - options->RephistTrackTime);
     rend_cache_clean();
     rend_cache_clean_v2_descs_as_dir();
+    if (authdir_mode_v3(options))
+      microdesc_cache_rebuild(NULL, 0);
 #define CLEAN_CACHES_INTERVAL (30*60)
     time_to_clean_caches = now + CLEAN_CACHES_INTERVAL;
   }

src/or/microdesc.c

@@ -23,6 +23,8 @@ struct microdesc_cache_t {
   tor_mmap_t *cache_content;
   /** Number of bytes used in the journal file. */
   size_t journal_len;
+  /** Number of bytes in descriptors removed as too old. */
+  size_t bytes_dropped;
 
   /** Total bytes of microdescriptor bodies we have added to this cache */
   uint64_t total_len_seen;
@@ -175,6 +177,8 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache,
       if (md2->last_listed < md->last_listed)
         md2->last_listed = md->last_listed;
       microdesc_free(md);
+      if (where != SAVED_NOWHERE)
+        cache->bytes_dropped += size;
       continue;
     }
 
@@ -204,15 +208,6 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache,
   if (f)
     finish_writing_to_file(open_file); /*XXX Check me.*/
 
-  {
-    size_t old_content_len =
-      cache->cache_content ? cache->cache_content->size : 0;
-    if (cache->journal_len > 16384 + old_content_len &&
-        cache->journal_len > old_content_len * 2) {
-      microdesc_cache_rebuild(cache);
-    }
-  }
-
   return added;
 }
 
@@ -233,6 +228,7 @@ microdesc_cache_clear(microdesc_cache_t *cache)
   }
   cache->total_len_seen = 0;
   cache->n_seen = 0;
+  cache->bytes_dropped = 0;
 }
 
 /** Reload the contents of <b>cache</b> from disk.  If it is empty, load it
@@ -261,6 +257,7 @@ microdesc_cache_reload(microdesc_cache_t *cache)
   journal_content = read_file_to_str(cache->journal_fname,
                                      RFTS_IGNORE_MISSING, &st);
   if (journal_content) {
+    cache->journal_len = (size_t) st.st_size;
     added = microdescs_add_to_cache(cache, journal_content,
                                     journal_content+st.st_size,
                                     SAVED_IN_JOURNAL, 0);
@@ -272,14 +269,81 @@ microdesc_cache_reload(microdesc_cache_t *cache)
   }
   log_notice(LD_DIR, "Reloaded microdescriptor cache.  Found %d descriptors.",
              total);
+
+  microdesc_cache_rebuild(cache, 0 /* don't force */);
+
   return 0;
 }
 
+/** By default, we remove any microdescriptors that have gone at least this
+ * long without appearing in a current consensus. */
+#define TOLERATE_MICRODESC_AGE (7*24*60*60)
+
+/** Remove all microdescriptors from <b>cache</b> that haven't been listed for
+ * a long time.  Does not rebuild the cache on disk.  If <b>cutoff</b> is
+ * positive, specifically remove microdescriptors that have been unlisted
+ * since <b>cutoff</b>.  If <b>force</b> is true, remove microdescriptors even
+ * if we have no current live microdescriptor consensus.
+ */
+void
+microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
+{
+  microdesc_t **mdp, *victim;
+  int dropped=0, kept=0;
+  size_t bytes_dropped = 0;
+  time_t now = time(NULL);
+
+  (void) force;
+  /* In 0.2.2, we let this proceed unconditionally: only authorities have
+   * microdesc caches. */
+
+  if (cutoff <= 0)
+    cutoff = now - TOLERATE_MICRODESC_AGE;
+
+  for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) {
+    if ((*mdp)->last_listed < cutoff) {
+      ++dropped;
+      victim = *mdp;
+      mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp);
+      bytes_dropped += victim->bodylen;
+      microdesc_free(victim);
+    } else {
+      ++kept;
+      mdp = HT_NEXT(microdesc_map, &cache->map, mdp);
+    }
+  }
+
+  if (dropped) {
+    log_notice(LD_DIR, "Removed %d/%d microdescriptors as old.",
+               dropped,dropped+kept);
+    cache->bytes_dropped += bytes_dropped;
+  }
+}
+
+static int
+should_rebuild_md_cache(microdesc_cache_t *cache)
+{
+    const size_t old_len =
+      cache->cache_content ? cache->cache_content->size : 0;
+    const size_t journal_len = cache->journal_len;
+    const size_t dropped = cache->bytes_dropped;
+
+    if (journal_len < 16384)
+      return 0; /* Don't bother, not enough has happened yet. */
+    if (dropped > (journal_len + old_len) / 3)
+      return 1; /* We could save 1/3 or more of the currently used space. */
+    if (journal_len > old_len / 2)
+      return 1; /* We should append to the regular file */
+
+    return 0;
+}
+
 /** Regenerate the main cache file for <b>cache</b>, clear the journal file,
  * and update every microdesc_t in the cache with pointers to its new
- * location. */
+ * location.  If <b>force</b> is true, do this unconditionally.  If
+ * <b>force</b> is false, do it only if we expect to save space on disk. */
 int
-microdesc_cache_rebuild(microdesc_cache_t *cache)
+microdesc_cache_rebuild(microdesc_cache_t *cache, int force)
 {
   open_file_t *open_file;
   FILE *f;
@@ -289,7 +353,20 @@ microdesc_cache_rebuild(microdesc_cache_t *cache)
   off_t off = 0;
   int orig_size, new_size;
 
+  if (cache == NULL) {
+    cache = the_microdesc_cache;
+    if (cache == NULL)
+      return 0;
+  }
+
+  /* Remove dead descriptors */
+  microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/);
+
+  if (!force && !should_rebuild_md_cache(cache))
+    return 0;
+
   log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
+
   orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
   orig_size += (int)cache->journal_len;
 
@@ -344,6 +421,7 @@ microdesc_cache_rebuild(microdesc_cache_t *cache)
 
   write_str_to_file(cache->journal_fname, "", 1);
   cache->journal_len = 0;
+  cache->bytes_dropped = 0;
 
   new_size = (int)cache->cache_content->size;
   log_info(LD_DIR, "Done rebuilding microdesc cache. "

src/or/microdesc.h

@@ -21,7 +21,8 @@ smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache,
                         smartlist_t *descriptors, saved_location_t where,
                         int no_save);
 
-int microdesc_cache_rebuild(microdesc_cache_t *cache);
+void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force);
+int microdesc_cache_rebuild(microdesc_cache_t *cache, int force);
 int microdesc_cache_reload(microdesc_cache_t *cache);
 void microdesc_cache_clear(microdesc_cache_t *cache);