Merge branch 'maint-0.3.3' into release-0.3.3

changes/bug24782

@@ -0,0 +1,4 @@
+  o Minor features (config options):
+    - Change the way the default value for MaxMemInQueues is calculated. We now
+      use 0.4 * RAM if the system have 8 GB RAM or more, otherwise we use the
+      former value of 0.75 * RAM. Closes ticket 24782.

src/common/compat.c

@@ -3409,8 +3409,8 @@ get_total_system_memory_impl(void)
  * Try to find out how much physical memory the system has. On success,
  * return 0 and set *<b>mem_out</b> to that value. On failure, return -1.
  */
-int
-get_total_system_memory(size_t *mem_out)
+MOCK_IMPL(int,
+get_total_system_memory, (size_t *mem_out))
 {
   static size_t mem_cached=0;
   uint64_t m = get_total_system_memory_impl();

src/common/compat.h

@@ -699,7 +699,7 @@ char *make_path_absolute(char *fname);
 
 char **get_environment(void);
 
-int get_total_system_memory(size_t *mem_out);
+MOCK_DECL(int, get_total_system_memory, (size_t *mem_out));
 
 int compute_num_cpus(void);
 

src/or/config.c

@@ -773,8 +773,6 @@ static void config_maybe_load_geoip_files_(const or_options_t *options,
 static int options_validate_cb(void *old_options, void *options,
                                void *default_options,
                                int from_setconf, char **msg);
-static uint64_t compute_real_max_mem_in_queues(const uint64_t val,
-                                               int log_guess);
 static void cleanup_protocol_warning_severity_level(void);
 static void set_protocol_warning_severity_level(int warning_severity);
 
@@ -4522,7 +4520,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
 /* Given the value that the user has set for MaxMemInQueues, compute the
  * actual maximum value.  We clip this value if it's too low, and autodetect
  * it if it's set to 0. */
-static uint64_t
+STATIC uint64_t
 compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
 {
   uint64_t result;
@@ -4530,11 +4528,6 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
   if (val == 0) {
 #define ONE_GIGABYTE (U64_LITERAL(1) << 30)
 #define ONE_MEGABYTE (U64_LITERAL(1) << 20)
-#if SIZEOF_VOID_P >= 8
-#define MAX_DEFAULT_MAXMEM (8*ONE_GIGABYTE)
-#else
-#define MAX_DEFAULT_MAXMEM (2*ONE_GIGABYTE)
-#endif
     /* The user didn't pick a memory limit.  Choose a very large one
      * that is still smaller than the system memory */
     static int notice_sent = 0;
@@ -4549,14 +4542,30 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
       result = ONE_GIGABYTE;
 #endif /* SIZEOF_VOID_P >= 8 */
     } else {
-      /* We detected it, so let's pick 3/4 of the total RAM as our limit. */
-      const uint64_t avail = (ram / 4) * 3;
+      /* We detected the amount of memory available. */
+      uint64_t avail = 0;
 
-      /* Make sure it's in range from 0.25 GB to 8 GB. */
-      if (avail > MAX_DEFAULT_MAXMEM) {
+      if (ram >= (8 * ONE_GIGABYTE)) {
+        /* If we have 8 GB, or more, RAM available, we set the MaxMemInQueues
+         * to 0.4 * RAM. The idea behind this value is that the amount of RAM
+         * is more than enough for a single relay and should allow the relay
+         * operator to run two relays if they have additional bandwidth
+         * available.
+         */
+        avail = (ram / 5) * 2;
+      } else {
+        /* If we have less than 8 GB of RAM available, we use the "old" default
+         * for MaxMemInQueues of 0.75 * RAM.
+         */
+        avail = (ram / 4) * 3;
+      }
+
+      /* Make sure it's in range from 0.25 GB to 8 GB for 64-bit and 0.25 to 2
+       * GB for 32-bit. */
+      if (avail > MAX_DEFAULT_MEMORY_QUEUE_SIZE) {
         /* If you want to use more than this much RAM, you need to configure
            it yourself */
-        result = MAX_DEFAULT_MAXMEM;
+        result = MAX_DEFAULT_MEMORY_QUEUE_SIZE;
       } else if (avail < ONE_GIGABYTE / 4) {
         result = ONE_GIGABYTE / 4;
       } else {

src/or/config.h

@@ -22,6 +22,13 @@
  * expose more information than we're comfortable with. */
 #define MIN_HEARTBEAT_PERIOD (30*60)
 
+/** Maximum default value for MaxMemInQueues, in bytes. */
+#if SIZEOF_VOID_P >= 8
+#define MAX_DEFAULT_MEMORY_QUEUE_SIZE (U64_LITERAL(8) << 30)
+#else
+#define MAX_DEFAULT_MEMORY_QUEUE_SIZE (U64_LITERAL(2) << 30)
+#endif
+
 MOCK_DECL(const char*, get_dirportfrontpage, (void));
 MOCK_DECL(const or_options_t *, get_options, (void));
 MOCK_DECL(or_options_t *, get_options_mutable, (void));
@@ -258,6 +265,10 @@ STATIC int parse_port_config(smartlist_t *out,
                   const unsigned flags);
 
 STATIC int check_bridge_distribution_setting(const char *bd);
+
+STATIC uint64_t compute_real_max_mem_in_queues(const uint64_t val,
+                                               int log_guess);
+
 #endif /* defined(CONFIG_PRIVATE) */
 
 #endif /* !defined(TOR_CONFIG_H) */

src/test/test_config.c

@@ -585,6 +585,22 @@ test_config_parse_transport_options_line(void *arg)
   }
 }
 
+/* Mocks needed for the compute_max_mem_in_queues test */
+static int get_total_system_memory_mock(size_t *mem_out);
+
+static size_t total_system_memory_output = 0;
+static int total_system_memory_return = 0;
+
+static int
+get_total_system_memory_mock(size_t *mem_out)
+{
+  if (! mem_out)
+    return -1;
+
+  *mem_out = total_system_memory_output;
+  return total_system_memory_return;
+}
+
 /* Mocks needed for the transport plugin line test */
 
 static void pt_kickstart_proxy_mock(const smartlist_t *transport_list,
@@ -5596,6 +5612,72 @@ test_config_include_opened_file_list(void *data)
   tor_free(dir);
 }
 
+static void
+test_config_compute_max_mem_in_queues(void *data)
+{
+#define GIGABYTE(x) (U64_LITERAL(x) << 30)
+#define MEGABYTE(x) (U64_LITERAL(x) << 20)
+  (void)data;
+  MOCK(get_total_system_memory, get_total_system_memory_mock);
+
+  /* We are unable to detect the amount of memory on the system. Tor will try
+   * to use some sensible default values for 64-bit and 32-bit systems. */
+  total_system_memory_return = -1;
+
+#if SIZEOF_VOID_P >= 8
+  /* We are on a 64-bit system. */
+  tt_int_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, GIGABYTE(8));
+#else
+  /* We are on a 32-bit system. */
+  tt_int_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, GIGABYTE(1));
+#endif
+
+  /* We are able to detect the amount of RAM on the system. */
+  total_system_memory_return = 0;
+
+  /* We are running on a system with one gigabyte of RAM. */
+  total_system_memory_output = GIGABYTE(1);
+
+  /* We have 0.75 * RAM available. */
+  tt_int_op(compute_real_max_mem_in_queues(0, 0), OP_EQ,
+            3 * (GIGABYTE(1) / 4));
+
+  /* We are running on a tiny machine with 256 MB of RAM. */
+  total_system_memory_output = MEGABYTE(256);
+
+  /* We will now enforce a minimum of 256 MB of RAM available for the
+   * MaxMemInQueues here, even though we should only have had 0.75 * 256 = 192
+   * MB available. */
+  tt_int_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, MEGABYTE(256));
+
+  /* We are running on a machine with 8 GB of RAM. */
+  total_system_memory_output = GIGABYTE(8);
+
+  /* We will have 0.4 * RAM available. */
+  tt_int_op(compute_real_max_mem_in_queues(0, 0), OP_EQ,
+            2 * (GIGABYTE(8) / 5));
+
+  /* We are running on a machine with 16 GB of RAM. */
+  total_system_memory_output = GIGABYTE(16);
+
+  /* We will have 0.4 * RAM available. */
+  tt_int_op(compute_real_max_mem_in_queues(0, 0), OP_EQ,
+            2 * (GIGABYTE(16) / 5));
+
+  /* We are running on a machine with 32 GB of RAM. */
+  total_system_memory_output = GIGABYTE(32);
+
+  /* We will at maximum get MAX_DEFAULT_MEMORY_QUEUE_SIZE here. */
+  tt_int_op(compute_real_max_mem_in_queues(0, 0), OP_EQ,
+            MAX_DEFAULT_MEMORY_QUEUE_SIZE);
+
+ done:
+  UNMOCK(get_total_system_memory);
+
+#undef GIGABYTE
+#undef MEGABYTE
+}
+
 #define CONFIG_TEST(name, flags)                          \
   { #name, test_config_ ## name, flags, NULL, NULL }
 
@@ -5644,6 +5726,7 @@ struct testcase_t config_tests[] = {
   CONFIG_TEST(check_bridge_distribution_setting_invalid, 0),
   CONFIG_TEST(check_bridge_distribution_setting_unrecognised, 0),
   CONFIG_TEST(include_opened_file_list, 0),
+  CONFIG_TEST(compute_max_mem_in_queues, 0),
   END_OF_TESTCASES
 };