Add config parsers for multiple tcp socket readers and a helper to see if we can offer it on the platform

4 files changed, 43 insertions, 4 deletions

diff --git a/ot_config.c b/ot_config.c
index a0dbef1..272acf1 100644
--- a/ot_config.c
+++ b/ot_config.c
@@ -14,6 +14,7 @@
 
 /* Opentracker */
 #include "ot_accesslist.h"
+#include "ot_event.h"
 #include "ot_config.h"
 #include "ot_http.h"
 #include "ot_ip.h"
@@ -32,7 +33,8 @@ const char   *g_statefile;
 static ot_listen *g_listen;
 static size_t     g_listen_count;
 static int        g_listen_shared;
-static int        g_listen_workers = 1;
+static int        g_listen_workers_udp = 1;
+static int        g_listen_workers_tcp = 0;
 
 static void panic(const char *routine) __attribute__((noreturn));
 static void panic(const char *routine) {
@@ -196,6 +198,7 @@ typedef enum {
     CONFIG_LISTEN_TCP,
     CONFIG_LISTEN_UDP,
     CONFIG_LISTEN_UDP_WORKERS,
+    CONFIG_LISTEN_TCP_WORKERS,
     CONFIG_LISTEN_SHARED,
     CONFIG_ACCESS_WHITELIST,
     CONFIG_ACCESS_BLACKLIST,
@@ -224,6 +227,7 @@ static const config_key config_keys[] = {
     CONFIG_KEY("listen.tcp",               CONFIG_LISTEN_TCP),
     CONFIG_KEY("listen.udp",               CONFIG_LISTEN_UDP),
     CONFIG_KEY("listen.udp.workers",       CONFIG_LISTEN_UDP_WORKERS),
+    CONFIG_KEY("listen.tcp.workers",       CONFIG_LISTEN_TCP_WORKERS),
     CONFIG_KEY("listen.shared",            CONFIG_LISTEN_SHARED),
 #ifdef WANT_ACCESSLIST_WHITE
     CONFIG_KEY("access.whitelist",         CONFIG_ACCESS_WHITELIST),
@@ -318,7 +322,7 @@ static int parse_configfile(char *config_filename) {
         if (!off || !listen_scan_options(p + off, &shared, &workers))
           goto parse_error;
         if (kind != CONFIG_LISTEN_UDP)
-          listen_append(tmpip, tmpport, FLAG_TCP, shared, OT_LISTEN_INHERIT), ++bound;
+          listen_append(tmpip, tmpport, FLAG_TCP, shared, workers), ++bound;
         if (kind != CONFIG_LISTEN_TCP)
           listen_append(tmpip, tmpport, FLAG_UDP, shared, workers), ++bound;
         break;
@@ -326,7 +330,12 @@ static int parse_configfile(char *config_filename) {
       case CONFIG_LISTEN_UDP_WORKERS:
         if (!scan_uint(p, &value, UINT_MAX))
           goto parse_error;
-        g_listen_workers = (int)value;
+        g_listen_workers_udp = (int)value;
+        break;
+      case CONFIG_LISTEN_TCP_WORKERS:
+        if (!scan_uint(p, &value, UINT_MAX))
+          goto parse_error;
+        g_listen_workers_tcp = (int)value;
         break;
       case CONFIG_LISTEN_SHARED:
         if (!scan_uint(p, &value, 1))
@@ -446,7 +455,16 @@ static void listen_resolve_defaults(void) {
     if (entry->shared == OT_LISTEN_INHERIT)
       entry->shared = g_listen_shared;
     if (entry->workers == OT_LISTEN_INHERIT)
-      entry->workers = g_listen_workers;
+      entry->workers = entry->proto == FLAG_TCP ? g_listen_workers_tcp : g_listen_workers_udp;
+
+    /* A tcp worker hands a connection it cannot finish to the event loop,
+       which only one thread may touch where the set is ours rather than the
+       kernel's */
+    if (entry->proto == FLAG_TCP && entry->workers > 1 && !event_can_share()) {
+      fprintf(stderr, "Warning: Multiple workers impossible on this platform with event mechanism %s chosen.\n",
+              event_mechanism());
+      entry->workers = 1;
+    }
   }
 }
 
diff --git a/ot_event.c b/ot_event.c
index b4c4a55..61cc07b 100644
--- a/ot_event.c
+++ b/ot_event.c
@@ -40,6 +40,9 @@ static struct epoll_event *g_ready;
 static int                 g_ready_space;
 
 const char *event_mechanism(void) { return "epoll"; }
+/* The set lives in the kernel, and epoll_ctl during another thread's
+   epoll_wait is what it is for */
+int         event_can_share(void) { return 1; }
 
 int event_init(void) {
   g_ready       = NULL;
@@ -118,6 +121,8 @@ static struct kevent *g_ready;
 static int            g_ready_space;
 
 const char *event_mechanism(void) { return "kqueue"; }
+/* The set lives in the kernel, and kevent may be called from any thread */
+int         event_can_share(void) { return 1; }
 
 int event_init(void) {
   g_ready       = NULL;
@@ -229,6 +234,8 @@ static int           *g_where;
 static int            g_count, g_space, g_where_space;
 
 const char *event_mechanism(void) { return "poll"; }
+/* The arrays below are ours and nothing guards them */
+int         event_can_share(void) { return 0; }
 
 int event_init(void) {
   g_pfds = NULL, g_generation = NULL, g_where = NULL;
diff --git a/ot_event.h b/ot_event.h
index aa007d0..efdefff 100644
--- a/ot_event.h
+++ b/ot_event.h
@@ -38,6 +38,11 @@ int  event_close(int fd);
    out with up to max ready descriptors. Returns how many, or -1. */
 int  event_wait(ot_event *out, int max, int timeout_ms);
 
+/* Whether the registered set may be changed from another thread while this
+   one is waiting on it. True where the kernel keeps the set, false where
+   this module keeps it in memory of its own. */
+int  event_can_share(void);
+
 /* Which mechanism was compiled in, for the version string */
 const char *event_mechanism(void);
 
diff --git a/tests/t_event.c b/tests/t_event.c
index 706a285..1908dc9 100644
--- a/tests/t_event.c
+++ b/tests/t_event.c
@@ -263,9 +263,18 @@ static void case_add_nothing_then_ask(void) {
   pair_close();
 }
 
+/* Whether a worker thread may register a descriptor while this one waits
+   depends on who keeps the set. Tying the answer to the mechanism's name
+   here means a backend added later cannot quietly forget to say. */
+static void case_sharing_matches_the_mechanism(void) {
+  test_case("only a set the kernel keeps may be changed from another thread");
+  CHECK_INT(event_can_share(), strcmp(event_mechanism(), "poll") ? 1 : 0);
+}
+
 void test_event(void) {
   CHECK_INT(event_init(), 0);
   CHECK(event_mechanism() != NULL);
+  case_sharing_matches_the_mechanism();
 
   case_readable();
   case_writable();