24 files changed, 861 insertions, 528 deletions

diff --git a/Makefile b/Makefile
index 2e52367..61eb872 100644
--- a/Makefile
+++ b/Makefile
@@ -69,8 +69,8 @@ CFLAGS_libowfat=-I$(LIBOWFAT_HEADERS)
 LDFLAGS_libowfat=-L$(LIBOWFAT_LIBRARY) -lowfat
 
 BINARY =opentracker
-HEADERS=trackerlogic.h ot_urlencoded.h ot_batch.h ot_buf.h ot_event.h ot_fdstate.h ot_fmt.h ot_hex.h ot_ip.h ot_config.h ot_scan.h ot_socket.h ot_mutex.h ot_random.h ot_stats.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h ot_rijndael.h
-SOURCES=opentracker.c trackerlogic.c ot_urlencoded.c ot_batch.c ot_buf.c ot_event.c ot_fdstate.c ot_fmt.c ot_hex.c ot_ip.c ot_config.c ot_scan.c ot_socket.c ot_mutex.c ot_random.c ot_stats.c ot_vector.c ot_clean.c ot_udp.c ot_iovec.c ot_fullscrape.c ot_accesslist.c ot_http.c ot_livesync.c ot_rijndael.c
+HEADERS=trackerlogic.h ot_urlencoded.h ot_batch.h ot_buf.h ot_event.h ot_fdstate.h ot_fmt.h ot_hex.h ot_ip.h ot_config.h ot_scan.h ot_socket.h ot_tcp.h ot_workqueue.h ot_mutex.h ot_random.h ot_stats.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h ot_rijndael.h
+SOURCES=opentracker.c trackerlogic.c ot_urlencoded.c ot_batch.c ot_buf.c ot_event.c ot_fdstate.c ot_fmt.c ot_hex.c ot_ip.c ot_config.c ot_scan.c ot_socket.c ot_tcp.c ot_workqueue.c ot_mutex.c ot_random.c ot_stats.c ot_vector.c ot_clean.c ot_udp.c ot_iovec.c ot_fullscrape.c ot_accesslist.c ot_http.c ot_livesync.c ot_rijndael.c
 SOURCES_proxy=proxy.c ot_batch.c ot_fdstate.c ot_vector.c ot_mutex.c ot_random.c ot_scan.c ot_ip.c ot_socket.c
 
 all: $(BINARY) $(BINARY).debug
@@ -96,8 +96,8 @@ proxy.debug: $(SOURCES_proxy) $(HEADERS)
 # has to. tests/stubs.c supplies the few symbols those modules reference but
 # the tests never reach.
 TESTBIN=tests/runner
-TEST_SOURCES=tests/runner.c tests/test.c tests/stubs.c tests/t_urlencoded.c tests/t_iovec.c tests/t_vector.c tests/t_random.c tests/t_hex.c tests/t_accesslist.c tests/t_scan.c tests/t_buf.c tests/t_ip.c tests/t_socket.c tests/t_mutex.c tests/t_batch.c tests/t_fdstate.c tests/t_event.c tests/t_fmt.c
-TEST_UNITS=ot_urlencoded.c ot_fmt.c ot_event.c ot_batch.c ot_fdstate.c ot_buf.c ot_ip.c ot_socket.c ot_iovec.c ot_vector.c ot_random.c ot_hex.c ot_scan.c ot_accesslist.c ot_mutex.c
+TEST_SOURCES=tests/runner.c tests/test.c tests/stubs.c tests/t_urlencoded.c tests/t_iovec.c tests/t_vector.c tests/t_random.c tests/t_hex.c tests/t_accesslist.c tests/t_scan.c tests/t_buf.c tests/t_ip.c tests/t_socket.c tests/t_workqueue.c tests/t_batch.c tests/t_fdstate.c tests/t_event.c tests/t_fmt.c
+TEST_UNITS=ot_urlencoded.c ot_fmt.c ot_event.c ot_batch.c ot_fdstate.c ot_buf.c ot_ip.c ot_socket.c ot_iovec.c ot_vector.c ot_random.c ot_hex.c ot_scan.c ot_accesslist.c ot_mutex.c ot_workqueue.c
 TEST_HEADERS=tests/test.h $(HEADERS)
 # Tests reach code that sits behind feature flags, so the units under test are
 # built with those features enabled. A test binary is not a production binary.
diff --git a/man4/opentracker.conf.4 b/man4/opentracker.conf.4
index aa82e1d..1bd1c7a 100644
--- a/man4/opentracker.conf.4
+++ b/man4/opentracker.conf.4
@@ -42,6 +42,14 @@ in the thread that also serves TCP. See
 .Sx TUNING
 before raising it.
 .Pp
+.It listen.tcp.workers Ar threads
+Specifies how many threads accept and answer HTTP connections, for every
+listen statement that does not carry a
+.Cm workers
+option of its own. Defaults to 0, which serves them from the event loop. See
+.Sx TUNING
+before raising it.
+.Pp
 .It listen.shared Ar boolean
 Permits another process to bind the same address, for every listen statement
 that does not carry a
@@ -101,6 +109,14 @@ group only where the answer is yes.
 in the source distribution sweeps the setting against a running tracker.
 Measure on the machine that will carry the traffic.
 .Pp
+A TCP worker accepts a connection and answers it on its own stack, without
+the event loop seeing it at all; only a connection it cannot finish is handed
+over. One worker serves less than none, because accepting leaves the event
+loop's thread without anything gained in return, so the setting is worth
+either 0 or at least 2.
+.Pa tests/http_load.c
+in the source distribution drives the tracker over HTTP to measure it.
+.Pp
 On FreeBSD the
 .Xr accf_data 9
 filter has the kernel hold a new connection until data arrives on it, so the
diff --git a/opentracker.c b/opentracker.c
index b47759e..2f2746d 100644
--- a/opentracker.c
+++ b/opentracker.c
@@ -29,12 +29,14 @@
 #include "ot_ip.h"
 #include "ot_livesync.h"
 #include "ot_mutex.h"
+#include "ot_workqueue.h"
 #include "ot_random.h"
 #include "ot_scan.h"
 #include "ot_event.h"
 #include "ot_fdstate.h"
 #include "ot_socket.h"
 #include "ot_stats.h"
+#include "ot_tcp.h"
 #include "ot_udp.h"
 #include "trackerlogic.h"
 
@@ -96,142 +98,6 @@ static void install_signal_handlers(void) {
 }
 
 
-static ssize_t header_complete(char *request, ssize_t byte_count) {
-  ssize_t i = 0, state = 0;
-
-  for (i = 1; i < byte_count; i += 2)
-    if (request[i] <= 13) {
-      i--;
-      for (state = 0; i < byte_count; ++i) {
-        char c = request[i];
-        if (c == '\r' || c == '\n')
-          state = (state >> 2) | ((c << 6) & 0xc0);
-        else
-          break;
-        if (state >= 0xa0 || state == 0x99)
-          return i + 1;
-      }
-    }
-  return 0;
-}
-
-static void handle_read(const int64_t sock, struct ot_workstruct *ws) {
-  struct http_data *cookie     = fdstate_cookie(sock);
-  ssize_t           byte_count = read((int)sock, ws->inbuf, G_INBUF_SIZE);
-
-  if (byte_count == 0 || (byte_count == -1 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR)) {
-    http_close(sock);
-    return;
-  }
-
-  if (byte_count == -1)
-    return;
-
-  ws->request      = NULL;
-  ws->request_size = 0;
-  ws->header_size  = 0;
-  ws->keep_alive   = 0;
-
-  /* If we get the whole request in one packet, handle it without copying */
-  if (!cookie->request.data) {
-    if ((ws->header_size = header_complete(ws->inbuf, byte_count))) {
-      ws->request      = ws->inbuf;
-      ws->request_size = byte_count;
-      http_handle_request(sock, ws);
-    } else if (buf_cat(&cookie->request, ws->inbuf, (size_t)byte_count))
-      http_close(sock);
-    return;
-  }
-
-  if (buf_cat(&cookie->request, ws->inbuf, (size_t)byte_count) || cookie->request.size > G_INBUF_SIZE) {
-    http_issue_error(sock, ws, CODE_HTTPERROR_500);
-    return;
-  }
-
-  while ((ws->header_size = header_complete(cookie->request.data, (ssize_t)cookie->request.size))) {
-    ws->request      = cookie->request.data;
-    ws->request_size = (ssize_t)cookie->request.size;
-    http_handle_request(sock, ws);
-#ifdef WANT_KEEPALIVE
-    if (!ws->keep_alive)
-#endif
-      return;
-  }
-}
-
-static void handle_write(const int64_t sock) {
-  struct http_data *cookie     = fdstate_cookie(sock);
-  int               chunked    = 0;
-  int               keep_alive = 0;
-
-  if (cookie) {
-    chunked    = (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) != 0;
-    keep_alive = (cookie->flag & STRUCT_HTTP_FLAG_KEEPALIVE) != 0;
-
-    if (cookie->batch) {
-      if (batch_send(sock, cookie->batch) == BATCH_FAILED) {
-        http_close(sock);
-        return;
-      }
-      if (batch_pending(cookie->batch))
-        return;
-    }
-  }
-
-  /* In a chunked transfer, wait for the next piece once everything queued has gone out */
-  if (chunked)
-    http_interest_set(sock, cookie, 0);
-  else if (keep_alive) {
-    /* The reply is out and the client asked to keep the connection, so go
-       back to waiting for the next request */
-    http_interest_set(sock, cookie, EVENT_IN);
-    fdstate_set_expiry(sock, time_future_seconds(OT_CLIENT_TIMEOUT));
-  } else
-    http_close(sock);
-}
-
-/* The fast path, assuming that the complete request has usually arrived by the time a
-   connection is accepted, so it is answered here outside the event loop. The more load
-   a tracker has, the more likely it is that by the time it accepts, the roundtrip to
-   the peer has completed.
-   If the request could not be completely handled, the stack cookie is promoted to heap
-   and the connection handed over to the event loop to finish */
-static void handle_accept(const int64_t serversocket, struct ot_workstruct *ws) {
-  int64_t  sock;
-  ot_ip6   ip;
-  uint16_t port;
-
-  while ((sock = socket_accept(serversocket, ip, &port)) != -1) {
-    struct http_data cookie;
-    uint32_t         generation;
-
-    memset(&cookie, 0, sizeof(cookie));
-    memcpy(cookie.ip, ip, sizeof(ot_ip6));
-    cookie.interest = EVENT_IN;
-    cookie.flag     = STRUCT_HTTP_FLAG_ONSTACK;
-
-    generation = fdstate_open(sock, &cookie, time_future_seconds(OT_CLIENT_TIMEOUT));
-    if (!fdstate_claim((int)sock, generation)) {
-      fdstate_close(sock);
-      event_close((int)sock);
-      continue;
-    }
-
-    stats_issue_event(ws, EVENT_ACCEPT, FLAG_TCP, (uintptr_t)ip);
-
-    handle_read(sock, ws);
-
-    /* Gone already if it was answered and closed inline */
-    if (fdstate_generation((int)sock) != generation)
-      continue;
-
-    if (http_adopt(sock, &cookie) == -1)
-      http_close(sock);
-    else
-      fdstate_release((int)sock);
-  }
-}
-
 /* How many ready descriptors one wait hands back at a time */
 #define OT_EVENT_BATCH 512
 
@@ -244,20 +110,9 @@ static void *server_mainloop(void *args) {
 
   (void)args;
 
-  /* Initialize our "thread local storage" */
-  memset(&ws, 0, sizeof(ws));
-  stats_counters_register(&ws);
-  ws.inbuf  = malloc(G_INBUF_SIZE);
-  ws.outbuf = malloc(G_OUTBUF_SIZE);
-#ifdef _DEBUG_HTTPERROR
-  ws.debugbuf = malloc(G_DEBUGBUF_SIZE);
-#endif
-
-  if (!ws.inbuf || !ws.outbuf)
+  if (workstruct_init(&ws) == -1)
     panic("Initializing worker failed");
 
-  random_bytes(ws.rand48_state, sizeof(ws.rand48_state));
-
   while (g_opentracker_running) {
     ot_event ready[OT_EVENT_BATCH];
     int64_t  sock;
@@ -277,22 +132,24 @@ static void *server_mainloop(void *args) {
       cookie = fdstate_cookie((int)sock);
 
       if ((intptr_t)cookie == FLAG_TCP)
-        handle_accept(sock, &ws);
+        tcp_accept(sock, &ws);
       else if ((intptr_t)cookie == FLAG_UDP)
         handle_udp6(sock, &ws);
       else if ((intptr_t)cookie == FLAG_SELFPIPE)
         while (read((int)sock, ws.inbuf, G_INBUF_SIZE) > 0)
           ;
-      else {
+      else if (fdstate_claim((int)sock, ready[i].generation)) {
         if (ready[i].what & EVENT_IN)
-          handle_read(sock, &ws);
+          tcp_read(sock, &ws);
         /* The read may have closed it, so look again before writing */
         if ((ready[i].what & EVENT_OUT) && fdstate_generation((int)sock) == ready[i].generation)
-          handle_write(sock);
+          tcp_write(sock);
+        if (fdstate_generation((int)sock) == ready[i].generation)
+          fdstate_release((int)sock);
       }
     }
 
-    while ((sock = mutex_workqueue_popresult(&iovec_entries, &iovector, &is_partial)) != -1)
+    while ((sock = workqueue_popresult(&iovec_entries, &iovector, &is_partial)) != -1)
       http_sendiovecdata(sock, &ws, iovec_entries, iovector, is_partial);
 
     if (time_now_seconds() > next_timeout_check) {
@@ -308,13 +165,7 @@ static void *server_mainloop(void *args) {
 
   /* This work struct is about to go out of scope, so hand its tally over
      before anything can read it there */
-  stats_counters_retire(&ws);
-
-  free(ws.inbuf);
-  free(ws.outbuf);
-#ifdef _DEBUG_HTTPERROR
-  free(ws.debugbuf);
-#endif
+  workstruct_deinit(&ws);
   return 0;
 }
 
@@ -323,7 +174,7 @@ static void *server_mainloop(void *args) {
    member asked for it before binding, the first one included, so whether
    this entry will have workers has to be known here. */
 static socket_sharing listen_sharing_for(const ot_listen *entry) {
-  if (entry->proto == FLAG_UDP && entry->workers > 1 && socket_can_balance())
+  if (entry->proto != FLAG_MCA && entry->workers > 1 && socket_can_balance())
     return SOCKET_BALANCED;
   return entry->shared ? SOCKET_SHARED : SOCKET_EXCLUSIVE;
 }
@@ -379,10 +230,41 @@ static void listen_bind_all(void) {
     listen_bind_one(entries + i);
 }
 
-/* The socket the event loop was using becomes the first worker's. Where the
-   kernel spreads datagrams, every further worker binds one of its own, so
-   none of them share a receive queue.
-   All others read from the first */
+/* The socket the event loop was using is passed to the first worker.
+   If start up tests show that the kernel can spread connections over
+   a group bound to one address, every worker binds one of its own.
+   If not, all workers call accept on the first. */
+static void listen_start_tcp_workers(void) {
+  size_t     count, i;
+  ot_listen *entries = config_listen_entries(&count);
+
+  for (i = 0; i < count; ++i) {
+    ot_listen *entry   = entries + i;
+    int        balance = listen_sharing_for(entry) == SOCKET_BALANCED;
+    int        worker;
+
+    if (entry->proto != FLAG_TCP || entry->workers <= 0)
+      continue;
+
+    event_del((int)entry->sock);
+    tcp_worker_install(entry->sock);
+
+    for (worker = 1; worker < entry->workers; ++worker) {
+      int64_t sock = entry->sock;
+
+      if (balance) {
+        sock = socket_tcp_listen(entry->ip, entry->port, SOCKET_BALANCED);
+        if (sock == -1) {
+          fprintf(stderr, "Warning: could not bind another socket for tcp worker %d: %s. Sharing the first instead.\n", worker,
+                  strerror(errno));
+          sock = entry->sock;
+        }
+      }
+      tcp_worker_install(sock);
+    }
+  }
+}
+
 static void listen_start_udp_workers(void) {
   size_t     count, i;
   ot_listen *entries = config_listen_entries(&count);
@@ -543,9 +425,11 @@ int main(int argc, char **argv) {
 
   install_signal_handlers();
 
+  listen_start_tcp_workers();
   listen_start_udp_workers();
 
   server_mainloop(0);
+  tcp_workers_deinit();
   udp_workers_deinit();
 
   /* No thread is serving any more, so the teardown can run where a debugger
diff --git a/opentracker.conf.sample b/opentracker.conf.sample
index 1d535e8..f075703 100644
--- a/opentracker.conf.sample
+++ b/opentracker.conf.sample
@@ -21,9 +21,9 @@
 #
 #      A listen statement takes options after the address:
 #
-#        workers=<count>  handle this udp socket in blocking reads in that
-#                         many dedicated worker threads instead of in the
-#                         event loop. Defaults to 1. Ignored for tcp.
+#        workers=<count>  handle this socket in that many dedicated worker
+#                         threads instead of in the event loop. Defaults to 1
+#                         for udp and 0 for tcp.
 #        shared           allow another process to bind the same address and
 #                         let the kernel share the traffic. Only useful if you
 #                         deliberately run a second tracker there.
@@ -35,6 +35,7 @@
 #      anywhere.
 #
 # listen.udp.workers 4
+# listen.tcp.workers 4
 # listen.shared 1
 #
 #      Raising the worker count pays off where the kernel spreads arriving
diff --git a/ot_fdstate.c b/ot_fdstate.c
index 7d742da..09daf50 100644
--- a/ot_fdstate.c
+++ b/ot_fdstate.c
@@ -91,7 +91,7 @@ uint32_t fdstate_open(int fd, void *cookie, ot_time expiry) {
   }
 
   word = atomic_load_explicit(&slot->state, memory_order_relaxed);
-  atomic_store_explicit(&slot->cookie, cookie, memory_order_relaxed);
+  atomic_store_explicit(&slot->cookie, cookie, memory_order_release);
   atomic_store_explicit(&slot->expiry, expiry, memory_order_relaxed);
   /* Publishes the two above to whoever claims the slot next */
   atomic_store_explicit(&slot->state, FD_WORD(FD_GEN(word), FD_IDLE), memory_order_release);
@@ -183,14 +183,17 @@ int fdstate_next_expired(ot_time now, int *cursor) {
 void *fdstate_cookie(int fd) {
   ot_fdslot *slot = fdstate_slot(fd);
 
-  return slot ? atomic_load_explicit(&slot->cookie, memory_order_relaxed) : NULL;
+  /* Pairs with the release in fdstate_set_cookie and fdstate_open */
+  return slot ? atomic_load_explicit(&slot->cookie, memory_order_acquire) : NULL;
 }
 
 void fdstate_set_cookie(int fd, void *cookie) {
   ot_fdslot *slot = fdstate_slot(fd);
 
+  /* Publishes what the cookie points at to whoever reads it next, which may
+     be another thread entirely */
   if (slot)
-    atomic_store_explicit(&slot->cookie, cookie, memory_order_relaxed);
+    atomic_store_explicit(&slot->cookie, cookie, memory_order_release);
 }
 
 void fdstate_set_expiry(int fd, ot_time expiry) {
diff --git a/ot_fullscrape.c b/ot_fullscrape.c
index a86d6d7..ae29308 100644
--- a/ot_fullscrape.c
+++ b/ot_fullscrape.c
@@ -23,6 +23,7 @@
 #include "ot_fullscrape.h"
 #include "ot_iovec.h"
 #include "ot_mutex.h"
+#include "ot_workqueue.h"
 #include "ot_urlencoded.h"
 #include "trackerlogic.h"
 
@@ -54,7 +55,7 @@ static void *fullscrape_worker(void *args) {
 
   while (g_opentracker_running) {
     ot_tasktype tasktype = TASK_FULLSCRAPE;
-    ot_taskid   taskid   = mutex_workqueue_poptask(&tasktype);
+    ot_taskid   taskid   = workqueue_poptask(&tasktype);
     if (!taskid)
       break;
 #ifdef WANT_COMPRESSION_ZSTD
@@ -68,7 +69,7 @@ static void *fullscrape_worker(void *args) {
     else
 #endif
       fullscrape_make(taskid, tasktype);
-    mutex_workqueue_pushchunked(taskid, NULL);
+    workqueue_pushchunked(taskid, NULL);
   }
   return NULL;
 }
@@ -79,12 +80,12 @@ void fullscrape_init(void) {
 }
 
 void fullscrape_deinit(void) {
-  mutex_workqueue_wakeall();
+  workqueue_wakeall();
   pthread_join( thread_id, NULL );
 }
 
 void fullscrape_deliver( int64_t sock, ot_tasktype tasktype ) {
-  mutex_workqueue_pushtask( sock, TASK_FULLSCRAPE | tasktype );
+  workqueue_pushtask( sock, TASK_FULLSCRAPE | tasktype );
 }
 
 static void fullscrape_write_one( ot_tasktype mode, char **cursor, ot_torrent *torrent, ot_hash *hash ) {
@@ -170,7 +171,7 @@ static void fullscrape_make(int taskid, ot_tasktype mode) {
       if (r > re) {
         iovector.iov_len = r - (char *)iovector.iov_base;
 
-        if (mutex_workqueue_pushchunked(taskid, &iovector)) {
+        if (workqueue_pushchunked(taskid, &iovector)) {
           free(iovector.iov_base);
           mutex_bucket_unlock(bucket, 0);
           return;
@@ -200,7 +201,7 @@ static void fullscrape_make(int taskid, ot_tasktype mode) {
 
   /* Send rest of data */
   iovector.iov_len = r - (char *)iovector.iov_base;
-  if (mutex_workqueue_pushchunked(taskid, &iovector))
+  if (workqueue_pushchunked(taskid, &iovector))
     free(iovector.iov_base);
 }
 
@@ -251,7 +252,7 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
       while (!strm.avail_out) {
         iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base;
 
-        if (mutex_workqueue_pushchunked(taskid, &iovector)) {
+        if (workqueue_pushchunked(taskid, &iovector)) {
           free(iovector.iov_base);
           mutex_bucket_unlock(bucket, 0);
           return;
@@ -291,7 +292,7 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
     fprintf(stderr, "deflate() failed while in fullscrape_make()'s endgame.\n");
 
   iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base;
-  if (mutex_workqueue_pushchunked(taskid, &iovector)) {
+  if (workqueue_pushchunked(taskid, &iovector)) {
     free(iovector.iov_base);
     deflateEnd(&strm);
     return;
@@ -314,7 +315,7 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
 
       /* Only pass the new buffer if there actually was some data left in the buffer */
       iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base;
-      if (!iovector.iov_len || mutex_workqueue_pushchunked(taskid, &iovector))
+      if (!iovector.iov_len || workqueue_pushchunked(taskid, &iovector))
         free(iovector.iov_base);
   }
 
@@ -380,7 +381,7 @@ static void fullscrape_make_zstd(int taskid, ot_tasktype mode) {
       while (outbuf.pos + OT_SCRAPE_MAXENTRYLEN > outbuf.size) {
         iovector.iov_len = outbuf.size;
 
-        if (mutex_workqueue_pushchunked(taskid, &iovector)) {
+        if (workqueue_pushchunked(taskid, &iovector)) {
           free(iovector.iov_base);
           ZSTD_freeCCtx(zstream);
           return mutex_bucket_unlock(bucket, 0);
@@ -418,7 +419,7 @@ static void fullscrape_make_zstd(int taskid, ot_tasktype mode) {
   more_bytes = ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_end);
 
   iovector.iov_len = outbuf.pos;
-  if (mutex_workqueue_pushchunked(taskid, &iovector)) {
+  if (workqueue_pushchunked(taskid, &iovector)) {
     free(iovector.iov_base);
     ZSTD_freeCCtx(zstream);
     return;
@@ -443,7 +444,7 @@ static void fullscrape_make_zstd(int taskid, ot_tasktype mode) {
 
       /* Only pass the new buffer if there actually was some data left in the buffer */
       iovector.iov_len = outbuf.pos;
-      if (!iovector.iov_len || mutex_workqueue_pushchunked(taskid, &iovector))
+      if (!iovector.iov_len || workqueue_pushchunked(taskid, &iovector))
         free(iovector.iov_base);
   }
 
diff --git a/ot_fullscrape.h b/ot_fullscrape.h
index 0e7690c..b759a27 100644
--- a/ot_fullscrape.h
+++ b/ot_fullscrape.h
@@ -9,6 +9,7 @@
 #ifdef WANT_FULLSCRAPE
 
 #include "ot_mutex.h"
+#include "ot_workqueue.h"
 
 void fullscrape_init(void);
 void fullscrape_deinit(void);
diff --git a/ot_http.c b/ot_http.c
index bb6ff24..b0e1de3 100644
--- a/ot_http.c
+++ b/ot_http.c
@@ -22,6 +22,7 @@
 #include "ot_iovec.h"
 #include "ot_ip.h"
 #include "ot_mutex.h"
+#include "ot_workqueue.h"
 #include "ot_scan.h"
 #include "ot_event.h"
 #include "ot_fdstate.h"
@@ -39,22 +40,30 @@ ssize_t      g_stats_path_len;
 
 enum { SUCCESS_HTTP_HEADER_LENGTH = 80, SUCCESS_HTTP_SIZE_OFF = 17 };
 
-int http_adopt(const int64_t sock, struct http_data *cookie) {
-  struct http_data *heap = malloc(sizeof(*heap));
+struct http_data *http_adopt(const int64_t sock) {
+  struct http_data *cookie = fdstate_cookie(sock), *heap;
+  int               interest;
 
-  if (!heap)
-    return -1;
+  if (!cookie || !(cookie->flag & STRUCT_HTTP_FLAG_ONSTACK))
+    return cookie;
+
+  if (!(heap = malloc(sizeof(*heap))))
+    return NULL;
 
   *heap = *cookie;
   heap->flag &= ~STRUCT_HTTP_FLAG_ONSTACK;
+
+  /* Publishing the cookie is what lets another thread reach this connection,
+     so nothing may touch it afterwards, not even to read what it asked for */
+  interest = heap->interest;
   fdstate_set_cookie((int)sock, heap);
 
-  if (event_add((int)sock, heap->interest, fdstate_generation((int)sock)) == -1) {
+  if (event_add((int)sock, interest, fdstate_generation((int)sock)) == -1) {
     fdstate_set_cookie((int)sock, cookie);
     free(heap);
-    return -1;
+    return NULL;
   }
-  return 0;
+  return heap;
 }
 
 /* Give up a connection: the one place that knows everything hanging off it. */
@@ -69,7 +78,7 @@ void http_close(const int64_t sock) {
     free(cookie->batch);
     buf_reset(&cookie->request);
     if (cookie->flag & (STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER))
-      mutex_workqueue_canceltask(sock);
+      workqueue_canceltask(sock);
     if (!(cookie->flag & STRUCT_HTTP_FLAG_ONSTACK))
       free(cookie);
   }
@@ -348,13 +357,17 @@ static ssize_t http_handle_stats(const int64_t sock, struct ot_workstruct *ws, c
     }
 #endif
 #endif
-    /* Pass this task to the worker thread */
     cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED;
 
     /* Clients waiting for us should not easily timeout */
     fdstate_set_expiry(sock, 0);
-    fullscrape_deliver(sock, format);
     http_interest_set(sock, cookie, 0);
+
+    /* Everything this connection needs has been written, so it can be given
+       a home of its own and only then be made reachable by a worker thread */
+    if (!http_adopt(sock))
+      HTTPERROR_500;
+    fullscrape_deliver(sock, format);
     return ws->reply_size = -2;
   }
 #endif
@@ -366,6 +379,9 @@ static ssize_t http_handle_stats(const int64_t sock, struct ot_workstruct *ws, c
 
     /* Complex stats also include expensive memory debugging tools */
     fdstate_set_expiry(sock, 0);
+
+    if (!http_adopt(sock))
+      HTTPERROR_500;
     stats_deliver(sock, mode);
     return ws->reply_size = -2;
   }
@@ -448,8 +464,11 @@ static ssize_t http_handle_fullscrape(const int64_t sock, struct ot_workstruct *
   cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED;
   /* Clients waiting for us should not easily timeout */
   fdstate_set_expiry(sock, 0);
-  fullscrape_deliver(sock, format);
   http_interest_set(sock, cookie, 0);
+
+  if (!http_adopt(sock))
+    HTTPERROR_500;
+  fullscrape_deliver(sock, format);
   return ws->reply_size = -2;
 }
 #endif
diff --git a/ot_http.h b/ot_http.h
index 82d645b..9cacd21 100644
--- a/ot_http.h
+++ b/ot_http.h
@@ -40,9 +40,10 @@ void           http_interest_set(const int64_t s, struct http_data *cookie, int
 /* Close a connection and release everything that belongs to it */
 void           http_close(const int64_t s);
 
-/* Promote a stack cookie to a heap one for persistence and handing it over
-   to the event loop. */
-int            http_adopt(const int64_t s, struct http_data *cookie);
+/* Promote a stack cookie to a heap one, for persistence and for handing it
+   over to the event loop or to another thread. Does nothing to one that is
+   already on the heap. Returns the connection's state, or NULL. */
+struct http_data *http_adopt(const int64_t s);
 
 ssize_t        http_handle_request(const int64_t s, struct ot_workstruct *ws);
 ssize_t        http_sendiovecdata(const int64_t s, struct ot_workstruct *ws, size_t iovec_entries, struct iovec *iovector, int is_partial);
diff --git a/ot_mutex.c b/ot_mutex.c
index 5392264..d856d5f 100644
--- a/ot_mutex.c
+++ b/ot_mutex.c
@@ -31,7 +31,6 @@ static pthread_mutex_t bucket_mutex[OT_BUCKET_COUNT];
 static _Atomic size_t  g_torrent_count;
 
 /* Self pipe from opentracker.c */
-extern int             g_self_pipe[2];
 
 ot_vector *mutex_bucket_lock(int bucket) {
   pthread_mutex_lock(bucket_mutex + bucket);
@@ -60,237 +59,17 @@ void mutex_bucket_unlock_by_hash(ot_hash const hash, int delta_torrentcount) {
 
 size_t mutex_get_torrent_count(void) { return atomic_load_explicit(&g_torrent_count, memory_order_relaxed); }
 
-/* TaskQueue Magic */
-
-struct ot_task {
-  ot_taskid       taskid;
-  ot_tasktype     tasktype;
-  int64_t         sock;
-  size_t          iovec_entries;
-  struct iovec   *iovec;
-  struct ot_task *next;
-};
-
-static ot_taskid       next_free_taskid = 1;
-static struct ot_task *tasklist;
-static pthread_mutex_t tasklist_mutex;
-static pthread_cond_t  tasklist_being_filled;
-
-int mutex_workqueue_pushtask(int64_t sock, ot_tasktype tasktype) {
-  struct ot_task **tmptask, *task;
-
-  task = malloc(sizeof(struct ot_task));
-  if (!task)
-    return -1;
-
-  task->taskid        = 0;
-  task->tasktype      = tasktype;
-  task->sock          = sock;
-  task->iovec_entries = 0;
-  task->iovec         = NULL;
-  task->next          = 0;
-
-  /* Want exclusive access to tasklist */
-  pthread_mutex_lock(&tasklist_mutex);
-
-  /* Skip to end of list */
-  tmptask = &tasklist;
-  while (*tmptask)
-    tmptask = &(*tmptask)->next;
-  *tmptask = task;
-
-  /* Inform waiting workers and release lock */
-  pthread_cond_broadcast(&tasklist_being_filled);
-  pthread_mutex_unlock(&tasklist_mutex);
-  return 0;
-}
-
-void mutex_workqueue_canceltask(int64_t sock) {
-  struct ot_task **task;
-
-  /* Want exclusive access to tasklist */
-  pthread_mutex_lock(&tasklist_mutex);
-
-  for (task = &tasklist; *task; task = &((*task)->next))
-    if ((*task)->sock == sock) {
-      struct iovec   *iovec = (*task)->iovec;
-      struct ot_task *ptask = *task;
-      size_t                   i;
-
-      /* Free task's iovec, the buffers it points at and the vector itself */
-      for (i = 0; i < (*task)->iovec_entries; ++i)
-        free(iovec[i].iov_base);
-      free(iovec);
-
-      *task = (*task)->next;
-      free(ptask);
-      break;
-    }
-
-  /* Release lock */
-  pthread_mutex_unlock(&tasklist_mutex);
-}
-
-ot_taskid mutex_workqueue_poptask(ot_tasktype *tasktype) {
-  struct ot_task *task;
-  ot_taskid       taskid = 0;
-
-  /* Want exclusive access to tasklist */
-  pthread_mutex_lock(&tasklist_mutex);
-
-  while (!taskid) {
-    /* Skip to the first unassigned task this worker wants to do */
-    for (task = tasklist; task; task = task->next)
-      if (!task->taskid && (TASK_CLASS_MASK & task->tasktype) == *tasktype) {
-        /* If we found an outstanding task, assign a taskid to it
-           and leave the loop */
-        task->taskid = taskid = ++next_free_taskid;
-        *tasktype             = task->tasktype;
-        break;
-      }
-
-    /* Wait until the next task is being fed, unless the tracker is on its
-       way out, in which case the worker is meant to leave rather than wait */
-    if (!taskid) {
-      if (!g_opentracker_running)
-        break;
-      pthread_cond_wait(&tasklist_being_filled, &tasklist_mutex);
-    }
-  }
-
-  /* Release lock */
-  pthread_mutex_unlock(&tasklist_mutex);
-
-  return taskid;
-}
-
-void mutex_workqueue_pushsuccess(ot_taskid taskid) {
-  struct ot_task **task;
-
-  /* Want exclusive access to tasklist */
-  pthread_mutex_lock(&tasklist_mutex);
-
-  for (task = &tasklist; *task; task = &((*task)->next))
-    if ((*task)->taskid == taskid) {
-      struct ot_task *ptask = *task;
-      *task = (*task)->next;
-      free(ptask);
-      break;
-    }
-
-  /* Release lock */
-  pthread_mutex_unlock(&tasklist_mutex);
-}
-
-int mutex_workqueue_pushresult(ot_taskid taskid, size_t iovec_entries, struct iovec *iovec) {
-  struct ot_task *task;
-  const char      byte = 'o';
-
-  /* Want exclusive access to tasklist */
-  pthread_mutex_lock(&tasklist_mutex);
-
-  for (task = tasklist; task; task = task->next)
-    if (task->taskid == taskid) {
-      task->iovec_entries = iovec_entries;
-      task->iovec         = iovec;
-      task->tasktype      = TASK_DONE;
-      break;
-    }
-
-  /* Release lock */
-  pthread_mutex_unlock(&tasklist_mutex);
-
-  /* A full pipe has already done this nudge's job, so a failure is fine */
-  (void)!write(g_self_pipe[1], &byte, 1);
-
-  /* Indicate whether the worker has to throw away results */
-  return task ? 0 : -1;
-}
-
-int mutex_workqueue_pushchunked(ot_taskid taskid, struct iovec *iovec) {
-  struct ot_task *task;
-  const char      byte = 'o';
-
-  /* Want exclusive access to tasklist */
-  pthread_mutex_lock(&tasklist_mutex);
-
-  for (task = tasklist; task; task = task->next)
-    if (task->taskid == taskid) {
-      if (iovec) {
-        if (iovec_append(&task->iovec_entries, &task->iovec, iovec))
-          task->tasktype = TASK_DONE_PARTIAL;
-        else
-          task = NULL;
-      } else
-        task->tasktype = TASK_DONE;
-      break;
-    }
-
-  /* Release lock */
-  pthread_mutex_unlock(&tasklist_mutex);
-
-  /* A full pipe has already done this nudge's job, so a failure is fine */
-  (void)!write(g_self_pipe[1], &byte, 1);
-
-  /* Indicate whether the worker has to throw away results */
-  return task ? 0 : -1;
-}
-
-int64_t mutex_workqueue_popresult(size_t *iovec_entries, struct iovec **iovec, int *is_partial) {
-  struct ot_task **task;
-  int64_t sock = -1;
-
-  *is_partial = 0;
-
-  /* Want exclusive access to tasklist */
-  pthread_mutex_lock(&tasklist_mutex);
-
-  for (task = &tasklist; *task; task = &((*task)->next))
-    if (((*task)->tasktype & TASK_CLASS_MASK) == TASK_DONE) {
-      struct ot_task *ptask = *task;
-      *iovec_entries        = ptask->iovec_entries;
-      *iovec                = ptask->iovec;
-      sock                  = ptask->sock;
-
-      if ((*task)->tasktype == TASK_DONE) {
-        *task = ptask->next;
-        free(ptask);
-      } else {
-        ptask->iovec_entries = 0;
-        ptask->iovec         = NULL;
-        *is_partial          = 1;
-        /* Prevent task from showing up immediately again unless new data was added */
-        (*task)->tasktype    = TASK_FULLSCRAPE;
-      }
-      break;
-    }
-
-  /* Release lock */
-  pthread_mutex_unlock(&tasklist_mutex);
-  return sock;
-}
-
 void mutex_init(void) {
   size_t i;
-  pthread_mutex_init(&tasklist_mutex, NULL);
-  pthread_cond_init(&tasklist_being_filled, NULL);
   for (i = 0; i < OT_BUCKET_COUNT; ++i)
     pthread_mutex_init(bucket_mutex + i, NULL);
   memset(all_torrents, 0, sizeof(all_torrents));
 }
 
 /* Let every worker parked on the queue see that the tracker is stopping */
-void mutex_workqueue_wakeall(void) {
-  pthread_mutex_lock(&tasklist_mutex);
-  pthread_cond_broadcast(&tasklist_being_filled);
-  pthread_mutex_unlock(&tasklist_mutex);
-}
-
 void mutex_deinit(void) {
   size_t i;
   for (i = 0; i < OT_BUCKET_COUNT; ++i)
     pthread_mutex_destroy(bucket_mutex + i);
-  pthread_mutex_destroy(&tasklist_mutex);
-  pthread_cond_destroy(&tasklist_being_filled);
   memset(all_torrents, 0, sizeof(all_torrents));
 }
diff --git a/ot_mutex.h b/ot_mutex.h
index c048fdb..c1218d7 100644
--- a/ot_mutex.h
+++ b/ot_mutex.h
@@ -10,7 +10,6 @@
 #include "trackerlogic.h"
 
 void       mutex_init(void);
-void       mutex_workqueue_wakeall(void);
 void       mutex_deinit(void);
 
 ot_vector *mutex_bucket_lock(int bucket);
@@ -21,60 +20,4 @@ void       mutex_bucket_unlock_by_hash(ot_hash const hash, int delta_torrentcoun
 
 size_t     mutex_get_torrent_count(void);
 
-typedef enum {
-  TASK_STATS_CONNS               = 0x0001,
-  TASK_STATS_TCP                 = 0x0002,
-  TASK_STATS_UDP                 = 0x0003,
-  TASK_STATS_SCRAPE              = 0x0004,
-  TASK_STATS_FULLSCRAPE          = 0x0005,
-  TASK_STATS_TPB                 = 0x0006,
-  TASK_STATS_HTTPERRORS          = 0x0007,
-  TASK_STATS_VERSION             = 0x0008,
-  TASK_STATS_RENEW               = 0x000a,
-  TASK_STATS_SYNCS               = 0x000b,
-  TASK_STATS_COMPLETED           = 0x000c,
-  TASK_STATS_NUMWANTS            = 0x000d,
-
-  TASK_STATS                     = 0x0100, /* Mask */
-  TASK_STATS_TORRENTS            = 0x0101,
-  TASK_STATS_PEERS               = 0x0102,
-  TASK_STATS_SLASH24S            = 0x0103,
-  TASK_STATS_TOP10               = 0x0104,
-  TASK_STATS_TOP100              = 0x0105,
-  TASK_STATS_EVERYTHING          = 0x0106,
-  TASK_STATS_FULLLOG             = 0x0107,
-  TASK_STATS_WOODPECKERS         = 0x0108,
-
-  TASK_FULLSCRAPE                = 0x0200, /* Default mode */
-  TASK_FULLSCRAPE_TPB_BINARY     = 0x0201,
-  TASK_FULLSCRAPE_TPB_ASCII      = 0x0202,
-  TASK_FULLSCRAPE_TPB_ASCII_PLUS = 0x0203,
-  TASK_FULLSCRAPE_TPB_URLENCODED = 0x0204,
-  TASK_FULLSCRAPE_TRACKERSTATE   = 0x0205,
-
-  TASK_DMEM                      = 0x0300,
-
-  TASK_DONE                      = 0x0f00,
-  TASK_DONE_PARTIAL              = 0x0f01,
-
-  TASK_FLAG_GZIP                 = 0x1000,
-  TASK_FLAG_BZIP2                = 0x2000,
-  TASK_FLAG_ZSTD                 = 0x4000,
-  TASK_FLAG_CHUNKED              = 0x8000,
-
-  TASK_TASK_MASK                 = 0x0fff,
-  TASK_CLASS_MASK                = 0x0f00,
-  TASK_FLAGS_MASK                = 0xf000
-} ot_tasktype;
-
-typedef unsigned long ot_taskid;
-
-int        mutex_workqueue_pushtask(int64_t sock, ot_tasktype tasktype);
-void       mutex_workqueue_canceltask(int64_t sock);
-void       mutex_workqueue_pushsuccess(ot_taskid taskid);
-ot_taskid  mutex_workqueue_poptask(ot_tasktype *tasktype);
-int        mutex_workqueue_pushresult(ot_taskid taskid, size_t iovec_entries, struct iovec *iovector);
-int        mutex_workqueue_pushchunked(ot_taskid taskid, struct iovec *iovec);
-int64_t    mutex_workqueue_popresult(size_t *iovec_entries, struct iovec **iovector, int *is_partial);
-
 #endif
diff --git a/ot_stats.c b/ot_stats.c
index 5f97a2f..d38355a 100644
--- a/ot_stats.c
+++ b/ot_stats.c
@@ -23,6 +23,7 @@
 #include "ot_iovec.h"
 #include "ot_ip.h"
 #include "ot_mutex.h"
+#include "ot_workqueue.h"
 #include "ot_fmt.h"
 #include "ot_stats.h"
 #include "trackerlogic.h"
@@ -803,17 +804,17 @@ static void *stats_worker(void *args) {
 
   while (g_opentracker_running) {
     ot_tasktype tasktype = TASK_STATS;
-    ot_taskid   taskid   = mutex_workqueue_poptask(&tasktype);
+    ot_taskid   taskid   = workqueue_poptask(&tasktype);
     if (!taskid)
       break;
     stats_make(&iovec_entries, &iovector, tasktype);
-    if (mutex_workqueue_pushresult(taskid, iovec_entries, iovector))
+    if (workqueue_pushresult(taskid, iovec_entries, iovector))
       iovec_free(&iovec_entries, &iovector);
   }
   return NULL;
 }
 
-void stats_deliver(int64_t sock, int tasktype) { mutex_workqueue_pushtask(sock, tasktype); }
+void stats_deliver(int64_t sock, int tasktype) { workqueue_pushtask(sock, tasktype); }
 
 static pthread_t thread_id;
 void stats_init(void) {
@@ -822,6 +823,6 @@ void stats_init(void) {
 }
 
 void stats_deinit(void) {
-  mutex_workqueue_wakeall();
+  workqueue_wakeall();
   pthread_join(thread_id, NULL);
 }
diff --git a/ot_sync.c b/ot_sync.c
index 3668aa1..c19cf36 100644
--- a/ot_sync.c
+++ b/ot_sync.c
@@ -14,6 +14,7 @@
 #include "ot_fmt.h"
 #include "ot_iovec.h"
 #include "ot_mutex.h"
+#include "ot_workqueue.h"
 #include "ot_scan.h"
 #include "ot_stats.h"
 #include "ot_sync.h"
@@ -139,10 +140,10 @@ static void *sync_worker(void *args) {
 
   while (1) {
     ot_tasktype tasktype = TASK_SYNC_OUT;
-    ot_taskid   taskid   = mutex_workqueue_poptask(&tasktype);
+    ot_taskid   taskid   = workqueue_poptask(&tasktype);
     sync_make(&iovec_entries, &iovector);
     stats_issue_event(ws, EVENT_SYNC_OUT, FLAG_TCP, iovec_length(&iovec_entries, &iovector));
-    if (mutex_workqueue_pushresult(taskid, iovec_entries, iovector))
+    if (workqueue_pushresult(taskid, iovec_entries, iovector))
       iovec_free(&iovec_entries, &iovector);
   }
   return NULL;
@@ -158,7 +159,7 @@ void sync_deinit( ) {
 }
 
 void sync_deliver( int64_t socket ) {
-  mutex_workqueue_pushtask( socket, TASK_SYNC_OUT );
+  workqueue_pushtask( socket, TASK_SYNC_OUT );
 }
 
 #endif
diff --git a/ot_tcp.c b/ot_tcp.c
new file mode 100644
index 0000000..47697ca
--- /dev/null
+++ b/ot_tcp.c
@@ -0,0 +1,238 @@
+/* This software was written by Dirk Engling <erdgeist@erdgeist.org>
+   It is considered beerware. Prost. Skol. Cheers or whatever.
+
+   $id$ */
+
+/* System */
+#include <errno.h>
+#include <pthread.h>
+#include <stdatomic.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+/* Opentracker */
+#include "ot_batch.h"
+#include "ot_buf.h"
+#include "ot_event.h"
+#include "ot_fdstate.h"
+#include "ot_http.h"
+#include "ot_mutex.h"
+#include "ot_socket.h"
+#include "ot_stats.h"
+#include "ot_tcp.h"
+#include "trackerlogic.h"
+
+static ssize_t header_complete(char *request, ssize_t byte_count) {
+  ssize_t i = 0, state = 0;
+
+  for (i = 1; i < byte_count; i += 2)
+    if (request[i] <= 13) {
+      i--;
+      for (state = 0; i < byte_count; ++i) {
+        char c = request[i];
+        if (c == '\r' || c == '\n')
+          state = (state >> 2) | ((c << 6) & 0xc0);
+        else
+          break;
+        if (state >= 0xa0 || state == 0x99)
+          return i + 1;
+      }
+    }
+  return 0;
+}
+
+void tcp_read(const int64_t sock, struct ot_workstruct *ws) {
+  struct http_data *cookie     = fdstate_cookie(sock);
+  ssize_t           byte_count = read((int)sock, ws->inbuf, G_INBUF_SIZE);
+
+  if (byte_count == 0 || (byte_count == -1 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR)) {
+    http_close(sock);
+    return;
+  }
+
+  if (byte_count == -1)
+    return;
+
+  ws->request      = NULL;
+  ws->request_size = 0;
+  ws->header_size  = 0;
+  ws->keep_alive   = 0;
+
+  /* If we get the whole request in one packet, handle it without copying */
+  if (!cookie->request.data) {
+    if ((ws->header_size = header_complete(ws->inbuf, byte_count))) {
+      ws->request      = ws->inbuf;
+      ws->request_size = byte_count;
+      http_handle_request(sock, ws);
+    } else if (buf_cat(&cookie->request, ws->inbuf, (size_t)byte_count))
+      http_close(sock);
+    return;
+  }
+
+  if (buf_cat(&cookie->request, ws->inbuf, (size_t)byte_count) || cookie->request.size > G_INBUF_SIZE) {
+    http_issue_error(sock, ws, CODE_HTTPERROR_500);
+    return;
+  }
+
+  while ((ws->header_size = header_complete(cookie->request.data, (ssize_t)cookie->request.size))) {
+    ws->request      = cookie->request.data;
+    ws->request_size = (ssize_t)cookie->request.size;
+    http_handle_request(sock, ws);
+#ifdef WANT_KEEPALIVE
+    if (!ws->keep_alive)
+#endif
+      return;
+  }
+}
+
+void tcp_write(const int64_t sock) {
+  struct http_data *cookie     = fdstate_cookie(sock);
+  int               chunked    = 0;
+  int               keep_alive = 0;
+
+  if (cookie) {
+    chunked    = (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) != 0;
+    keep_alive = (cookie->flag & STRUCT_HTTP_FLAG_KEEPALIVE) != 0;
+
+    if (cookie->batch) {
+      if (batch_send(sock, cookie->batch) == BATCH_FAILED) {
+        http_close(sock);
+        return;
+      }
+      if (batch_pending(cookie->batch))
+        return;
+    }
+  }
+
+  /* In a chunked transfer, wait for the next piece once everything queued has gone out */
+  if (chunked)
+    http_interest_set(sock, cookie, 0);
+  else if (keep_alive) {
+    /* The reply is out and the client asked to keep the connection, so go
+       back to waiting for the next request */
+    http_interest_set(sock, cookie, EVENT_IN);
+    fdstate_set_expiry(sock, time_future_seconds(OT_CLIENT_TIMEOUT));
+  } else
+    http_close(sock);
+}
+
+/* The fast path, assuming that the complete request has usually arrived by the time a
+   connection is accepted, so it is answered here outside the event loop. The more load
+   a tracker has, the more likely it is that by the time it accepts, the roundtrip to
+   the peer has completed.
+   If the request could not be completely handled, the stack cookie is promoted to heap
+   and the connection handed over to the event loop to finish */
+void tcp_accept(const int64_t serversocket, struct ot_workstruct *ws) {
+  int64_t  sock;
+  ot_ip6   ip;
+  uint16_t port;
+
+  while ((sock = socket_accept(serversocket, ip, &port)) != -1) {
+    struct http_data cookie;
+    uint32_t         generation;
+
+    memset(&cookie, 0, sizeof(cookie));
+    memcpy(cookie.ip, ip, sizeof(ot_ip6));
+    cookie.interest = EVENT_IN;
+    cookie.flag     = STRUCT_HTTP_FLAG_ONSTACK;
+
+    generation = fdstate_open(sock, &cookie, time_future_seconds(OT_CLIENT_TIMEOUT));
+    if (!fdstate_claim((int)sock, generation)) {
+      fdstate_close(sock);
+      event_close((int)sock);
+      continue;
+    }
+
+    stats_issue_event(ws, EVENT_ACCEPT, FLAG_TCP, (uintptr_t)ip);
+
+    tcp_read(sock, ws);
+
+    /* Gone already if it was answered and closed inline */
+    if (fdstate_generation((int)sock) != generation)
+      continue;
+
+    /* Handling it may already have given it a heap home and handed it to
+       another thread, and then it is no longer this frame's to touch */
+    if (fdstate_cookie((int)sock) != &cookie)
+      fdstate_release((int)sock);
+    else if (!http_adopt(sock))
+      http_close(sock);
+    else
+      fdstate_release((int)sock);
+  }
+}
+
+/* One of these per worker, each allocated on its own so that growing the
+   array below never moves the one a running worker holds. */
+struct tcp_worker {
+  pthread_t   thread;
+  int64_t     sock;
+  _Atomic int left;
+};
+
+static struct tcp_worker **g_workers;
+static size_t              g_worker_count;
+
+/* A worker blocks in accept and nowhere else: the connection it is handed is
+   non blocking, so a client that stops talking is given to the event loop
+   rather than keeping this thread. */
+static void *tcp_worker(void *args) {
+  struct tcp_worker   *me = args;
+  struct ot_workstruct ws;
+
+  if (workstruct_init(&ws) == -1) {
+    fprintf(stderr, "Warning: a tcp worker could not get its buffers and will not run.\n");
+    atomic_store(&me->left, 1);
+    return NULL;
+  }
+
+  while (g_opentracker_running)
+    tcp_accept(me->sock, &ws);
+
+  workstruct_deinit(&ws);
+  atomic_store(&me->left, 1);
+  return NULL;
+}
+
+void tcp_worker_install(int64_t sock) {
+  struct tcp_worker **grown = realloc(g_workers, (g_worker_count + 1) * sizeof(*g_workers));
+  struct tcp_worker  *worker;
+
+  if (!grown || !(worker = calloc(1, sizeof(*worker)))) {
+    fprintf(stderr, "Warning: out of memory installing a tcp worker, this socket stays with the workers already running.\n");
+    g_workers = grown ? grown : g_workers;
+    return;
+  }
+  g_workers    = grown;
+  worker->sock = sock;
+
+  socket_set_blocking((int)sock, 1);
+  socket_defer_accept((int)sock);
+
+  if (pthread_create(&worker->thread, NULL, tcp_worker, worker))
+    free(worker);
+  else
+    g_workers[g_worker_count++] = worker;
+}
+
+void tcp_workers_deinit(void) {
+  size_t i;
+
+  for (i = 0; i < g_worker_count; ++i) {
+    struct tcp_worker *worker = g_workers[i];
+
+    /* Repeated because a signal only interrupts a call already under way */
+    while (!atomic_load(&worker->left)) {
+      pthread_kill(worker->thread, OT_SIGNAL_WAKE);
+      usleep(1000);
+    }
+    pthread_join(worker->thread, NULL);
+    free(worker);
+  }
+
+  free(g_workers);
+  g_workers      = NULL;
+  g_worker_count = 0;
+}
diff --git a/ot_tcp.h b/ot_tcp.h
new file mode 100644
index 0000000..96b6ec1
--- /dev/null
+++ b/ot_tcp.h
@@ -0,0 +1,28 @@
+/* This software was written by Dirk Engling <erdgeist@erdgeist.org>
+   It is considered beerware. Prost. Skol. Cheers or whatever.
+
+   $id$ */
+
+#ifndef OT_TCP_H__
+#define OT_TCP_H__
+
+#include <stdint.h>
+
+#include "trackerlogic.h"
+
+/* Take whatever connections are waiting on a listening socket and answer
+   them, each on this thread's stack, handing a connection that cannot be
+   finished to the event loop */
+void tcp_accept(const int64_t serversocket, struct ot_workstruct *ws);
+
+/* Handler connections in the event loop has become readable or writable */
+void tcp_read(const int64_t sock, struct ot_workstruct *ws);
+void tcp_write(const int64_t sock);
+
+/* One thread accepting on one socket. Called once per worker. */
+void tcp_worker_install(int64_t sock);
+
+/* Stop every worker installed above and wait for it to be gone */
+void tcp_workers_deinit(void);
+
+#endif
diff --git a/ot_udp.c b/ot_udp.c
index 1ec5148..00ec566 100644
--- a/ot_udp.c
+++ b/ot_udp.c
@@ -220,26 +220,17 @@ int handle_udp6(int64_t serversocket, struct ot_workstruct *ws) {
 static void *udp_worker(void *args) {
   struct udp_worker   *me = args;
   struct ot_workstruct ws;
-  memset(&ws, 0, sizeof(ws));
 
-  stats_counters_register(&ws);
-  ws.inbuf  = malloc(G_INBUF_SIZE);
-  ws.outbuf = malloc(G_OUTBUF_SIZE);
-#ifdef _DEBUG_HTTPERROR
-  ws.debugbuf = malloc(G_DEBUGBUF_SIZE);
-#endif
+  if (workstruct_init(&ws) == -1) {
+    fprintf(stderr, "Warning: a udp worker could not get its buffers and will not run.\n");
+    atomic_store(&me->left, 1);
+    return NULL;
+  }
 
   while (g_opentracker_running)
     handle_udp6(me->sock, &ws);
 
-  stats_counters_retire(&ws);
-
-  free(ws.inbuf);
-  free(ws.outbuf);
-#ifdef _DEBUG_HTTPERROR
-  free(ws.debugbuf);
-#endif
-
+  workstruct_deinit(&ws);
   atomic_store(&me->left, 1);
   return NULL;
 }
diff --git a/ot_workqueue.c b/ot_workqueue.c
new file mode 100644
index 0000000..aac366c
--- /dev/null
+++ b/ot_workqueue.c
@@ -0,0 +1,243 @@
+/* This software was written by Dirk Engling <erdgeist@erdgeist.org>
+   It is considered beerware. Prost. Skol. Cheers or whatever.
+
+   $id$ */
+
+/* System */
+#include <pthread.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/uio.h>
+#include <unistd.h>
+
+/* Opentracker */
+#include "ot_iovec.h"
+#include "ot_workqueue.h"
+#include "trackerlogic.h"
+
+/* Pushing a result wakes the thread that will send it */
+extern int g_self_pipe[2];
+
+struct ot_task {
+  ot_taskid       taskid;
+  ot_tasktype     tasktype;
+  int64_t         sock;
+  size_t          iovec_entries;
+  struct iovec   *iovec;
+  struct ot_task *next;
+};
+
+static ot_taskid       next_free_taskid = 1;
+static struct ot_task *tasklist;
+static pthread_mutex_t tasklist_mutex;
+static pthread_cond_t  tasklist_being_filled;
+
+int workqueue_pushtask(int64_t sock, ot_tasktype tasktype) {
+  struct ot_task **tmptask, *task;
+
+  task = malloc(sizeof(struct ot_task));
+  if (!task)
+    return -1;
+
+  task->taskid        = 0;
+  task->tasktype      = tasktype;
+  task->sock          = sock;
+  task->iovec_entries = 0;
+  task->iovec         = NULL;
+  task->next          = 0;
+
+  /* Want exclusive access to tasklist */
+  pthread_mutex_lock(&tasklist_mutex);
+
+  /* Skip to end of list */
+  tmptask = &tasklist;
+  while (*tmptask)
+    tmptask = &(*tmptask)->next;
+  *tmptask = task;
+
+  /* Inform waiting workers and release lock */
+  pthread_cond_broadcast(&tasklist_being_filled);
+  pthread_mutex_unlock(&tasklist_mutex);
+  return 0;
+}
+
+void workqueue_canceltask(int64_t sock) {
+  struct ot_task **task;
+
+  /* Want exclusive access to tasklist */
+  pthread_mutex_lock(&tasklist_mutex);
+
+  for (task = &tasklist; *task; task = &((*task)->next))
+    if ((*task)->sock == sock) {
+      struct iovec   *iovec = (*task)->iovec;
+      struct ot_task *ptask = *task;
+      size_t                   i;
+
+      /* Free task's iovec, the buffers it points at and the vector itself */
+      for (i = 0; i < (*task)->iovec_entries; ++i)
+        free(iovec[i].iov_base);
+      free(iovec);
+
+      *task = (*task)->next;
+      free(ptask);
+      break;
+    }
+
+  /* Release lock */
+  pthread_mutex_unlock(&tasklist_mutex);
+}
+
+ot_taskid workqueue_poptask(ot_tasktype *tasktype) {
+  struct ot_task *task;
+  ot_taskid       taskid = 0;
+
+  /* Want exclusive access to tasklist */
+  pthread_mutex_lock(&tasklist_mutex);
+
+  while (!taskid) {
+    /* Skip to the first unassigned task this worker wants to do */
+    for (task = tasklist; task; task = task->next)
+      if (!task->taskid && (TASK_CLASS_MASK & task->tasktype) == *tasktype) {
+        /* If we found an outstanding task, assign a taskid to it
+           and leave the loop */
+        task->taskid = taskid = ++next_free_taskid;
+        *tasktype             = task->tasktype;
+        break;
+      }
+
+    /* Wait until the next task is being fed, unless the tracker is on its
+       way out, in which case the worker is meant to leave rather than wait */
+    if (!taskid) {
+      if (!g_opentracker_running)
+        break;
+      pthread_cond_wait(&tasklist_being_filled, &tasklist_mutex);
+    }
+  }
+
+  /* Release lock */
+  pthread_mutex_unlock(&tasklist_mutex);
+
+  return taskid;
+}
+
+void workqueue_pushsuccess(ot_taskid taskid) {
+  struct ot_task **task;
+
+  /* Want exclusive access to tasklist */
+  pthread_mutex_lock(&tasklist_mutex);
+
+  for (task = &tasklist; *task; task = &((*task)->next))
+    if ((*task)->taskid == taskid) {
+      struct ot_task *ptask = *task;
+      *task = (*task)->next;
+      free(ptask);
+      break;
+    }
+
+  /* Release lock */
+  pthread_mutex_unlock(&tasklist_mutex);
+}
+
+int workqueue_pushresult(ot_taskid taskid, size_t iovec_entries, struct iovec *iovec) {
+  struct ot_task *task;
+  const char      byte = 'o';
+
+  /* Want exclusive access to tasklist */
+  pthread_mutex_lock(&tasklist_mutex);
+
+  for (task = tasklist; task; task = task->next)
+    if (task->taskid == taskid) {
+      task->iovec_entries = iovec_entries;
+      task->iovec         = iovec;
+      task->tasktype      = TASK_DONE;
+      break;
+    }
+
+  /* Release lock */
+  pthread_mutex_unlock(&tasklist_mutex);
+
+  /* A full pipe has already done this nudge's job, so a failure is fine */
+  (void)!write(g_self_pipe[1], &byte, 1);
+
+  /* Indicate whether the worker has to throw away results */
+  return task ? 0 : -1;
+}
+
+int workqueue_pushchunked(ot_taskid taskid, struct iovec *iovec) {
+  struct ot_task *task;
+  const char      byte = 'o';
+
+  /* Want exclusive access to tasklist */
+  pthread_mutex_lock(&tasklist_mutex);
+
+  for (task = tasklist; task; task = task->next)
+    if (task->taskid == taskid) {
+      if (iovec) {
+        if (iovec_append(&task->iovec_entries, &task->iovec, iovec))
+          task->tasktype = TASK_DONE_PARTIAL;
+        else
+          task = NULL;
+      } else
+        task->tasktype = TASK_DONE;
+      break;
+    }
+
+  /* Release lock */
+  pthread_mutex_unlock(&tasklist_mutex);
+
+  /* A full pipe has already done this nudge's job, so a failure is fine */
+  (void)!write(g_self_pipe[1], &byte, 1);
+
+  /* Indicate whether the worker has to throw away results */
+  return task ? 0 : -1;
+}
+
+int64_t workqueue_popresult(size_t *iovec_entries, struct iovec **iovec, int *is_partial) {
+  struct ot_task **task;
+  int64_t sock = -1;
+
+  *is_partial = 0;
+
+  /* Want exclusive access to tasklist */
+  pthread_mutex_lock(&tasklist_mutex);
+
+  for (task = &tasklist; *task; task = &((*task)->next))
+    if (((*task)->tasktype & TASK_CLASS_MASK) == TASK_DONE) {
+      struct ot_task *ptask = *task;
+      *iovec_entries        = ptask->iovec_entries;
+      *iovec                = ptask->iovec;
+      sock                  = ptask->sock;
+
+      if ((*task)->tasktype == TASK_DONE) {
+        *task = ptask->next;
+        free(ptask);
+      } else {
+        ptask->iovec_entries = 0;
+        ptask->iovec         = NULL;
+        *is_partial          = 1;
+        /* Prevent task from showing up immediately again unless new data was added */
+        (*task)->tasktype    = TASK_FULLSCRAPE;
+      }
+      break;
+    }
+
+  /* Release lock */
+  pthread_mutex_unlock(&tasklist_mutex);
+  return sock;
+}
+
+void workqueue_wakeall(void) {
+  pthread_mutex_lock(&tasklist_mutex);
+  pthread_cond_broadcast(&tasklist_being_filled);
+  pthread_mutex_unlock(&tasklist_mutex);
+}
+
+void workqueue_init(void) {
+  pthread_mutex_init(&tasklist_mutex, NULL);
+  pthread_cond_init(&tasklist_being_filled, NULL);
+}
+
+void workqueue_deinit(void) {
+  pthread_mutex_destroy(&tasklist_mutex);
+  pthread_cond_destroy(&tasklist_being_filled);
+}
diff --git a/ot_workqueue.h b/ot_workqueue.h
new file mode 100644
index 0000000..f99a7b5
--- /dev/null
+++ b/ot_workqueue.h
@@ -0,0 +1,78 @@
+/* This software was written by Dirk Engling <erdgeist@erdgeist.org>
+   It is considered beerware. Prost. Skol. Cheers or whatever.
+
+   $id$ */
+
+#ifndef OT_WORKQUEUE_H__
+#define OT_WORKQUEUE_H__
+
+#include <stdint.h>
+#include <sys/uio.h>
+
+#include "trackerlogic.h"
+
+/* Work the tracker hands to a thread of its own: a full scrape or an
+   expensive stats page, which take too long to build while a client waits.
+   The task goes in here, the reply comes back out. */
+
+typedef enum {
+  TASK_STATS_CONNS               = 0x0001,
+  TASK_STATS_TCP                 = 0x0002,
+  TASK_STATS_UDP                 = 0x0003,
+  TASK_STATS_SCRAPE              = 0x0004,
+  TASK_STATS_FULLSCRAPE          = 0x0005,
+  TASK_STATS_TPB                 = 0x0006,
+  TASK_STATS_HTTPERRORS          = 0x0007,
+  TASK_STATS_VERSION             = 0x0008,
+  TASK_STATS_RENEW               = 0x000a,
+  TASK_STATS_SYNCS               = 0x000b,
+  TASK_STATS_COMPLETED           = 0x000c,
+  TASK_STATS_NUMWANTS            = 0x000d,
+
+  TASK_STATS                     = 0x0100, /* Mask */
+  TASK_STATS_TORRENTS            = 0x0101,
+  TASK_STATS_PEERS               = 0x0102,
+  TASK_STATS_SLASH24S            = 0x0103,
+  TASK_STATS_TOP10               = 0x0104,
+  TASK_STATS_TOP100              = 0x0105,
+  TASK_STATS_EVERYTHING          = 0x0106,
+  TASK_STATS_FULLLOG             = 0x0107,
+  TASK_STATS_WOODPECKERS         = 0x0108,
+
+  TASK_FULLSCRAPE                = 0x0200, /* Default mode */
+  TASK_FULLSCRAPE_TPB_BINARY     = 0x0201,
+  TASK_FULLSCRAPE_TPB_ASCII      = 0x0202,
+  TASK_FULLSCRAPE_TPB_ASCII_PLUS = 0x0203,
+  TASK_FULLSCRAPE_TPB_URLENCODED = 0x0204,
+  TASK_FULLSCRAPE_TRACKERSTATE   = 0x0205,
+
+  TASK_DMEM                      = 0x0300,
+
+  TASK_DONE                      = 0x0f00,
+  TASK_DONE_PARTIAL              = 0x0f01,
+
+  TASK_FLAG_GZIP                 = 0x1000,
+  TASK_FLAG_BZIP2                = 0x2000,
+  TASK_FLAG_ZSTD                 = 0x4000,
+  TASK_FLAG_CHUNKED              = 0x8000,
+
+  TASK_TASK_MASK                 = 0x0fff,
+  TASK_CLASS_MASK                = 0x0f00,
+  TASK_FLAGS_MASK                = 0xf000
+} ot_tasktype;
+
+typedef unsigned long ot_taskid;
+
+int        workqueue_pushtask(int64_t sock, ot_tasktype tasktype);
+void       workqueue_canceltask(int64_t sock);
+void       workqueue_pushsuccess(ot_taskid taskid);
+ot_taskid  workqueue_poptask(ot_tasktype *tasktype);
+int        workqueue_pushresult(ot_taskid taskid, size_t iovec_entries, struct iovec *iovector);
+int        workqueue_pushchunked(ot_taskid taskid, struct iovec *iovec);
+int64_t    workqueue_popresult(size_t *iovec_entries, struct iovec **iovector, int *is_partial);
+
+void       workqueue_init(void);
+void       workqueue_deinit(void);
+void       workqueue_wakeall(void);
+
+#endif
diff --git a/tests/integration/test_tcpworkers.py b/tests/integration/test_tcpworkers.py
new file mode 100644
index 0000000..e43612a
--- /dev/null
+++ b/tests/integration/test_tcpworkers.py
@@ -0,0 +1,67 @@
+"""The tracker with its tcp side handed to worker threads.
+
+With workers configured, a connection is accepted and answered on a worker's
+own stack. Anything it cannot finish there is copied to the heap and handed
+to the event loop, which is the one place where a worker and the main thread
+touch the same structures, so these drive all three outcomes: answered
+inline, held open, and parked waiting for another thread entirely.
+"""
+
+import threading
+import unittest
+import urllib.parse
+
+from harness import TrackerTestCase, bdecode
+
+WORKERS = 4
+
+
+class TcpWorkers(TrackerTestCase):
+    config = "listen.tcp.workers %d\n" % WORKERS
+
+    def announce_path(self, name, port=6881):
+        return "/announce?info_hash=%s&peer_id=%s&port=%d&left=0" % (
+            urllib.parse.quote_from_bytes(self.hash_for(name), safe=""),
+            "x" * 20,
+            port,
+        )
+
+    def test_many_at_once_are_all_answered(self):
+        """Every worker gets something to do, and the swarm adds up."""
+        info_hash = self.hash_for("workers-swarm")
+        errors = []
+
+        def announce(index):
+            try:
+                self.tracker.announce(info_hash, peer_id=b"-OT0001-%012d" % index, port=7000 + index, left=0)
+            except Exception as problem:  # noqa: BLE001 - reported, not swallowed
+                errors.append("%d: %r" % (index, problem))
+
+        threads = [threading.Thread(target=announce, args=(i,)) for i in range(32)]
+        for thread in threads:
+            thread.start()
+        for thread in threads:
+            thread.join()
+
+        self.assertEqual(errors, [], "\n".join(errors))
+        self.assertEqual(self.tracker.scrape(info_hash)[b"files"][info_hash][b"complete"], 32)
+
+    def test_a_kept_connection_outlives_the_worker_that_took_it(self):
+        """Keep-alive is the plain case of a connection a worker hands on."""
+        connection = self.tracker.connect()
+        try:
+            for i in range(4):
+                connection.send(self.announce_path("workers-kept"))
+                status, _, body = connection.reply()
+                self.assertIn(b"200", status)
+                self.assertIn(b"complete", bdecode(body))
+        finally:
+            connection.close()
+
+    def test_a_full_scrape_asked_for_by_a_worker_still_arrives(self):
+        """The reply is built by another thread, so the worker parks the
+           connection waiting for nothing at all before handing it over."""
+        for i in range(20):
+            self.tracker.announce(bytes([i]) + b"\0" * 19, port=7100 + i, left=0)
+        body = self.tracker.get("/scrape")
+        self.assertEqual(len(bdecode(body)[b"files"]), 20)
diff --git a/tests/runner.c b/tests/runner.c
index 71a5e2a..38bfb86 100644
--- a/tests/runner.c
+++ b/tests/runner.c
@@ -27,7 +27,7 @@ static const struct module modules[] = {{"ot_urlencoded", test_urlencoded},
                                         {"ot_buf", test_buf},
                                         {"ot_ip", test_ip},
                                         {"ot_socket", test_socket},
-                                        {"ot_mutex", test_mutex},
+                                        {"ot_workqueue", test_workqueue},
                                         {"ot_batch", test_batch},
                                         {"ot_fdstate", test_fdstate},
                                         {"ot_event", test_event},
diff --git a/tests/t_mutex.c b/tests/t_workqueue.c
index 3307d70..13e0b2e 100644
--- a/tests/t_mutex.c
+++ b/tests/t_workqueue.c
@@ -8,7 +8,7 @@
 #include <string.h>
 
 /* Opentracker */
-#include "ot_mutex.h"
+#include "ot_workqueue.h"
 #include "test.h"
 
 /* A worker asks for a class of task and blocks until one turns up, so a task
@@ -20,25 +20,25 @@ static void case_task_goes_to_its_own_class(void) {
   ot_taskid   taskid;
 
   test_case("a worker is handed the task of the class it asked for");
-  CHECK_INT(mutex_workqueue_pushtask(41, TASK_FULLSCRAPE_TPB_ASCII), 0);
+  CHECK_INT(workqueue_pushtask(41, TASK_FULLSCRAPE_TPB_ASCII), 0);
   asked  = TASK_FULLSCRAPE;
-  taskid = mutex_workqueue_poptask(&asked);
+  taskid = workqueue_poptask(&asked);
   CHECK(taskid != 0);
 
   test_case("and the full task type, not just its class");
   CHECK_INT(asked, TASK_FULLSCRAPE_TPB_ASCII);
-  mutex_workqueue_canceltask(41);
+  workqueue_canceltask(41);
 }
 
 static void case_flags_survive_the_queue(void) {
   ot_tasktype asked;
 
   test_case("flags outside the class mask are carried through");
-  CHECK_INT(mutex_workqueue_pushtask(42, TASK_FULLSCRAPE_TPB_URLENCODED | TASK_FLAG_GZIP), 0);
+  CHECK_INT(workqueue_pushtask(42, TASK_FULLSCRAPE_TPB_URLENCODED | TASK_FLAG_GZIP), 0);
   asked = TASK_FULLSCRAPE;
-  CHECK(mutex_workqueue_poptask(&asked) != 0);
+  CHECK(workqueue_poptask(&asked) != 0);
   CHECK_INT(asked, TASK_FULLSCRAPE_TPB_URLENCODED | TASK_FLAG_GZIP);
-  mutex_workqueue_canceltask(42);
+  workqueue_canceltask(42);
 }
 
 static void case_classes_do_not_cross(void) {
@@ -46,22 +46,22 @@ static void case_classes_do_not_cross(void) {
   ot_taskid   taskid;
 
   test_case("a stats worker passes over a fullscrape task");
-  CHECK_INT(mutex_workqueue_pushtask(43, TASK_FULLSCRAPE_TPB_BINARY), 0);
-  CHECK_INT(mutex_workqueue_pushtask(44, TASK_STATS_PEERS), 0);
+  CHECK_INT(workqueue_pushtask(43, TASK_FULLSCRAPE_TPB_BINARY), 0);
+  CHECK_INT(workqueue_pushtask(44, TASK_STATS_PEERS), 0);
 
   asked  = TASK_STATS;
-  taskid = mutex_workqueue_poptask(&asked);
+  taskid = workqueue_poptask(&asked);
   CHECK(taskid != 0);
   CHECK_INT(asked, TASK_STATS_PEERS);
 
   test_case("and leaves it for the one that wants it");
   asked  = TASK_FULLSCRAPE;
-  taskid = mutex_workqueue_poptask(&asked);
+  taskid = workqueue_poptask(&asked);
   CHECK(taskid != 0);
   CHECK_INT(asked, TASK_FULLSCRAPE_TPB_BINARY);
 
-  mutex_workqueue_canceltask(43);
-  mutex_workqueue_canceltask(44);
+  workqueue_canceltask(43);
+  workqueue_canceltask(44);
 }
 
 static void case_every_task_gets_its_own_id(void) {
@@ -69,36 +69,36 @@ static void case_every_task_gets_its_own_id(void) {
   ot_taskid   first, second;
 
   test_case("two queued tasks are handed out under different ids");
-  CHECK_INT(mutex_workqueue_pushtask(45, TASK_FULLSCRAPE), 0);
-  CHECK_INT(mutex_workqueue_pushtask(46, TASK_FULLSCRAPE), 0);
+  CHECK_INT(workqueue_pushtask(45, TASK_FULLSCRAPE), 0);
+  CHECK_INT(workqueue_pushtask(46, TASK_FULLSCRAPE), 0);
 
   asked  = TASK_FULLSCRAPE;
-  first  = mutex_workqueue_poptask(&asked);
+  first  = workqueue_poptask(&asked);
   asked  = TASK_FULLSCRAPE;
-  second = mutex_workqueue_poptask(&asked);
+  second = workqueue_poptask(&asked);
   CHECK(first != 0);
   CHECK(second != 0);
   CHECK(first != second);
 
-  mutex_workqueue_canceltask(45);
-  mutex_workqueue_canceltask(46);
+  workqueue_canceltask(45);
+  workqueue_canceltask(46);
 }
 
 static void case_cancelled_task_is_not_handed_out(void) {
   ot_tasktype asked;
 
   test_case("a cancelled task is gone before any worker sees it");
-  CHECK_INT(mutex_workqueue_pushtask(47, TASK_FULLSCRAPE_TPB_ASCII_PLUS), 0);
-  CHECK_INT(mutex_workqueue_pushtask(48, TASK_FULLSCRAPE), 0);
-  mutex_workqueue_canceltask(47);
+  CHECK_INT(workqueue_pushtask(47, TASK_FULLSCRAPE_TPB_ASCII_PLUS), 0);
+  CHECK_INT(workqueue_pushtask(48, TASK_FULLSCRAPE), 0);
+  workqueue_canceltask(47);
 
   asked = TASK_FULLSCRAPE;
-  CHECK(mutex_workqueue_poptask(&asked) != 0);
+  CHECK(workqueue_poptask(&asked) != 0);
   CHECK_INT(asked, TASK_FULLSCRAPE);
-  mutex_workqueue_canceltask(48);
+  workqueue_canceltask(48);
 }
 
-void test_mutex(void) {
+void test_workqueue(void) {
   case_task_goes_to_its_own_class();
   case_flags_survive_the_queue();
   case_classes_do_not_cross();
diff --git a/tests/test.h b/tests/test.h
index 0eddc82..d58742c 100644
--- a/tests/test.h
+++ b/tests/test.h
@@ -86,7 +86,7 @@ void test_scan(void);
 void test_buf(void);
 void test_ip(void);
 void test_socket(void);
-void test_mutex(void);
+void test_workqueue(void);
 void test_batch(void);
 void test_fdstate(void);
 void test_event(void);
diff --git a/trackerlogic.c b/trackerlogic.c
index 477c4a6..b0b7f4e 100644
--- a/trackerlogic.c
+++ b/trackerlogic.c
@@ -20,6 +20,7 @@
 #include "ot_ip.h"
 #include "ot_livesync.h"
 #include "ot_mutex.h"
+#include "ot_workqueue.h"
 #include "ot_random.h"
 #include "ot_stats.h"
 #include "ot_udp.h"
@@ -560,6 +561,7 @@ void trackerlogic_init(void) {
 
   /* Initialise background worker threads */
   mutex_init();
+  workqueue_init();
   clean_init();
   fullscrape_init();
   accesslist_init();
@@ -567,6 +569,35 @@ void trackerlogic_init(void) {
   stats_init();
 }
 
+int workstruct_init(struct ot_workstruct *ws) {
+  memset(ws, 0, sizeof(*ws));
+  ws->inbuf  = malloc(G_INBUF_SIZE);
+  ws->outbuf = malloc(G_OUTBUF_SIZE);
+#ifdef _DEBUG_HTTPERROR
+  ws->debugbuf = malloc(G_DEBUGBUF_SIZE);
+#endif
+  if (!ws->inbuf || !ws->outbuf)
+    return -1;
+
+  /* Picking which peers to hand out starts here, so two workers seeded alike
+     would hand out alike */
+  random_bytes(ws->rand48_state, sizeof(ws->rand48_state));
+
+  stats_counters_register(ws);
+  return 0;
+}
+
+void workstruct_deinit(struct ot_workstruct *ws) {
+  /* The tally goes to the global before this struct goes out of scope */
+  stats_counters_retire(ws);
+
+  free(ws->inbuf);
+  free(ws->outbuf);
+#ifdef _DEBUG_HTTPERROR
+  free(ws->debugbuf);
+#endif
+}
+
 void trackerlogic_deinit(void) {
   int    delta_torrentcount = 0;
   size_t bucket, j;
@@ -594,5 +625,6 @@ void trackerlogic_deinit(void) {
   }
 
   /* Release mutexes */
+  workqueue_deinit();
   mutex_deinit();
 }
diff --git a/trackerlogic.h b/trackerlogic.h
index d269267..53edabd 100644
--- a/trackerlogic.h
+++ b/trackerlogic.h
@@ -236,6 +236,12 @@ struct ot_workstruct {
   int      keep_alive;
 };
 
+/* Everything a thread that serves requests needs for itself. Set up and torn
+   down in one place so that a new kind of worker cannot miss a piece.
+   Returns -1 if the buffers could not be had. */
+int  workstruct_init(struct ot_workstruct *ws);
+void workstruct_deinit(struct ot_workstruct *ws);
+
 /*
    Exported functions
 */