Unify the peerlists for v4 and v6

This commit finishes what started with unifying v4 and v6 trackers, from then on, torrents
carried a peer list per family, with obvious redundancies. With the race to save memory,
these became burdening enough that now these peerlist pointers have been moved into a
single struct, holding both v4 and v6 peers in their respective ot_vectors, so the per-
torrent cost shrunk by another pointer's size.

Each peerlist also lost its peer count cache, as in most cases, it's trivial to load it
directly from the ot_vector's entry count. The seed counts that were kept per family only
to be aggregated later for every consumer, are also merged now.

Total savings depend on the tracker's actual load. In some corner cases, where only one
address family dominates the torrents, we actually use a few bytes more in the ot_vector
for the absent family, but even that mostly becomes noise: before it actually adds to the
memory load, the peer payload will be the deciding factor.

Most wins are on what often is experienced on trackers: A long tail of abandoned torrents
now needs eight fewer bytes, slipping into a smaller allocation unit on glibc's allocator.

2 files changed, 78 insertions, 31 deletions

diff --git a/tests/t_clean.c b/tests/t_clean.c
index 0237cc6..ce17bc3 100644
--- a/tests/t_clean.c
+++ b/tests/t_clean.c
@@ -33,6 +33,20 @@ static void announce(struct ot_workstruct *ws, ot_hash *hash, uint8_t last_octet
   add_peer_to_torrent_and_return_peers(FLAG_TCP, ws, 0);
 }
 
+/* A distinct v4 peer addressed by two octets, to build swarms past 256 */
+static size_t announce_wide(struct ot_workstruct *ws, ot_hash *hash, unsigned int which, size_t amount) {
+  static const uint8_t v4mapped[12] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff};
+
+  ws->reply = ws->outbuf;
+  memset(&ws->peer, 0, sizeof(ws->peer));
+  memcpy(&ws->peer, v4mapped, sizeof(v4mapped));
+  ((uint8_t *)&ws->peer)[14] = (uint8_t)(which >> 8);
+  ((uint8_t *)&ws->peer)[15] = (uint8_t)which;
+  OT_SETPORT(&ws->peer, &(uint16_t){6881});
+  ws->hash = hash;
+  return add_peer_to_torrent_and_return_peers(FLAG_TCP, ws, amount);
+}
+
 static size_t peers_in(ot_hash *hash) {
   char   reply[1024];
   size_t len = return_udp_scrape_for_torrent(*hash, reply);
@@ -157,6 +171,39 @@ static void case_empty_torrent_timeout_bounds_a_counter_only_torrent(void) {
   workstruct_deinit(&ws);
 }
 
+static void case_a_bucketed_swarm_survives_clean_and_serves(void) {
+  struct ot_workstruct ws;
+  ot_hash              hash;
+  ot_torrent          *torrent;
+  unsigned int         i;
+  const unsigned int   n = 600; /* past OT_PEER_BUCKET_MINCOUNT (512) */
+
+  test_case("a swarm large enough to be bucketed cleans, splits and serves without loss");
+  memset(hash, 0x34, sizeof(hash));
+  CHECK_INT(workstruct_init(&ws, G_INBUF_SIZE), 0);
+
+  for (i = 0; i < n; ++i)
+    announce_wide(&ws, &hash, i, 0);
+  CHECK_INT(peers_in(&hash), n);
+
+  torrent = torrent_of(&hash);
+  CHECK(torrent != NULL);
+  CHECK(torrent->peer_list != NULL);
+
+  /* A clean with a minute elapsed splits the flat vector into buckets */
+  step_clock_back(torrent, -1);
+  CHECK_INT(clean_single_torrent(torrent), 0);
+  CHECK(torrent->peer_list != NULL);
+  CHECK_INT(OT_VECTOR_HASBUCKETS(&torrent->peer_list->peers4), 1);
+  CHECK_INT(peers_in(&hash), n);
+
+  /* Serve a selection back, which walks the bucket list */
+  CHECK(announce_wide(&ws, &hash, 1, 200) > 0);
+  CHECK_INT(peers_in(&hash), n);
+
+  workstruct_deinit(&ws);
+}
+
 void test_clean(void) {
   sigset_t hup;
 
@@ -170,5 +217,6 @@ void test_clean(void) {
   case_a_clock_that_went_back_keeps_the_swarm();
   case_an_idle_swarm_still_ages_out();
   case_empty_torrent_timeout_bounds_a_counter_only_torrent();
+  case_a_bucketed_swarm_survives_clean_and_serves();
   trackerlogic_deinit();
 }
diff --git a/tests/t_vector.c b/tests/t_vector.c
index f0a9bf7..42ce31a 100644
--- a/tests/t_vector.c
+++ b/tests/t_vector.c
@@ -200,17 +200,16 @@ static void case_insert_compare_prefix(void) {
    into the first address byte, so the peers are distinct but arrive in no
    particular order: a list generated in ascending order would come back
    sorted whether or not vector_redistribute_buckets sorted it. */
-static void build_flat_peerlist(ot_peerlist *peer_list, size_t count) {
+static void build_flat_vector(ot_vector *peers, size_t count) {
   size_t i;
 
-  memset(peer_list, 0, sizeof(*peer_list));
-  peer_list->peers.data  = malloc(count * OT_PEER_SIZE4);
-  peer_list->peers.size  = count;
-  peer_list->peers.space = count;
-  peer_list->peer_count  = count;
+  memset(peers, 0, sizeof(*peers));
+  peers->data  = malloc(count * OT_PEER_SIZE4);
+  peers->size  = count;
+  peers->space = count;
 
   for (i = 0; i < count; ++i) {
-    uint8_t *peer = (uint8_t *)peer_list->peers.data + i * OT_PEER_SIZE4;
+    uint8_t *peer = (uint8_t *)peers->data + i * OT_PEER_SIZE4;
 
     memset(peer, 0, OT_PEER_SIZE4);
     peer[0] = (uint8_t)i;
@@ -222,19 +221,19 @@ static void build_flat_peerlist(ot_peerlist *peer_list, size_t count) {
 
 static void case_redistribute_holds_every_peer(void) {
   const size_t count = 513;
-  ot_peerlist  peer_list;
+  ot_vector    peers;
   size_t       bucket, total_size = 0, empty = 0, largest = 0;
   ot_vector   *buckets;
 
   test_case("a peer list past the threshold is split into buckets");
-  build_flat_peerlist(&peer_list, count);
-  CHECK(peer_list.peers.data != NULL);
-  vector_redistribute_buckets(&peer_list, OT_PEER_SIZE4);
-  CHECK_INT(OT_PEERLIST_HASBUCKETS(&peer_list), 1);
-  CHECK_INT(peer_list.peers.size, 16);
+  build_flat_vector(&peers, count);
+  CHECK(peers.data != NULL);
+  vector_redistribute_buckets(&peers, OT_PEER_SIZE4, count);
+  CHECK_INT(OT_VECTOR_HASBUCKETS(&peers), 1);
+  CHECK_INT(peers.size, 16);
 
-  buckets = peer_list.peers.data;
-  for (bucket = 0; bucket < peer_list.peers.size; ++bucket) {
+  buckets = peers.data;
+  for (bucket = 0; bucket < peers.size; ++bucket) {
     total_size += buckets[bucket].size;
     if (!buckets[bucket].size)
       ++empty;
@@ -252,23 +251,23 @@ static void case_redistribute_holds_every_peer(void) {
   CHECK_INT(empty, 0);
   CHECK(largest < count / 2);
 
-  vector_clean_list(peer_list.peers.data, peer_list.peers.size);
+  vector_clean_list(peers.data, peers.size);
 }
 
 static void case_redistribute_sorts_each_bucket(void) {
   const size_t count = 1000;
-  ot_peerlist  peer_list;
+  ot_vector    peers;
   size_t       bucket, i, out_of_order = 0;
   ot_vector   *buckets;
 
   test_case("every bucket comes back sorted, so bsearch can be used on it");
-  build_flat_peerlist(&peer_list, count);
-  CHECK(peer_list.peers.data != NULL);
-  vector_redistribute_buckets(&peer_list, OT_PEER_SIZE4);
-  CHECK_INT(OT_PEERLIST_HASBUCKETS(&peer_list), 1);
+  build_flat_vector(&peers, count);
+  CHECK(peers.data != NULL);
+  vector_redistribute_buckets(&peers, OT_PEER_SIZE4, count);
+  CHECK_INT(OT_VECTOR_HASBUCKETS(&peers), 1);
 
-  buckets = peer_list.peers.data;
-  for (bucket = 0; bucket < peer_list.peers.size; ++bucket)
+  buckets = peers.data;
+  for (bucket = 0; bucket < peers.size; ++bucket)
     for (i = 1; i < buckets[bucket].size; ++i) {
       const uint8_t *prev = (uint8_t *)buckets[bucket].data + (i - 1) * OT_PEER_SIZE4;
       const uint8_t *next = (uint8_t *)buckets[bucket].data + i * OT_PEER_SIZE4;
@@ -278,19 +277,19 @@ static void case_redistribute_sorts_each_bucket(void) {
     }
   CHECK_INT(out_of_order, 0);
 
-  vector_clean_list(peer_list.peers.data, peer_list.peers.size);
+  vector_clean_list(peers.data, peers.size);
 }
 
 static void case_redistribute_below_threshold(void) {
-  ot_peerlist peer_list;
+  ot_vector peers;
 
   test_case("a small peer list is left flat");
-  build_flat_peerlist(&peer_list, 100);
-  CHECK(peer_list.peers.data != NULL);
-  vector_redistribute_buckets(&peer_list, OT_PEER_SIZE4);
-  CHECK_INT(OT_PEERLIST_HASBUCKETS(&peer_list), 0);
-  CHECK_INT(peer_list.peers.size, 100);
-  free(peer_list.peers.data);
+  build_flat_vector(&peers, 100);
+  CHECK(peers.data != NULL);
+  vector_redistribute_buckets(&peers, OT_PEER_SIZE4, 100);
+  CHECK_INT(OT_VECTOR_HASBUCKETS(&peers), 0);
+  CHECK_INT(peers.size, 100);
+  free(peers.data);
 }
 
 void test_vector(void) {