summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Mk/tests.mk19
-rw-r--r--man5/opentracker.conf.513
-rw-r--r--opentracker.conf.sample20
-rw-r--r--src/opentracker.c2
-rw-r--r--src/ot_config.c6
-rw-r--r--src/ot_livesync.c46
-rw-r--r--src/ot_livesync.h7
-rw-r--r--src/ot_worker.c2
-rw-r--r--src/trackerlogic.c6
-rw-r--r--src/trackerlogic.h2
-rw-r--r--tests/integration/harness.py34
-rw-r--r--tests/integration/test_livesync.py208
-rw-r--r--tests/sync_daemon.pl (renamed from sync_daemon.pl)0
-rw-r--r--tests/t_trackerlogic.c2
14 files changed, 341 insertions, 26 deletions
diff --git a/Mk/tests.mk b/Mk/tests.mk
index 46f308b..2efdd72 100644
--- a/Mk/tests.mk
+++ b/Mk/tests.mk
@@ -29,7 +29,13 @@ LDFLAGS_integration=$(LINK) -fsanitize=address,undefined
29CFLAGS_tsan=$(OPTS_debug) $(CPPFLAGS) $(OT_CFLAGS) $(FEATURES_integration) -fsanitize=thread -fno-omit-frame-pointer 29CFLAGS_tsan=$(OPTS_debug) $(CPPFLAGS) $(OT_CFLAGS) $(FEATURES_integration) -fsanitize=thread -fno-omit-frame-pointer
30LDFLAGS_tsan=$(LINK) -fsanitize=thread 30LDFLAGS_tsan=$(LINK) -fsanitize=thread
31 31
32BINARIES+=$(TESTBIN) $(TESTBIN).poll $(TESTBIN).sanitize opentracker.integration opentracker.tsan 32# Live sync is a compile time feature, so reaching it from a black box test
33# needs a binary of its own. tests/integration/test_livesync.py skips itself
34# against a build without it, which is what -v is for.
35FEATURES_livesync=$(FEATURES_integration) -DWANT_SYNC_LIVE
36CFLAGS_livesync=$(OPTS_debug) $(CPPFLAGS) $(OT_CFLAGS) $(FEATURES_livesync) -fsanitize=address,undefined -fno-omit-frame-pointer
37
38BINARIES+=$(TESTBIN) $(TESTBIN).poll $(TESTBIN).sanitize opentracker.integration opentracker.tsan opentracker.livesync
33 39
34$(TESTBIN): $(TEST_SOURCES) $(TEST_UNITS) $(TEST_HEADERS) $(BUILDDEPS) 40$(TESTBIN): $(TEST_SOURCES) $(TEST_UNITS) $(TEST_HEADERS) $(BUILDDEPS)
35 $(CC) -o $@ $(CFLAGS_test) $(TEST_SOURCES) $(TEST_UNITS) 41 $(CC) -o $@ $(CFLAGS_test) $(TEST_SOURCES) $(TEST_UNITS)
@@ -46,6 +52,9 @@ opentracker.integration: $(SOURCES) $(HEADERS) $(BUILDDEPS)
46opentracker.tsan: $(SOURCES) $(HEADERS) $(BUILDDEPS) 52opentracker.tsan: $(SOURCES) $(HEADERS) $(BUILDDEPS)
47 $(CC) -o $@ $(CFLAGS_tsan) $(SOURCES) $(LDFLAGS_tsan) 53 $(CC) -o $@ $(CFLAGS_tsan) $(SOURCES) $(LDFLAGS_tsan)
48 54
55opentracker.livesync: $(SOURCES) $(HEADERS) $(BUILDDEPS)
56 $(CC) -o $@ $(CFLAGS_livesync) $(SOURCES) $(LDFLAGS_integration)
57
49check: $(TESTBIN) $(BINARY) 58check: $(TESTBIN) $(BINARY)
50 ./$(TESTBIN) 59 ./$(TESTBIN)
51 60
@@ -61,12 +70,18 @@ check-integration: opentracker.integration
61check-tsan: opentracker.tsan 70check-tsan: opentracker.tsan
62 OPENTRACKER=./opentracker.tsan $(PYTHON) -m unittest discover -s tests/integration 71 OPENTRACKER=./opentracker.tsan $(PYTHON) -m unittest discover -s tests/integration
63 72
73# The whole suite against a build that has live sync, which is also the only
74# way the live sync tests stop skipping.
75check-livesync: opentracker.livesync
76 OPENTRACKER=./opentracker.livesync $(PYTHON) -m unittest discover -s tests/integration
77
64# Cheapest first, so a quick failure comes back quickly. 78# Cheapest first, so a quick failure comes back quickly.
65check-all: 79check-all:
66 $(MAKE) check 80 $(MAKE) check
67 $(MAKE) check-poll 81 $(MAKE) check-poll
68 $(MAKE) check-sanitize 82 $(MAKE) check-sanitize
69 $(MAKE) check-integration 83 $(MAKE) check-integration
84 $(MAKE) check-livesync
70 $(MAKE) check-tsan 85 $(MAKE) check-tsan
71 86
72.PHONY: check check-poll check-sanitize check-integration check-tsan check-all 87.PHONY: check check-poll check-sanitize check-integration check-livesync check-tsan check-all
diff --git a/man5/opentracker.conf.5 b/man5/opentracker.conf.5
index 67e624f..9a836ae 100644
--- a/man5/opentracker.conf.5
+++ b/man5/opentracker.conf.5
@@ -79,7 +79,18 @@ Specifies the path to the stats location. You can configure opentracker to appea
79Specifies the IP address or network of the reverse proxies. Opentracker will take the X-Forwarded-For address instead of the source IP address. Can be added more than once. 79Specifies the IP address or network of the reverse proxies. Opentracker will take the X-Forwarded-For address instead of the source IP address. Can be added more than once.
80.Pp 80.Pp
81.It livesync.cluster.listen Ar ip_address:port 81.It livesync.cluster.listen Ar ip_address:port
82Specifies the IP address and port opentracker will listen on for incoming live sync packets to keep a cluster of opentrackers synchronized. 82Specifies the IP address and port opentracker will listen on for incoming live
83sync packets to keep a cluster of opentrackers synchronized. Every member of a
84cluster has to agree on the port, which is also the one packets are sent to.
85.Pp
86.It livesync.cluster.group Ar ip_address
87Specifies the multicast group a cluster talks in. For reaosns of backwards
88comaptibility with legacy trackers, it defaults to 224.0.23.5, which,
89technically sits in a range IANA assigns on request.
90
91A cluster of its own is better served by an address out of 239.192.0.0/14,
92which RFC 2365 sets aside for private use. Packets carry a ttl of 1 either
93way and never leave the local link.
83.Pp 94.Pp
84.It livesync.cluster.node_ip Ar ip_address 95.It livesync.cluster.node_ip Ar ip_address
85Specifies one trusted IP address for sync between trackers running in a cluster. Can be added more than once. 96Specifies one trusted IP address for sync between trackers running in a cluster. Can be added more than once.
diff --git a/opentracker.conf.sample b/opentracker.conf.sample
index 491c88a..7b35369 100644
--- a/opentracker.conf.sample
+++ b/opentracker.conf.sample
@@ -138,12 +138,22 @@
138# livesync.cluster.listen 192.168.0.1:9696 138# livesync.cluster.listen 192.168.0.1:9696
139# 139#
140# Note that two udp sockets will be opened. One on ip address 0.0.0.0 140# Note that two udp sockets will be opened. One on ip address 0.0.0.0
141# port 9696, that will join the multicast group 224.0.42.23 for incoming 141# port 9696, that will join the multicast group for incoming udp packets
142# udp packets and one on ip address 192.168.0.1 port 9696 for outgoing 142# and one on ip address 192.168.0.1 port 9696 for outgoing ones. Every
143# udp packets. 143# member of a cluster has to agree on the port as well as the group.
144#
145# The group a cluster talks in.
146#
147# livesync.cluster.group 239.192.23.5
148#
149# For reaosns of backwards comaptibility with legacy trackers, it
150# defaults to 224.0.23.5, which, technically sits in a range IANA
151# assigns on request.
152#
153# A cluster of your own is better served by an address out of
154# 239.192.0.0/14, which RFC 2365 sets aside for private use. Packets
155# carry a ttl of 1 either way, so they never leave the local link.
144# 156#
145# As of now one and only one ip address must be given, if opentracker
146# was built with the WANT_SYNC_LIVE feature.
147# 157#
148 158
149# IV) Sync between trackers running in a cluster is restricted to packets 159# IV) Sync between trackers running in a cluster is restricted to packets
diff --git a/src/opentracker.c b/src/opentracker.c
index 14b817e..60a6d8c 100644
--- a/src/opentracker.c
+++ b/src/opentracker.c
@@ -109,7 +109,7 @@ static void *server_mainloop(void *args) {
109 109
110 (void)args; 110 (void)args;
111 111
112 if (workstruct_init(&ws) == -1) 112 if (workstruct_init(&ws, G_INBUF_SIZE) == -1)
113 panic("Initializing worker failed"); 113 panic("Initializing worker failed");
114 114
115 while (g_opentracker_running) { 115 while (g_opentracker_running) {
diff --git a/src/ot_config.c b/src/ot_config.c
index 95136b5..4064d1b 100644
--- a/src/ot_config.c
+++ b/src/ot_config.c
@@ -208,6 +208,7 @@ typedef enum {
208 CONFIG_TRACKER_REDIRECT_URL, 208 CONFIG_TRACKER_REDIRECT_URL,
209 CONFIG_LIVESYNC_CLUSTER_NODE_IP, 209 CONFIG_LIVESYNC_CLUSTER_NODE_IP,
210 CONFIG_LIVESYNC_CLUSTER_LISTEN, 210 CONFIG_LIVESYNC_CLUSTER_LISTEN,
211 CONFIG_LIVESYNC_CLUSTER_GROUP,
211} config_kind; 212} config_kind;
212 213
213typedef struct { 214typedef struct {
@@ -247,6 +248,7 @@ static const config_key config_keys[] = {
247#ifdef WANT_SYNC_LIVE 248#ifdef WANT_SYNC_LIVE
248 CONFIG_KEY("livesync.cluster.node_ip", CONFIG_LIVESYNC_CLUSTER_NODE_IP), 249 CONFIG_KEY("livesync.cluster.node_ip", CONFIG_LIVESYNC_CLUSTER_NODE_IP),
249 CONFIG_KEY("livesync.cluster.listen", CONFIG_LIVESYNC_CLUSTER_LISTEN), 250 CONFIG_KEY("livesync.cluster.listen", CONFIG_LIVESYNC_CLUSTER_LISTEN),
251 CONFIG_KEY("livesync.cluster.group", CONFIG_LIVESYNC_CLUSTER_GROUP),
250#endif 252#endif
251}; 253};
252static const size_t config_entry_count = sizeof(config_keys) / sizeof(config_keys[0]); 254static const size_t config_entry_count = sizeof(config_keys) / sizeof(config_keys[0]);
@@ -378,6 +380,10 @@ static int parse_configfile(char *config_filename) {
378 set_config_option(&g_redirecturl, p); 380 set_config_option(&g_redirecturl, p);
379 break; 381 break;
380#ifdef WANT_SYNC_LIVE 382#ifdef WANT_SYNC_LIVE
383 case CONFIG_LIVESYNC_CLUSTER_GROUP:
384 if (!ip_parse(p, tmpip) || livesync_set_group(tmpip) == -1)
385 goto parse_error;
386 break;
381 case CONFIG_LIVESYNC_CLUSTER_NODE_IP: 387 case CONFIG_LIVESYNC_CLUSTER_NODE_IP:
382 if (!scan_ip6_net(p, &tmpnet)) 388 if (!scan_ip6_net(p, &tmpnet))
383 goto parse_error; 389 goto parse_error;
diff --git a/src/ot_livesync.c b/src/ot_livesync.c
index 083fe1a..85577f7 100644
--- a/src/ot_livesync.c
+++ b/src/ot_livesync.c
@@ -4,6 +4,7 @@
4/* System */ 4/* System */
5#include <arpa/inet.h> 5#include <arpa/inet.h>
6#include <pthread.h> 6#include <pthread.h>
7#include <stdio.h>
7#include <stdlib.h> 8#include <stdlib.h>
8#include <string.h> 9#include <string.h>
9#include <sys/socket.h> 10#include <sys/socket.h>
@@ -21,7 +22,10 @@
21 22
22#ifdef WANT_SYNC_LIVE 23#ifdef WANT_SYNC_LIVE
23 24
24ot_ip6 groupip_1 = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 224, 0, 23, 5}; 25/* The multicast group, legacy opentrackers joined by default 224.0.23.5, so it
26 stays the default */
27static ot_ip6 g_group = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 224, 0, 23, 5};
28static uint16_t g_port = LIVESYNC_PORT;
25 29
26#define LIVESYNC_INCOMING_BUFFSIZE (256 * 256) 30#define LIVESYNC_INCOMING_BUFFSIZE (256 * 256)
27 31
@@ -59,7 +63,7 @@ void livesync_init(void) {
59 uint32_t packet_type4 = htonl(OT_SYNC_PEER4); 63 uint32_t packet_type4 = htonl(OT_SYNC_PEER4);
60 64
61 if (g_socket_in == -1) 65 if (g_socket_in == -1)
62 exerr("No socket address for live sync specified."); 66 return;
63 67
64 /* Prepare outgoing peers buffer */ 68 /* Prepare outgoing peers buffer */
65 memcpy(g_v6_buf.data, &g_tracker_id, sizeof(g_tracker_id)); 69 memcpy(g_v6_buf.data, &g_tracker_id, sizeof(g_tracker_id));
@@ -94,6 +98,14 @@ void livesync_deinit(void) {
94 close(g_socket_out); 98 close(g_socket_out);
95} 99}
96 100
101int livesync_set_group(const ot_ip6 group) {
102 /* v4 multicast is 224.0.0.0/4, and the socket layer has nothing else */
103 if (!ip_is_v4mapped(group) || (group[12] & 0xf0) != 0xe0)
104 return -1;
105 memcpy(g_group, group, sizeof(ot_ip6));
106 return 0;
107}
108
97void livesync_bind_mcast(ot_ip6 ip, uint16_t port) { 109void livesync_bind_mcast(ot_ip6 ip, uint16_t port) {
98 if (!ip_is_v4mapped(ip)) 110 if (!ip_is_v4mapped(ip))
99 exerr("v6 mcast support not yet available."); 111 exerr("v6 mcast support not yet available.");
@@ -101,7 +113,9 @@ void livesync_bind_mcast(ot_ip6 ip, uint16_t port) {
101 if (g_socket_in != -1) 113 if (g_socket_in != -1)
102 exerr("Error: Livesync listen ip specified twice."); 114 exerr("Error: Livesync listen ip specified twice.");
103 115
104 if ((g_socket_in = socket_mcast_join(groupip_1, ip, port)) == -1) 116 g_port = port;
117
118 if ((g_socket_in = socket_mcast_join(g_group, ip, port)) == -1)
105 exerr("Error: Cant bind live sync incoming socket."); 119 exerr("Error: Cant bind live sync incoming socket.");
106 120
107 if ((g_socket_out = socket_mcast_bind(ip, port, 0)) == -1) 121 if ((g_socket_out = socket_mcast_bind(ip, port, 0)) == -1)
@@ -121,7 +135,7 @@ static void livesync_issue_peersync(sync_buffer *buf) {
121 has modified the protected element */ 135 has modified the protected element */
122 pthread_mutex_unlock(&g_outbuf_mutex); 136 pthread_mutex_unlock(&g_outbuf_mutex);
123 137
124 socket_mcast_send(g_socket_out, mycopy, fill, groupip_1, LIVESYNC_PORT); 138 socket_mcast_send(g_socket_out, mycopy, fill, g_group, g_port);
125} 139}
126 140
127static void livesync_handle_peersync(struct ot_workstruct *ws, size_t peer_size) { 141static void livesync_handle_peersync(struct ot_workstruct *ws, size_t peer_size) {
@@ -159,6 +173,9 @@ static void livesync_handle_peersync(struct ot_workstruct *ws, size_t peer_size)
159 stuck when there's not enough traffic to fill udp packets fast 173 stuck when there's not enough traffic to fill udp packets fast
160 enough */ 174 enough */
161void livesync_ticker(void) { 175void livesync_ticker(void) {
176 if (g_socket_out == -1)
177 return;
178
162 /* livesync_issue_peersync sets g_next_packet_time */ 179 /* livesync_issue_peersync sets g_next_packet_time */
163 pthread_mutex_lock(&g_outbuf_mutex); 180 pthread_mutex_lock(&g_outbuf_mutex);
164 if (time_now_seconds() > g_v6_buf.next_packet_time && g_v6_buf.fill > sizeof(g_tracker_id) + sizeof(uint32_t)) 181 if (time_now_seconds() > g_v6_buf.next_packet_time && g_v6_buf.fill > sizeof(g_tracker_id) + sizeof(uint32_t))
@@ -176,8 +193,14 @@ void livesync_ticker(void) {
176/* Inform live sync about whats going on. */ 193/* Inform live sync about whats going on. */
177void livesync_tell(struct ot_workstruct *ws) { 194void livesync_tell(struct ot_workstruct *ws) {
178 size_t peer_size; /* initialized in next line */ 195 size_t peer_size; /* initialized in next line */
179 ot_peer *peer_src = peer_from_peer6(&ws->peer, &peer_size); 196 ot_peer *peer_src;
180 sync_buffer *dest_buf = peer_size == OT_PEER_SIZE6 ? &g_v6_buf : &g_v4_buf; 197 sync_buffer *dest_buf;
198
199 if (g_socket_out == -1)
200 return;
201
202 peer_src = peer_from_peer6(&ws->peer, &peer_size);
203 dest_buf = peer_size == OT_PEER_SIZE6 ? &g_v6_buf : &g_v4_buf;
181 204
182 pthread_mutex_lock(&g_outbuf_mutex); 205 pthread_mutex_lock(&g_outbuf_mutex);
183 206
@@ -200,9 +223,12 @@ static void *livesync_worker(void *args) {
200 223
201 (void)args; 224 (void)args;
202 225
203 /* Initialize our "thread local storage" */ 226 if (workstruct_init(&ws, LIVESYNC_INCOMING_BUFFSIZE) == -1) {
204 ws.inbuf = ws.request = malloc(LIVESYNC_INCOMING_BUFFSIZE); 227 fprintf(stderr, "Warning: live sync could not get its buffers and will not run.\n");
205 ws.outbuf = ws.reply = 0; 228 atomic_store(&thread_left, 1);
229 return NULL;
230 }
231 ws.request = ws.inbuf;
206 232
207 while (g_opentracker_running) { 233 while (g_opentracker_running) {
208 uint32_t packet_type; 234 uint32_t packet_type;
@@ -233,7 +259,7 @@ static void *livesync_worker(void *args) {
233 } 259 }
234 } 260 }
235 261
236 free(ws.inbuf); 262 workstruct_deinit(&ws);
237 atomic_store(&thread_left, 1); 263 atomic_store(&thread_left, 1);
238 return NULL; 264 return NULL;
239} 265}
diff --git a/src/ot_livesync.h b/src/ot_livesync.h
index 4bd3af0..cf3c64c 100644
--- a/src/ot_livesync.h
+++ b/src/ot_livesync.h
@@ -7,7 +7,8 @@
7#include "trackerlogic.h" 7#include "trackerlogic.h"
8 8
9/* 9/*
10 Syncing is done as udp packets in the multicast domain 224.0.42.5 port 9696 10 Syncing is done as udp packets in the multicast group 224.0.23.5, on the
11 port livesync.cluster.listen names, 9696 by default
11 12
12 Each tracker should join the multicast group and send its live sync packets 13 Each tracker should join the multicast group and send its live sync packets
13 to that group, using a ttl of 1 14 to that group, using a ttl of 1
@@ -47,6 +48,10 @@
47void livesync_init(void); 48void livesync_init(void);
48void livesync_deinit(void); 49void livesync_deinit(void);
49 50
51/* The group to join and to send to, (defaults to 224.0.23.5).
52 Returns -1 if group is not a v4 multicast address. */
53int livesync_set_group(const ot_ip6 group);
54
50/* Join multicast group for listening and create sending socket */ 55/* Join multicast group for listening and create sending socket */
51void livesync_bind_mcast(ot_ip6 ip, uint16_t port); 56void livesync_bind_mcast(ot_ip6 ip, uint16_t port);
52 57
diff --git a/src/ot_worker.c b/src/ot_worker.c
index 432d272..ef50945 100644
--- a/src/ot_worker.c
+++ b/src/ot_worker.c
@@ -20,7 +20,7 @@ static size_t g_worker_count;
20static void *worker_trampoline(void *args) { 20static void *worker_trampoline(void *args) {
21 struct ot_worker *me = args; 21 struct ot_worker *me = args;
22 22
23 if (workstruct_init(&me->ws) == -1) 23 if (workstruct_init(&me->ws, G_INBUF_SIZE) == -1)
24 fprintf(stderr, "Warning: a worker could not get its buffers and will not run.\n"); 24 fprintf(stderr, "Warning: a worker could not get its buffers and will not run.\n");
25 else { 25 else {
26 me->run(me); 26 me->run(me);
diff --git a/src/trackerlogic.c b/src/trackerlogic.c
index 9565090..8e30a48 100644
--- a/src/trackerlogic.c
+++ b/src/trackerlogic.c
@@ -567,9 +567,11 @@ void trackerlogic_init(void) {
567 stats_init(); 567 stats_init();
568} 568}
569 569
570int workstruct_init(struct ot_workstruct *ws) { 570/* Live sync reads whole packets rather than requests and wants a larger
571 inbuf than the request path does, hence the size. */
572int workstruct_init(struct ot_workstruct *ws, size_t inbuf_size) {
571 memset(ws, 0, sizeof(*ws)); 573 memset(ws, 0, sizeof(*ws));
572 ws->inbuf = malloc(G_INBUF_SIZE); 574 ws->inbuf = malloc(inbuf_size);
573 ws->outbuf = malloc(G_OUTBUF_SIZE); 575 ws->outbuf = malloc(G_OUTBUF_SIZE);
574#ifdef _DEBUG_HTTPERROR 576#ifdef _DEBUG_HTTPERROR
575 ws->debugbuf = malloc(G_DEBUGBUF_SIZE); 577 ws->debugbuf = malloc(G_DEBUGBUF_SIZE);
diff --git a/src/trackerlogic.h b/src/trackerlogic.h
index 7292d36..692b6af 100644
--- a/src/trackerlogic.h
+++ b/src/trackerlogic.h
@@ -242,7 +242,7 @@ struct ot_workstruct {
242/* Everything a thread that serves requests needs for itself. Set up and torn 242/* Everything a thread that serves requests needs for itself. Set up and torn
243 down in one place so that a new kind of worker cannot miss a piece. 243 down in one place so that a new kind of worker cannot miss a piece.
244 Returns -1 if the buffers could not be had. */ 244 Returns -1 if the buffers could not be had. */
245int workstruct_init(struct ot_workstruct *ws); 245int workstruct_init(struct ot_workstruct *ws, size_t inbuf_size);
246void workstruct_deinit(struct ot_workstruct *ws); 246void workstruct_deinit(struct ot_workstruct *ws);
247 247
248/* 248/*
diff --git a/tests/integration/harness.py b/tests/integration/harness.py
index b98ce83..10aa6de 100644
--- a/tests/integration/harness.py
+++ b/tests/integration/harness.py
@@ -20,6 +20,9 @@ import urllib.error
20import urllib.parse 20import urllib.parse
21import urllib.request 21import urllib.request
22 22
23# groupip_1 in src/ot_livesync.c
24LIVESYNC_GROUP = "224.0.23.5"
25
23TRACKER = os.environ.get("OPENTRACKER", os.path.join(os.path.dirname(__file__), "..", "..", "opentracker.debug")) 26TRACKER = os.environ.get("OPENTRACKER", os.path.join(os.path.dirname(__file__), "..", "..", "opentracker.debug"))
24 27
25 28
@@ -76,6 +79,9 @@ class Tracker:
76 79
77 def __init__(self, extra_config="", files=None): 80 def __init__(self, extra_config="", files=None):
78 self.port = free_port() 81 self.port = free_port()
82 # Only used by a config that asks for live sync, but allocated either
83 # way so {livesync_port} always resolves.
84 self.livesync_port = free_port()
79 self.dir = tempfile.mkdtemp(prefix="ot-integration-") 85 self.dir = tempfile.mkdtemp(prefix="ot-integration-")
80 self.log = None 86 self.log = None
81 self.process = None 87 self.process = None
@@ -87,7 +93,7 @@ class Tracker:
87 with open(self.config, "w") as handle: 93 with open(self.config, "w") as handle:
88 handle.write("listen.tcp 127.0.0.1:%d\n" % self.port) 94 handle.write("listen.tcp 127.0.0.1:%d\n" % self.port)
89 handle.write("listen.udp 127.0.0.1:%d\n" % self.port) 95 handle.write("listen.udp 127.0.0.1:%d\n" % self.port)
90 handle.write(extra_config.replace("{dir}", self.dir)) 96 handle.write(extra_config.replace("{dir}", self.dir).replace("{livesync_port}", str(self.livesync_port)))
91 self.log = open(os.path.join(self.dir, "tracker.log"), "w+b") 97 self.log = open(os.path.join(self.dir, "tracker.log"), "w+b")
92 # Leak detection is left at the sanitizer's own default, which is on 98 # Leak detection is left at the sanitizer's own default, which is on
93 # where LeakSanitizer exists and off where it does not. Asking for it 99 # where LeakSanitizer exists and off where it does not. Asking for it
@@ -186,6 +192,32 @@ class Tracker:
186 def torrent_count(self): 192 def torrent_count(self):
187 return int(self.stats("torr").split()[0]) 193 return int(self.stats("torr").split()[0])
188 194
195 def livesync_send(self, payload, group=LIVESYNC_GROUP):
196 """Put a packet on the live sync group.
197
198 It has to go to the group rather than straight at the port: the
199 tracker binds two sockets there, the reader on the wildcard address
200 and the writer on the configured one, and the kernel gives a unicast
201 datagram to the more specific of the two, which nobody reads."""
202 sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
203 try:
204 sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF, socket.inet_aton("127.0.0.1"))
205 sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, 1)
206 sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 1)
207 sock.sendto(payload, (group, self.livesync_port))
208 finally:
209 sock.close()
210
211 def wait_until(self, predicate, timeout=5):
212 """A live sync packet is handled by another thread, so an effect of
213 one arrives some time after the send."""
214 deadline = time.time() + timeout
215 while time.time() < deadline:
216 if predicate():
217 return True
218 time.sleep(0.02)
219 return predicate()
220
189 # ------------------------------------------------------------ over udp 221 # ------------------------------------------------------------ over udp
190 222
191 def udp_connect(self): 223 def udp_connect(self):
diff --git a/tests/integration/test_livesync.py b/tests/integration/test_livesync.py
new file mode 100644
index 0000000..52e871f
--- /dev/null
+++ b/tests/integration/test_livesync.py
@@ -0,0 +1,208 @@
1"""Live sync, driven over the multicast group a cluster would use.
2
3Live sync sits behind WANT_SYNC_LIVE, so these only run against a binary
4built with it: make check-livesync. Against any other build they skip, which
5is what the feature list in -v is for.
6"""
7
8import socket
9import struct
10import subprocess
11import unittest
12
13from harness import TRACKER, TrackerTestCase
14
15# enum { OT_SYNC_PEER4, OT_SYNC_PEER6 } in src/ot_livesync.c
16SYNC_PEER4 = 0
17SYNC_PEER6 = 1
18
19# src/trackerlogic.h
20FLAG_SEEDING = 0x80
21FLAG_COMPLETED = 0x40
22FLAG_STOPPED = 0x20
23
24
25def has_livesync():
26 done = subprocess.run([TRACKER, "-v"], capture_output=True, timeout=30)
27 return b"WANT_SYNC_LIVE" in done.stderr
28
29
30def peer4(address, port, flags):
31 """Four bytes of address, two of port, one of flags, one the tracker
32 overwrites with its own idea of the time."""
33 return socket.inet_aton(address) + struct.pack("!H", port) + bytes([flags, 0])
34
35
36def peer6(address, port, flags):
37 return socket.inet_pton(socket.AF_INET6, address) + struct.pack("!H", port) + bytes([flags, 0])
38
39
40def packet(kind, entries, tracker_id=0xDEADBEEF):
41 body = b"".join(info_hash + peer for info_hash, peer in entries)
42 return struct.pack("!I", tracker_id) + struct.pack("!I", kind) + body
43
44
45@unittest.skipUnless(has_livesync(), "this build has no WANT_SYNC_LIVE")
46class LiveSyncIn(TrackerTestCase):
47 config = "livesync.cluster.listen 127.0.0.1:{livesync_port}\nlivesync.cluster.node_ip 127.0.0.1\n"
48
49 def counts(self, info_hash):
50 """Seeders and leechers a scrape reports for one torrent."""
51 files = self.tracker.scrape(info_hash)[b"files"]
52 if info_hash not in files:
53 return None
54 entry = files[info_hash]
55 return entry[b"complete"], entry[b"incomplete"]
56
57 def test_a_synced_seeder_arrives(self):
58 info_hash = self.hash_for("sync-seed")
59 self.tracker.livesync_send(packet(SYNC_PEER4, [(info_hash, peer4("10.1.2.3", 6881, FLAG_SEEDING))]))
60 self.assertTrue(self.tracker.wait_until(lambda: self.counts(info_hash) == (1, 0)),
61 "scrape says %r" % (self.counts(info_hash),))
62
63 def test_a_synced_leecher_arrives(self):
64 info_hash = self.hash_for("sync-leech")
65 self.tracker.livesync_send(packet(SYNC_PEER4, [(info_hash, peer4("10.1.2.4", 6881, 0))]))
66 self.assertTrue(self.tracker.wait_until(lambda: self.counts(info_hash) == (0, 1)),
67 "scrape says %r" % (self.counts(info_hash),))
68
69 def test_a_v6_peer_arrives(self):
70 info_hash = self.hash_for("sync-v6")
71 self.tracker.livesync_send(packet(SYNC_PEER6, [(info_hash, peer6("2001:db8::1", 6881, FLAG_SEEDING))]))
72 self.assertTrue(self.tracker.wait_until(lambda: self.counts(info_hash) == (1, 0)),
73 "scrape says %r" % (self.counts(info_hash),))
74
75 def test_one_packet_carries_several_torrents(self):
76 hashes = [self.hash_for("sync-many-%d" % i) for i in range(4)]
77 self.tracker.livesync_send(packet(SYNC_PEER4, [(h, peer4("10.1.2.%d" % (5 + i), 6881, FLAG_SEEDING))
78 for i, h in enumerate(hashes)]))
79 self.assertTrue(self.tracker.wait_until(lambda: all(self.counts(h) == (1, 0) for h in hashes)))
80
81 def test_a_stopped_peer_is_taken_away_again(self):
82 info_hash = self.hash_for("sync-stop")
83 peer = peer4("10.1.2.9", 6881, FLAG_SEEDING)
84 self.tracker.livesync_send(packet(SYNC_PEER4, [(info_hash, peer)]))
85 self.assertTrue(self.tracker.wait_until(lambda: self.counts(info_hash) == (1, 0)))
86
87 self.tracker.livesync_send(packet(SYNC_PEER4, [(info_hash, peer4("10.1.2.9", 6881, FLAG_STOPPED))]))
88 self.assertTrue(self.tracker.wait_until(lambda: self.counts(info_hash) in (None, (0, 0))),
89 "scrape still says %r" % (self.counts(info_hash),))
90
91 def test_a_truncated_entry_is_dropped(self):
92 """The loop takes whole hash and peer pairs and leaves a short tail
93 where it is, so the complete entry before it still lands."""
94 good, short = self.hash_for("sync-good"), self.hash_for("sync-short")
95 payload = packet(SYNC_PEER4, [(good, peer4("10.1.2.10", 6881, FLAG_SEEDING))]) + short + b"\x0a\x01"
96 self.tracker.livesync_send(payload)
97 self.assertTrue(self.tracker.wait_until(lambda: self.counts(good) == (1, 0)))
98 self.assertIsNone(self.counts(short))
99
100 def test_an_unknown_packet_type_is_ignored(self):
101 info_hash = self.hash_for("sync-unknown")
102 self.tracker.livesync_send(packet(99, [(info_hash, peer4("10.1.2.11", 6881, FLAG_SEEDING))]))
103 self.assertFalse(self.tracker.wait_until(lambda: self.counts(info_hash) is not None, timeout=1))
104
105 def test_synced_peers_are_counted(self):
106 """The live sync worker keeps its own tally like every other worker,
107 and ?mode=syncs is the sum of them."""
108 before = int(self.tracker.stats("syncs").split()[0])
109 hashes = [self.hash_for("sync-count-%d" % i) for i in range(3)]
110 self.tracker.livesync_send(packet(SYNC_PEER4, [(h, peer4("10.1.2.%d" % (20 + i), 6881, FLAG_SEEDING))
111 for i, h in enumerate(hashes)]))
112 self.assertTrue(self.tracker.wait_until(
113 lambda: int(self.tracker.stats("syncs").split()[0]) == before + 3),
114 "syncs says %s" % self.tracker.stats("syncs").split()[0])
115
116 def test_a_packet_with_nothing_but_a_header_is_ignored(self):
117 before = self.tracker.torrent_count()
118 self.tracker.livesync_send(struct.pack("!I", 0xDEADBEEF) + struct.pack("!I", SYNC_PEER4))
119 self.assertFalse(self.tracker.wait_until(lambda: self.tracker.torrent_count() != before, timeout=1))
120
121
122@unittest.skipUnless(has_livesync(), "this build has no WANT_SYNC_LIVE")
123class LiveSyncAccess(TrackerTestCase):
124 """Nothing blesses 127.0.0.1 here, so the tracker should take nothing
125 from it. Source address checking is all live sync has."""
126
127 config = "livesync.cluster.listen 127.0.0.1:{livesync_port}\nlivesync.cluster.node_ip 10.0.0.1\n"
128
129 def test_an_unblessed_sender_is_ignored(self):
130 info_hash = self.hash_for("sync-unblessed")
131 self.tracker.livesync_send(packet(SYNC_PEER4, [(info_hash, peer4("10.1.2.12", 6881, FLAG_SEEDING))]))
132 self.assertFalse(self.tracker.wait_until(lambda: self.tracker.torrent_count() != 0, timeout=1))
133
134
135@unittest.skipUnless(has_livesync(), "this build has no WANT_SYNC_LIVE")
136class LiveSyncUnconfigured(TrackerTestCase):
137 """A binary that has the feature but no cluster to join. Somebody whose
138 packager turned WANT_SYNC_LIVE on did not ask for live sync and should
139 not be able to tell that it is there."""
140
141 config = ""
142
143 def test_the_tracker_runs_anyway(self):
144 info_hash = self.hash_for("no-cluster")
145 self.tracker.announce(info_hash, port=6881, left=0)
146 self.assertEqual(self.tracker.torrent_count(), 1)
147
148 def test_nothing_is_said_about_the_missing_cluster(self):
149 self.assertNotIn("sync", self.tracker.output().lower())
150
151 def test_no_socket_is_bound_for_it(self):
152 """A configured tracker holds the wildcard address on that port, so
153 taking it here proves nothing was started."""
154 sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
155 try:
156 sock.bind(("0.0.0.0", self.tracker.livesync_port))
157 finally:
158 sock.close()
159
160 def test_announcing_stays_clean(self):
161 """Enough announces to pass the watermark that would flush an
162 outgoing buffer, if there were one to flush. The sanitizers in this
163 build turn anything the attempt touched into a failure."""
164 for i in range(200):
165 self.tracker.announce(self.hash_for("no-cluster-%d" % i), port=6881 + i, left=0)
166 self.assertEqual(self.tracker.torrent_count(), 200)
167
168
169# RFC 2365 sets 239.192.0.0/14 aside for exactly this kind of use
170OWN_GROUP = "239.192.0.7"
171
172
173@unittest.skipUnless(has_livesync(), "this build has no WANT_SYNC_LIVE")
174class LiveSyncGroup(TrackerTestCase):
175 config = ("livesync.cluster.group %s\n"
176 "livesync.cluster.listen 127.0.0.1:{livesync_port}\n"
177 "livesync.cluster.node_ip 127.0.0.1\n" % OWN_GROUP)
178
179 def test_the_configured_group_is_joined(self):
180 info_hash = self.hash_for("own-group")
181 self.tracker.livesync_send(packet(SYNC_PEER4, [(info_hash, peer4("10.1.3.1", 6881, FLAG_SEEDING))]),
182 group=OWN_GROUP)
183 self.assertTrue(self.tracker.wait_until(lambda: self.tracker.torrent_count() == 1))
184
185 def test_the_default_group_is_left_alone(self):
186 """A cluster that picked its own group should not hear the one every
187 other opentracker is on."""
188 info_hash = self.hash_for("not-our-group")
189 self.tracker.livesync_send(packet(SYNC_PEER4, [(info_hash, peer4("10.1.3.2", 6881, FLAG_SEEDING))]))
190 self.assertFalse(self.tracker.wait_until(lambda: self.tracker.torrent_count() != 0, timeout=1))
191
192
193@unittest.skipUnless(has_livesync(), "this build has no WANT_SYNC_LIVE")
194class LiveSyncGroupRefused(TrackerTestCase):
195 """A unicast address is not a group. The line is refused like any other
196 bad config line, which leaves the default standing."""
197
198 config = ("livesync.cluster.group 10.0.0.1\n"
199 "livesync.cluster.listen 127.0.0.1:{livesync_port}\n"
200 "livesync.cluster.node_ip 127.0.0.1\n")
201
202 def test_the_line_is_refused(self):
203 self.assertIn("Parse error", self.tracker.output())
204
205 def test_and_the_default_group_still_works(self):
206 info_hash = self.hash_for("fallback-group")
207 self.tracker.livesync_send(packet(SYNC_PEER4, [(info_hash, peer4("10.1.3.3", 6881, FLAG_SEEDING))]))
208 self.assertTrue(self.tracker.wait_until(lambda: self.tracker.torrent_count() == 1))
diff --git a/sync_daemon.pl b/tests/sync_daemon.pl
index 9e4bdb9..9e4bdb9 100644
--- a/sync_daemon.pl
+++ b/tests/sync_daemon.pl
diff --git a/tests/t_trackerlogic.c b/tests/t_trackerlogic.c
index f64977f..1cf9141 100644
--- a/tests/t_trackerlogic.c
+++ b/tests/t_trackerlogic.c
@@ -224,7 +224,7 @@ void test_trackerlogic(void) {
224 pthread_sigmask(SIG_BLOCK, &signal_mask, NULL); 224 pthread_sigmask(SIG_BLOCK, &signal_mask, NULL);
225 225
226 trackerlogic_init(); 226 trackerlogic_init();
227 CHECK_INT(workstruct_init(&ws), 0); 227 CHECK_INT(workstruct_init(&ws, G_INBUF_SIZE), 0);
228 228
229 case_an_announce_makes_a_torrent(); 229 case_an_announce_makes_a_torrent();
230 case_the_same_peer_twice_is_one_peer(); 230 case_the_same_peer_twice_is_one_peer();