summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.gitignore8
-rw-r--r--Makefile40
-rw-r--r--man4/opentracker.conf.439
-rw-r--r--opentracker.c16
-rw-r--r--opentracker.conf.sample14
-rw-r--r--ot_config.c2
-rw-r--r--ot_livesync.c22
-rw-r--r--ot_udp.c58
-rw-r--r--ot_udp.h3
-rw-r--r--tests/integration/test_shutdown.py58
-rw-r--r--tests/t_fdstate.c12
-rw-r--r--tests/t_socket.c57
-rw-r--r--tests/udp_wake_probe.c42
-rw-r--r--trackerlogic.h4
14 files changed, 310 insertions, 65 deletions
diff --git a/.gitignore b/.gitignore
index 8b4bec1..b5c40f0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,10 @@
1*.o 1*.o
2*.dSYM
3opentracker
4opentracker.debug
5opentracker.integration
6opentracker.tsan
7proxy
8proxy.debug
2tests/runner 9tests/runner
10tests/runner.sanitize
diff --git a/Makefile b/Makefile
index 1a8912b..f79d6a6 100644
--- a/Makefile
+++ b/Makefile
@@ -73,13 +73,6 @@ HEADERS=trackerlogic.h ot_urlencoded.h ot_batch.h ot_buf.h ot_event.h ot_fdstate
73SOURCES=opentracker.c trackerlogic.c ot_urlencoded.c ot_batch.c ot_buf.c ot_event.c ot_fdstate.c ot_fmt.c ot_hex.c ot_ip.c ot_config.c ot_scan.c ot_socket.c ot_mutex.c ot_random.c ot_stats.c ot_vector.c ot_clean.c ot_udp.c ot_iovec.c ot_fullscrape.c ot_accesslist.c ot_http.c ot_livesync.c ot_rijndael.c 73SOURCES=opentracker.c trackerlogic.c ot_urlencoded.c ot_batch.c ot_buf.c ot_event.c ot_fdstate.c ot_fmt.c ot_hex.c ot_ip.c ot_config.c ot_scan.c ot_socket.c ot_mutex.c ot_random.c ot_stats.c ot_vector.c ot_clean.c ot_udp.c ot_iovec.c ot_fullscrape.c ot_accesslist.c ot_http.c ot_livesync.c ot_rijndael.c
74SOURCES_proxy=proxy.c ot_batch.c ot_fdstate.c ot_vector.c ot_mutex.c ot_random.c ot_scan.c ot_ip.c ot_socket.c 74SOURCES_proxy=proxy.c ot_batch.c ot_fdstate.c ot_vector.c ot_mutex.c ot_random.c ot_scan.c ot_ip.c ot_socket.c
75 75
76OBJECTS = $(SOURCES:%.c=%.o)
77OBJECTS_debug = $(SOURCES:%.c=%.debug.o)
78OBJECTS_proxy = $(SOURCES_proxy:%.c=%.proxy.o)
79OBJECTS_proxy_debug = $(SOURCES_proxy:%.c=%.proxy.debug.o)
80
81.SUFFIXES: .proxy.debug.o .proxy.o .debug.o .o .c
82
83all: $(BINARY) $(BINARY).debug 76all: $(BINARY) $(BINARY).debug
84 77
85CFLAGS_production = $(CFLAGS) $(OPTS_production) $(FEATURES) 78CFLAGS_production = $(CFLAGS) $(OPTS_production) $(FEATURES)
@@ -88,32 +81,15 @@ CFLAGS_debug = $(CFLAGS) $(OPTS_debug) $(FEATURES)
88CFLAGS_proxy = $(CFLAGS_production) $(CFLAGS_libowfat) -DWANT_SYNC_LIVE 81CFLAGS_proxy = $(CFLAGS_production) $(CFLAGS_libowfat) -DWANT_SYNC_LIVE
89CFLAGS_proxy_debug = $(CFLAGS_debug) $(CFLAGS_libowfat) -DWANT_SYNC_LIVE 82CFLAGS_proxy_debug = $(CFLAGS_debug) $(CFLAGS_libowfat) -DWANT_SYNC_LIVE
90 83
91$(BINARY): $(OBJECTS) 84$(BINARY): $(SOURCES) $(HEADERS)
92 $(CC) -o $@ $(OBJECTS) $(LDFLAGS) 85 $(CC) -o $@ $(CFLAGS_production) $(SOURCES) $(LDFLAGS)
93 $(STRIP) $@ 86 $(STRIP) $@
94$(BINARY).debug: $(OBJECTS_debug) 87$(BINARY).debug: $(SOURCES) $(HEADERS)
95 $(CC) -o $@ $(OBJECTS_debug) $(LDFLAGS) 88 $(CC) -o $@ $(CFLAGS_debug) $(SOURCES) $(LDFLAGS)
96proxy: $(OBJECTS_proxy) 89proxy: $(SOURCES_proxy) $(HEADERS)
97 $(CC) -o $@ $(OBJECTS_proxy) $(CFLAGS_production) $(LDFLAGS) $(LDFLAGS_libowfat) 90 $(CC) -o $@ $(CFLAGS_proxy) $(SOURCES_proxy) $(LDFLAGS) $(LDFLAGS_libowfat)
98proxy.debug: $(OBJECTS_proxy_debug) 91proxy.debug: $(SOURCES_proxy) $(HEADERS)
99 $(CC) -o $@ $(OBJECTS_proxy_debug) $(LDFLAGS) $(LDFLAGS_libowfat) 92 $(CC) -o $@ $(CFLAGS_proxy_debug) $(SOURCES_proxy) $(LDFLAGS) $(LDFLAGS_libowfat)
100
101$(OBJECTS): $(HEADERS)
102$(OBJECTS_debug): $(HEADERS)
103$(OBJECTS_proxy): $(HEADERS)
104$(OBJECTS_proxy_debug): $(HEADERS)
105
106.c.proxy.debug.o :
107 $(CC) -c -o $@ $(CFLAGS_proxy_debug) $(<:.proxy.debug.o=.c)
108
109.c.proxy.o :
110 $(CC) -c -o $@ $(CFLAGS_proxy) $(<:.proxy.o=.c)
111
112.c.debug.o :
113 $(CC) -c -o $@ $(CFLAGS_debug) $(<:.debug.o=.c)
114
115.c.o :
116 $(CC) -c -o $@ $(CFLAGS_production) $<
117 93
118# Unit tests. The modules under test are compiled straight into the test 94# Unit tests. The modules under test are compiled straight into the test
119# binary, so a test may reach a static function by including the source if it 95# binary, so a test may reach a static function by including the source if it
diff --git a/man4/opentracker.conf.4 b/man4/opentracker.conf.4
index 2bc1389..b28879f 100644
--- a/man4/opentracker.conf.4
+++ b/man4/opentracker.conf.4
@@ -1,5 +1,5 @@
1.Dd 2024-04-18 1.Dd 2026-10-28
2.Dt opentracker.conf 5 2.Dt opentracker.conf 4
3.Os Unix 3.Os Unix
4.Sh NAME 4.Sh NAME
5.Nm opentracker.conf 5.Nm opentracker.conf
@@ -25,8 +25,29 @@ Specifies the address opentracker will listen on for TCP connections. Can be add
25.It listen.udp Ar address 25.It listen.udp Ar address
26Specifies the address opentracker will listen on for UDP connections. Can be added more than once. 26Specifies the address opentracker will listen on for UDP connections. Can be added more than once.
27.Pp 27.Pp
28A listen statement may carry options after the address, as in
29.Dl listen.udp 10.0.0.5:6969 workers=4 shared
30where
31.Cm workers Ns = Ns Ar count
32sets the worker count for that socket alone and is ignored for TCP, and
33.Cm shared
34permits another process to bind the same address.
35.Pp
28.It listen.udp.workers Ar threads 36.It listen.udp.workers Ar threads
29Specifies how many threads will be spawned to handle UDP connections. Defaults to 4. 37Specifies how many threads will be spawned to handle UDP requests, for every
38listen statement that does not carry a
39.Cm workers
40option of its own. Defaults to 1. A count of 0 serves UDP from the event loop,
41in the thread that also serves TCP. See
42.Sx TUNING
43before raising it.
44.Pp
45.It listen.shared Ar boolean
46Permits another process to bind the same address, for every listen statement
47that does not carry a
48.Cm shared
49option of its own. Defaults to 0, and is useful only where a second tracker is
50deliberately run on the same address.
30.Pp 51.Pp
31.It access.whitelist Ar path/to/whitelist 52.It access.whitelist Ar path/to/whitelist
32Specifies the path to the whitelist file containing all torrent hashes that opentracker will serve. Use this option if opentracker runs in a non-open mode. 53Specifies the path to the whitelist file containing all torrent hashes that opentracker will serve. Use this option if opentracker runs in a non-open mode.
@@ -67,6 +88,18 @@ Specifies the user opentracker will setuid to after binding to potentially privi
67.It tracker.redirect_url Ar URL 88.It tracker.redirect_url Ar URL
68Specifies the URL opentracker will redirect to in response to a "GET / HTTP" request. 89Specifies the URL opentracker will redirect to in response to a "GET / HTTP" request.
69.El 90.El
91.Sh TUNING
92A worker is a thread that blocks in a read until a datagram arrives. Where the
93kernel spreads arriving datagrams over a group of sockets bound to one address,
94opentracker binds one socket per worker and throughput rises with the worker
95count. Linux does so, and FreeBSD since 12.0. Where a kernel does not, every
96worker reads the same socket and they contend for its receive queue, so more
97than one worker costs throughput. opentracker asks at start up and binds a
98group only where the answer is yes.
99.Pp
100.Pa tests/udp_scaling.py
101in the source distribution sweeps the setting against a running tracker.
102Measure on the machine that will carry the traffic.
70.Sh EXAMPLES 103.Sh EXAMPLES
71To specify the address opentracker will listen on for both TCP and UDP connections: 104To specify the address opentracker will listen on for both TCP and UDP connections:
72.Dl listen.tcp_udp 0.0.0.0:6969 105.Dl listen.tcp_udp 0.0.0.0:6969
diff --git a/opentracker.c b/opentracker.c
index f579b92..e0f16d7 100644
--- a/opentracker.c
+++ b/opentracker.c
@@ -49,7 +49,9 @@ static void panic(const char *routine) {
49 exit(111); 49 exit(111);
50} 50}
51 51
52/* Say we are stopping, wake the loop, and let main do the work on its own stack. */ 52/* Say we are stopping, wake the loop, and let main do the work on its own
53 stack. Everything other than SIGINT falls through and does nothing, which
54 is what OT_SIGNAL_WAKE wants: returning from the read is all it is for. */
53static void signal_handler(int s) { 55static void signal_handler(int s) {
54 if (s == SIGINT) { 56 if (s == SIGINT) {
55 const char byte = 0; 57 const char byte = 0;
@@ -85,7 +87,8 @@ static void install_signal_handlers(void) {
85 sa.sa_handler = signal_handler; 87 sa.sa_handler = signal_handler;
86 sigemptyset(&sa.sa_mask); 88 sigemptyset(&sa.sa_mask);
87 sa.sa_flags = 0; 89 sa.sa_flags = 0;
88 if ((sigaction(SIGINT, &sa, NULL) == -1) || (sigaction(SIGALRM, &sa, NULL) == -1)) 90 if ((sigaction(SIGINT, &sa, NULL) == -1) || (sigaction(SIGALRM, &sa, NULL) == -1) ||
91 (sigaction(OT_SIGNAL_WAKE, &sa, NULL) == -1))
89 panic("install_signal_handlers"); 92 panic("install_signal_handlers");
90 93
91 sigaddset(&signal_mask, SIGINT); 94 sigaddset(&signal_mask, SIGINT);
@@ -397,7 +400,6 @@ static void listen_start_udp_workers(void) {
397 if (entry->proto != FLAG_UDP || entry->workers <= 0) 400 if (entry->proto != FLAG_UDP || entry->workers <= 0)
398 continue; 401 continue;
399 402
400 socket_set_blocking((int)entry->sock, 1);
401 event_del((int)entry->sock); 403 event_del((int)entry->sock);
402 udp_worker_install(entry->sock); 404 udp_worker_install(entry->sock);
403 405
@@ -409,8 +411,7 @@ static void listen_start_udp_workers(void) {
409 if (sock == -1) { 411 if (sock == -1) {
410 fprintf(stderr, "Warning: could not bind another socket for udp worker %d: %s. Sharing the first instead.\n", worker, strerror(errno)); 412 fprintf(stderr, "Warning: could not bind another socket for udp worker %d: %s. Sharing the first instead.\n", worker, strerror(errno));
411 sock = entry->sock; 413 sock = entry->sock;
412 } else 414 }
413 socket_set_blocking((int)sock, 1);
414 } 415 }
415 udp_worker_install(sock); 416 udp_worker_install(sock);
416 } 417 }
@@ -548,9 +549,10 @@ int main(int argc, char **argv) {
548 listen_start_udp_workers(); 549 listen_start_udp_workers();
549 550
550 server_mainloop(0); 551 server_mainloop(0);
552 udp_workers_deinit();
551 553
552 /* The loop has left, so no thread is serving any more and the teardown 554 /* No thread is serving any more, so the teardown can run where a debugger
553 can run where a debugger and the sanitizers can follow it */ 555 and the sanitizers can follow it */
554 trackerlogic_deinit(); 556 trackerlogic_deinit();
555#ifdef WANT_SYSLOGS 557#ifdef WANT_SYSLOGS
556 closelog(); 558 closelog();
diff --git a/opentracker.conf.sample b/opentracker.conf.sample
index 4adb736..1d535e8 100644
--- a/opentracker.conf.sample
+++ b/opentracker.conf.sample
@@ -23,7 +23,7 @@
23# 23#
24# workers=<count> handle this udp socket in blocking reads in that 24# workers=<count> handle this udp socket in blocking reads in that
25# many dedicated worker threads instead of in the 25# many dedicated worker threads instead of in the
26# event loop. Ignored for tcp. 26# event loop. Defaults to 1. Ignored for tcp.
27# shared allow another process to bind the same address and 27# shared allow another process to bind the same address and
28# let the kernel share the traffic. Only useful if you 28# let the kernel share the traffic. Only useful if you
29# deliberately run a second tracker there. 29# deliberately run a second tracker there.
@@ -37,6 +37,18 @@
37# listen.udp.workers 4 37# listen.udp.workers 4
38# listen.shared 1 38# listen.shared 1
39# 39#
40# Raising the worker count pays off where the kernel spreads arriving
41# datagrams over a group of sockets bound to one address. Linux does so
42# and FreeBSD has since 12.0, Darwin does not. opentracker asks at start
43# up and binds one socket per worker only where the answer is yes. Where
44# it is no, all workers read the one socket and contend for its receive
45# queue, which costs more than it returns. See the TUNING section of
46# opentracker.conf(5); tests/udp_scaling.py sweeps the setting on the
47# machine that will carry the traffic.
48#
49# A count of 0 answers udp from the event loop, in the thread that also
50# serves tcp, which is what opentracker did before.
51#
40# Note, that using 0.0.0.0 for udp sockets may yield surprising results. 52# Note, that using 0.0.0.0 for udp sockets may yield surprising results.
41# An answer packet sent on that socket will not necessarily have the 53# An answer packet sent on that socket will not necessarily have the
42# source address that the requesting client may expect, but any address 54# source address that the requesting client may expect, but any address
diff --git a/ot_config.c b/ot_config.c
index ee6bc12..a0dbef1 100644
--- a/ot_config.c
+++ b/ot_config.c
@@ -32,7 +32,7 @@ const char *g_statefile;
32static ot_listen *g_listen; 32static ot_listen *g_listen;
33static size_t g_listen_count; 33static size_t g_listen_count;
34static int g_listen_shared; 34static int g_listen_shared;
35static int g_listen_workers; 35static int g_listen_workers = 1;
36 36
37static void panic(const char *routine) __attribute__((noreturn)); 37static void panic(const char *routine) __attribute__((noreturn));
38static void panic(const char *routine) { 38static void panic(const char *routine) {
diff --git a/ot_livesync.c b/ot_livesync.c
index 45271bc..4a0ede1 100644
--- a/ot_livesync.c
+++ b/ot_livesync.c
@@ -8,6 +8,7 @@
8#include <pthread.h> 8#include <pthread.h>
9#include <stdlib.h> 9#include <stdlib.h>
10#include <string.h> 10#include <string.h>
11#include <sys/socket.h>
11#include <sys/types.h> 12#include <sys/types.h>
12#include <unistd.h> 13#include <unistd.h>
13 14
@@ -53,6 +54,8 @@ static sync_buffer g_v6_buf;
53static sync_buffer g_v4_buf; 54static sync_buffer g_v4_buf;
54 55
55static pthread_t thread_id; 56static pthread_t thread_id;
57static int thread_started;
58static _Atomic int thread_left;
56void livesync_init(void) { 59void livesync_init(void) {
57 uint32_t packet_type6 = htonl(OT_SYNC_PEER6); 60 uint32_t packet_type6 = htonl(OT_SYNC_PEER6);
58 uint32_t packet_type4 = htonl(OT_SYNC_PEER4); 61 uint32_t packet_type4 = htonl(OT_SYNC_PEER4);
@@ -73,16 +76,24 @@ void livesync_init(void) {
73 g_v6_buf.next_packet_time = time_future_seconds(LIVESYNC_MAXDELAY); 76 g_v6_buf.next_packet_time = time_future_seconds(LIVESYNC_MAXDELAY);
74 g_v4_buf.next_packet_time = time_future_seconds(LIVESYNC_MAXDELAY); 77 g_v4_buf.next_packet_time = time_future_seconds(LIVESYNC_MAXDELAY);
75 78
76 pthread_create(&thread_id, NULL, livesync_worker, NULL); 79 thread_started = !pthread_create(&thread_id, NULL, livesync_worker, NULL);
77} 80}
78 81
79void livesync_deinit(void) { 82void livesync_deinit(void) {
83 if (thread_started) {
84 /* Repeats so it interrupts a read already under way */
85 while (!atomic_load(&thread_left)) {
86 pthread_kill(thread_id, OT_SIGNAL_WAKE);
87 usleep(1000);
88 }
89 pthread_join(thread_id, NULL);
90 thread_started = 0;
91 }
92
80 if (g_socket_in != -1) 93 if (g_socket_in != -1)
81 close(g_socket_in); 94 close(g_socket_in);
82 if (g_socket_out != -1) 95 if (g_socket_out != -1)
83 close(g_socket_out); 96 close(g_socket_out);
84
85 pthread_cancel(thread_id);
86} 97}
87 98
88void livesync_bind_mcast(ot_ip6 ip, uint16_t port) { 99void livesync_bind_mcast(ot_ip6 ip, uint16_t port) {
@@ -195,7 +206,7 @@ static void *livesync_worker(void *args) {
195 ws.inbuf = ws.request = malloc(LIVESYNC_INCOMING_BUFFSIZE); 206 ws.inbuf = ws.request = malloc(LIVESYNC_INCOMING_BUFFSIZE);
196 ws.outbuf = ws.reply = 0; 207 ws.outbuf = ws.reply = 0;
197 208
198 while (1) { 209 while (g_opentracker_running) {
199 uint32_t packet_type; 210 uint32_t packet_type;
200 211
201 ws.request_size = socket_recv(g_socket_in, ws.inbuf, LIVESYNC_INCOMING_BUFFSIZE, in_ip, &in_port); 212 ws.request_size = socket_recv(g_socket_in, ws.inbuf, LIVESYNC_INCOMING_BUFFSIZE, in_ip, &in_port);
@@ -224,7 +235,8 @@ static void *livesync_worker(void *args) {
224 } 235 }
225 } 236 }
226 237
227 /* Never returns. */ 238 free(ws.inbuf);
239 atomic_store(&thread_left, 1);
228 return NULL; 240 return NULL;
229} 241}
230 242
diff --git a/ot_udp.c b/ot_udp.c
index 87741da..1ec5148 100644
--- a/ot_udp.c
+++ b/ot_udp.c
@@ -9,6 +9,7 @@
9#include <stdio.h> 9#include <stdio.h>
10#include <stdlib.h> 10#include <stdlib.h>
11#include <string.h> 11#include <string.h>
12#include <unistd.h>
12 13
13/* Opentracker */ 14/* Opentracker */
14#include "ot_ip.h" 15#include "ot_ip.h"
@@ -19,6 +20,17 @@
19#include "ot_udp.h" 20#include "ot_udp.h"
20#include "trackerlogic.h" 21#include "trackerlogic.h"
21 22
23/* One of these per worker, each allocated on its own so that growing the
24 array below never moves the one a running worker holds. */
25struct udp_worker {
26 pthread_t thread;
27 int64_t sock;
28 _Atomic int left;
29};
30
31static struct udp_worker **g_workers;
32static size_t g_worker_count;
33
22static uint32_t g_rijndael_round_key[44] = {0}; 34static uint32_t g_rijndael_round_key[44] = {0};
23static volatile uint32_t g_key_of_the_hour[2] = {0}; 35static volatile uint32_t g_key_of_the_hour[2] = {0};
24static volatile ot_time g_hour_of_the_key; 36static volatile ot_time g_hour_of_the_key;
@@ -206,7 +218,7 @@ int handle_udp6(int64_t serversocket, struct ot_workstruct *ws) {
206} 218}
207 219
208static void *udp_worker(void *args) { 220static void *udp_worker(void *args) {
209 int64_t sock = (int64_t)args; 221 struct udp_worker *me = args;
210 struct ot_workstruct ws; 222 struct ot_workstruct ws;
211 memset(&ws, 0, sizeof(ws)); 223 memset(&ws, 0, sizeof(ws));
212 224
@@ -218,7 +230,7 @@ static void *udp_worker(void *args) {
218#endif 230#endif
219 231
220 while (g_opentracker_running) 232 while (g_opentracker_running)
221 handle_udp6(sock, &ws); 233 handle_udp6(me->sock, &ws);
222 234
223 stats_counters_retire(&ws); 235 stats_counters_retire(&ws);
224 236
@@ -227,6 +239,8 @@ static void *udp_worker(void *args) {
227#ifdef _DEBUG_HTTPERROR 239#ifdef _DEBUG_HTTPERROR
228 free(ws.debugbuf); 240 free(ws.debugbuf);
229#endif 241#endif
242
243 atomic_store(&me->left, 1);
230 return NULL; 244 return NULL;
231} 245}
232 246
@@ -236,11 +250,47 @@ void udp_init(void) {
236} 250}
237 251
238void udp_worker_install(int64_t sock) { 252void udp_worker_install(int64_t sock) {
239 pthread_t thread_id; 253 struct udp_worker **grown = realloc(g_workers, (g_worker_count + 1) * sizeof(*g_workers));
254 struct udp_worker *worker;
255
256 if (!grown || !(worker = calloc(1, sizeof(*worker)))) {
257 fprintf(stderr, "Warning: out of memory installing a udp worker, this socket stays with the workers already running.\n");
258 g_workers = grown ? grown : g_workers;
259 return;
260 }
261 g_workers = grown;
262 worker->sock = sock;
240 263
241#ifdef _DEBUG 264#ifdef _DEBUG
242 fprintf(stderr, "Installing a worker on udp socket %ld\n", (unsigned long)sock); 265 fprintf(stderr, "Installing a worker on udp socket %ld\n", (unsigned long)sock);
243#endif 266#endif
244 pthread_create(&thread_id, NULL, udp_worker, (void *)sock); 267 socket_set_blocking((int)sock, 1);
268
269 if (pthread_create(&worker->thread, NULL, udp_worker, worker))
270 free(worker);
271 else
272 g_workers[g_worker_count++] = worker;
273}
274
275void udp_workers_deinit(void) {
276 size_t i;
277
278 for (i = 0; i < g_worker_count; ++i) {
279 struct udp_worker *worker = g_workers[i];
280
281 /* Repeated because a signal only interrupts a read already under way: a
282 worker that was between two reads when the first one arrived is inside
283 the next one by now. */
284 while (!atomic_load(&worker->left)) {
285 pthread_kill(worker->thread, OT_SIGNAL_WAKE);
286 usleep(1000);
287 }
288 pthread_join(worker->thread, NULL);
289 free(worker);
290 }
291
292 free(g_workers);
293 g_workers = NULL;
294 g_worker_count = 0;
245} 295}
246 296
diff --git a/ot_udp.h b/ot_udp.h
index 7d4432f..b731aca 100644
--- a/ot_udp.h
+++ b/ot_udp.h
@@ -13,6 +13,9 @@
13void udp_init(void); 13void udp_init(void);
14/* One thread reading one socket. Called once per worker. */ 14/* One thread reading one socket. Called once per worker. */
15void udp_worker_install(int64_t sock); 15void udp_worker_install(int64_t sock);
16
17/* Stop every worker installed above and wait for it to be gone */
18void udp_workers_deinit(void);
16int handle_udp6(int64_t serversocket, struct ot_workstruct *ws); 19int handle_udp6(int64_t serversocket, struct ot_workstruct *ws);
17 20
18#endif 21#endif
diff --git a/tests/integration/test_shutdown.py b/tests/integration/test_shutdown.py
new file mode 100644
index 0000000..063d779
--- /dev/null
+++ b/tests/integration/test_shutdown.py
@@ -0,0 +1,58 @@
1"""That every thread the tracker starts also ends when it is asked to stop.
2
3The rest of the suite stops a tracker too, so a thread left behind already
4shows up in teardown as the complaint that the tracker had to be killed.
5These ask for the arrangements that park a thread in a blocking read in the
6first place, and they time the exit, because a reader woken only by its own
7receive timeout passes a correctness check while taking a second over it.
8"""
9
10import time
11import unittest
12
13from harness import Tracker, TrackerTestCase
14
15# Generous enough for a platform where shutting the socket down does not wake
16# the reader and the receive timeout has to expire, tight enough that a reader
17# which is never woken at all fails instead of hanging the suite
18PATIENCE = 3.0
19
20
21def time_the_exit(extra_config):
22 tracker = Tracker(extra_config)
23 try:
24 # Serve one request of each kind first, so the threads that are about
25 # to be joined have actually run
26 tracker.announce(b"shutdown-probe\0\0\0\0\0\0")
27 sock, connection_id = tracker.udp_connect()
28 tracker.udp_announce(sock, connection_id, b"shutdown-probe\0\0\0\0\0\0")
29 sock.close()
30 started = time.time()
31 complaints = tracker.stop()
32 return time.time() - started, complaints
33 finally:
34 tracker._cleanup()
35
36
37class UdpWorkers(TrackerTestCase):
38 config = "listen.udp.workers 8\n"
39
40 def test_announce_reaches_one_of_eight_workers(self):
41 """Keeps the workers from being merely started and never used."""
42 sock, connection_id = self.tracker.udp_connect()
43 answer = self.tracker.udp_announce(sock, connection_id, self.hash_for("eight-workers"))
44 sock.close()
45 self.assertEqual(answer["seeders"], 1)
46
47
48class ShutdownEndsEveryThread(unittest.TestCase):
49 def test_eight_udp_workers_are_joined(self):
50 took, complaints = time_the_exit("listen.udp.workers 8\n")
51 self.assertEqual(complaints, [], "the tracker reported:\n" + "\n".join(complaints))
52 self.assertLess(took, PATIENCE, "stopping eight udp workers took %.1f s" % took)
53
54 def test_udp_in_the_event_loop_still_stops(self):
55 """Worker count zero leaves no thread to join, which must also work."""
56 took, complaints = time_the_exit("listen.udp.workers 0\n")
57 self.assertEqual(complaints, [], "the tracker reported:\n" + "\n".join(complaints))
58 self.assertLess(took, PATIENCE, "stopping without workers took %.1f s" % took)
diff --git a/tests/t_fdstate.c b/tests/t_fdstate.c
index bf8724a..421f076 100644
--- a/tests/t_fdstate.c
+++ b/tests/t_fdstate.c
@@ -101,6 +101,8 @@ static void case_out_of_range(void) {
101#define RECYCLE_AFTER 32 101#define RECYCLE_AFTER 32
102#define STRESS_CLAIMS 200000 102#define STRESS_CLAIMS 200000
103#define STRESS_GIVEUP (STRESS_CLAIMS * 100) 103#define STRESS_GIVEUP (STRESS_CLAIMS * 100)
104/* What the run has to reach for its result to mean anything */
105#define STRESS_CYCLES (STRESS_CLAIMS / (RECYCLE_AFTER * 4))
104 106
105static _Atomic long stress_claims, stress_recycles, stress_violations, stress_attempts; 107static _Atomic long stress_claims, stress_recycles, stress_violations, stress_attempts;
106static _Atomic int stress_lives[STRESS_SLOTS]; 108static _Atomic int stress_lives[STRESS_SLOTS];
@@ -116,8 +118,14 @@ static _Atomic uint64_t stress_next_cookie;
116#define STRESS_CONN_COOKIE(w) ((void *)(uintptr_t)((w) >> 32)) 118#define STRESS_CONN_COOKIE(w) ((void *)(uintptr_t)((w) >> 32))
117#define STRESS_CONN_GEN(w) ((uint32_t)(w)) 119#define STRESS_CONN_GEN(w) ((uint32_t)(w))
118 120
121/* The workers reach their claim target on their own, but the recycles that
122 make the run worth anything are the keeper's, and on a loaded machine the
123 keeper is the thread that loses. So the run goes on until both targets are
124 met rather than until the faster of the two is. */
119static int stress_running(void) { 125static int stress_running(void) {
120 return atomic_load(&stress_claims) < STRESS_CLAIMS && atomic_load(&stress_attempts) < STRESS_GIVEUP; 126 if (atomic_load(&stress_attempts) >= STRESS_GIVEUP)
127 return 0;
128 return atomic_load(&stress_claims) < STRESS_CLAIMS || atomic_load(&stress_recycles) <= STRESS_CYCLES;
121} 129}
122 130
123static void *stress_worker(void *arg) { 131static void *stress_worker(void *arg) {
@@ -203,7 +211,7 @@ static void case_recycling_under_load(void) {
203 211
204 test_case("and the run actually recycled descriptors, or it proved nothing"); 212 test_case("and the run actually recycled descriptors, or it proved nothing");
205 CHECK(atomic_load(&stress_claims) >= STRESS_CLAIMS); 213 CHECK(atomic_load(&stress_claims) >= STRESS_CLAIMS);
206 CHECK(atomic_load(&stress_recycles) > STRESS_CLAIMS / (RECYCLE_AFTER * 4)); 214 CHECK(atomic_load(&stress_recycles) > STRESS_CYCLES);
207 215
208 for (i = 0; i < STRESS_SLOTS; ++i) 216 for (i = 0; i < STRESS_SLOTS; ++i)
209 fdstate_close((int)i); 217 fdstate_close((int)i);
diff --git a/tests/t_socket.c b/tests/t_socket.c
index 8eef7fd..ab4b785 100644
--- a/tests/t_socket.c
+++ b/tests/t_socket.c
@@ -8,13 +8,18 @@
8#include <fcntl.h> 8#include <fcntl.h>
9#include <netinet/in.h> 9#include <netinet/in.h>
10#include <poll.h> 10#include <poll.h>
11#include <pthread.h>
12#include <signal.h>
13#include <stdatomic.h>
11#include <string.h> 14#include <string.h>
12#include <sys/socket.h> 15#include <sys/socket.h>
16#include <sys/time.h>
13#include <unistd.h> 17#include <unistd.h>
14 18
15/* Opentracker */ 19/* Opentracker */
16#include "ot_ip.h" 20#include "ot_ip.h"
17#include "ot_socket.h" 21#include "ot_socket.h"
22#include "trackerlogic.h"
18#include "test.h" 23#include "test.h"
19 24
20/* These cases talk to the loopback interface rather than to a mock, because 25/* These cases talk to the loopback interface rather than to a mock, because
@@ -144,6 +149,57 @@ static void case_udp_roundtrip(void) {
144 close(s); 149 close(s);
145} 150}
146 151
152/* udp_workers_deinit and livesync_deinit end a read another thread is parked
153 in by signalling that thread. Which mechanisms wake a blocked reader
154 differs between kernels -- shutting the socket down does so on Linux and
155 FreeBSD but not on Darwin, closing it does so on Darwin but not on Linux --
156 so the one the tracker relies on is checked rather than assumed. The
157 watchdog is the test's own: where a reader is not woken this fails with a
158 figure to look at instead of hanging the suite. */
159static _Atomic int reader_returned;
160static int reader_socket;
161
162static void reader_wake(int s) { (void)s; }
163
164static void *blocked_reader(void *args) {
165 char buf[32], ip[16];
166 uint16_t port = 0;
167
168 (void)args;
169 socket_recv(reader_socket, buf, sizeof(buf), ip, &port);
170 atomic_store(&reader_returned, 1);
171 return NULL;
172}
173
174static void case_a_signal_ends_a_blocking_read(void) {
175 struct sigaction sa;
176 struct timeval watchdog = {2, 0};
177 pthread_t thread;
178 int waited;
179
180 test_case("a signal ends a read another thread is parked in");
181 memset(&sa, 0, sizeof(sa));
182 sa.sa_handler = reader_wake;
183 sa.sa_flags = 0; /* SA_RESTART would resume the read instead of ending it */
184 CHECK_INT(sigaction(OT_SIGNAL_WAKE, &sa, NULL), 0);
185
186 atomic_store(&reader_returned, 0);
187 reader_socket = socket_udp_bind(ip_loopback, 0, SOCKET_EXCLUSIVE);
188 CHECK(reader_socket != -1);
189 CHECK_INT(socket_set_blocking(reader_socket, 1), 0);
190 CHECK_INT(setsockopt(reader_socket, SOL_SOCKET, SO_RCVTIMEO, &watchdog, sizeof(watchdog)), 0);
191
192 CHECK_INT(pthread_create(&thread, NULL, blocked_reader, NULL), 0);
193 usleep(100000); /* long enough to be sure it reached the read */
194 CHECK_INT(pthread_kill(thread, OT_SIGNAL_WAKE), 0);
195
196 for (waited = 0; waited < 1000 && !atomic_load(&reader_returned); ++waited)
197 usleep(1000);
198 CHECK(waited < 1000);
199 pthread_join(thread, NULL);
200 close(reader_socket);
201}
202
147/* An IPv4 peer must arrive in the v4 mapped form opentracker stores, however 203/* An IPv4 peer must arrive in the v4 mapped form opentracker stores, however
148 the kernel chose to report it. Linux fills a v4 mapped sockaddr_in6 here 204 the kernel chose to report it. Linux fills a v4 mapped sockaddr_in6 here
149 and the BSDs fill a plain sockaddr_in, so this is the case that tells the 205 and the BSDs fill a plain sockaddr_in, so this is the case that tells the
@@ -293,6 +349,7 @@ void test_socket(void) {
293 case_sharing_is_opt_in(); 349 case_sharing_is_opt_in();
294#endif 350#endif
295 case_udp_roundtrip(); 351 case_udp_roundtrip();
352 case_a_signal_ends_a_blocking_read();
296 case_udp_v4mapped(); 353 case_udp_v4mapped();
297 case_tcp_roundtrip(); 354 case_tcp_roundtrip();
298 case_connect_refused(); 355 case_connect_refused();
diff --git a/tests/udp_wake_probe.c b/tests/udp_wake_probe.c
index 8a0f7a8..e948c34 100644
--- a/tests/udp_wake_probe.c
+++ b/tests/udp_wake_probe.c
@@ -1,13 +1,17 @@
1/* Three ways to get a thread out of a blocking recvfrom, tested rather than 1/* Four ways to get a thread out of a blocking recvfrom, tested rather than
2 assumed: close the socket, shut it down, or give it a receive timeout. 2 assumed: close the socket, shut it down, give it a receive timeout, or
3 send it a signal.
3 4
4 This decides how udp workers can be stopped. They sit in recvfrom and are 5 This decides how udp workers are stopped. The sockets of a SO_REUSEPORT
5 never joined today, and the sockets of a SO_REUSEPORT group all share one 6 group all share one address, so a worker cannot be singled out by sending
6 address, so a worker cannot be singled out by sending it a datagram. 7 it a datagram, and the two mechanisms differ in what they cost while the
8 tracker is running: a receive timeout wakes every worker on every tick
9 forever, a signal and a shutdown cost nothing until the one moment they
10 are used.
7 11
8 cc -O2 -pthread -o /tmp/wake tests/udp_wake_probe.c && /tmp/wake 12 cc -O2 -pthread -o /tmp/wake tests/udp_wake_probe.c && /tmp/wake
9 13
10 On Linux 6.1: close does not wake it, shutdown does, SO_RCVTIMEO does. */ 14 On Linux 6.1: close does not wake it, the other three do. */
11#include <errno.h> 15#include <errno.h>
12#include <netinet/in.h> 16#include <netinet/in.h>
13#include <pthread.h> 17#include <pthread.h>
@@ -15,9 +19,12 @@
15#include <stdio.h> 19#include <stdio.h>
16#include <string.h> 20#include <string.h>
17#include <sys/socket.h> 21#include <sys/socket.h>
22#include <signal.h>
18#include <sys/time.h> 23#include <sys/time.h>
19#include <unistd.h> 24#include <unistd.h>
20 25
26#define WAKE_SIGNAL SIGUSR1
27
21static _Atomic int returned; 28static _Atomic int returned;
22static _Atomic int saved_errno; 29static _Atomic int saved_errno;
23static int sock; 30static int sock;
@@ -31,6 +38,11 @@ static void *reader(void *args) {
31 return NULL; 38 return NULL;
32} 39}
33 40
41/* A handler that does nothing is all it takes: the point is the return from
42 recvfrom with EINTR, not anything the handler does. SA_RESTART must stay
43 off or the kernel resumes the call instead of reporting it. */
44static void noop(int s) { (void)s; }
45
34static int bind_udp(int timeout_ms) { 46static int bind_udp(int timeout_ms) {
35 struct sockaddr_in address; 47 struct sockaddr_in address;
36 int s = socket(AF_INET, SOCK_DGRAM, 0); 48 int s = socket(AF_INET, SOCK_DGRAM, 0);
@@ -50,7 +62,7 @@ static int bind_udp(int timeout_ms) {
50 return s; 62 return s;
51} 63}
52 64
53static void try_it(const char *name, int timeout_ms, int do_close, int do_shutdown) { 65static void try_it(const char *name, int timeout_ms, int do_close, int do_shutdown, int do_signal) {
54 pthread_t thread; 66 pthread_t thread;
55 int waited; 67 int waited;
56 68
@@ -62,6 +74,8 @@ static void try_it(const char *name, int timeout_ms, int do_close, int do_shutdo
62 74
63 if (do_shutdown) 75 if (do_shutdown)
64 shutdown(sock, SHUT_RD); 76 shutdown(sock, SHUT_RD);
77 if (do_signal)
78 pthread_kill(thread, WAKE_SIGNAL);
65 if (do_close) 79 if (do_close)
66 close(sock); 80 close(sock);
67 81
@@ -92,8 +106,16 @@ static void try_it(const char *name, int timeout_ms, int do_close, int do_shutdo
92} 106}
93 107
94int main(void) { 108int main(void) {
95 try_it("close() the socket", 0, 1, 0); 109 struct sigaction sa;
96 try_it("shutdown(SHUT_RD)", 0, 0, 1); 110
97 try_it("SO_RCVTIMEO 500 ms", 500, 0, 0); 111 memset(&sa, 0, sizeof(sa));
112 sa.sa_handler = noop;
113 sa.sa_flags = 0;
114 sigaction(WAKE_SIGNAL, &sa, NULL);
115
116 try_it("close() the socket", 0, 1, 0, 0);
117 try_it("shutdown(SHUT_RD)", 0, 0, 1, 0);
118 try_it("SO_RCVTIMEO 500 ms", 500, 0, 0, 0);
119 try_it("pthread_kill(SIGUSR1)", 0, 0, 0, 1);
98 return 0; 120 return 0;
99} 121}
diff --git a/trackerlogic.h b/trackerlogic.h
index ad6c911..7d0693a 100644
--- a/trackerlogic.h
+++ b/trackerlogic.h
@@ -6,6 +6,7 @@
6#ifndef OT_TRACKERLOGIC_H__ 6#ifndef OT_TRACKERLOGIC_H__
7#define OT_TRACKERLOGIC_H__ 7#define OT_TRACKERLOGIC_H__
8 8
9#include <signal.h>
9#include <stdatomic.h> 10#include <stdatomic.h>
10#include <stdint.h> 11#include <stdint.h>
11#include <stdlib.h> 12#include <stdlib.h>
@@ -90,6 +91,9 @@ static inline ot_time time_future_seconds(int span) { return time_now_seconds()
90/* Cleared from the interrupt handler and read by every worker */ 91/* Cleared from the interrupt handler and read by every worker */
91extern _Atomic int g_opentracker_running; 92extern _Atomic int g_opentracker_running;
92 93
94/* Sent to a thread parked in a blocking read */
95#define OT_SIGNAL_WAKE SIGUSR1
96
93 97
94extern uint32_t g_tracker_id; 98extern uint32_t g_tracker_id;
95typedef enum { FLAG_TCP, FLAG_UDP, FLAG_MCA, FLAG_SELFPIPE } PROTO_FLAG; 99typedef enum { FLAG_TCP, FLAG_UDP, FLAG_MCA, FLAG_SELFPIPE } PROTO_FLAG;