From ad541c67761b8aa6da1566802c347cac4018d0c2 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Mon, 2 Nov 2026 20:50:24 +0100 Subject: Move the build sources into src/ --- DESIGN | 2 - Makefile | 6 +- Mk/proxy.mk | 2 +- Mk/tests.mk | 4 +- opentracker.c | 443 -------------------------- ot_accesslist.c | 601 ------------------------------------ ot_accesslist.h | 77 ----- ot_batch.c | 156 ---------- ot_batch.h | 46 --- ot_buf.c | 39 --- ot_buf.h | 25 -- ot_clean.c | 176 ----------- ot_clean.h | 28 -- ot_config.c | 586 ----------------------------------- ot_config.h | 33 -- ot_event.c | 357 --------------------- ot_event.h | 47 --- ot_fdstate.c | 202 ------------ ot_fdstate.h | 62 ---- ot_fmt.c | 40 --- ot_fmt.h | 19 -- ot_fullscrape.c | 455 --------------------------- ot_fullscrape.h | 23 -- ot_hex.c | 56 ---- ot_hex.h | 27 -- ot_http.c | 810 ------------------------------------------------ ot_http.h | 53 ---- ot_iovec.c | 87 ------ ot_iovec.h | 18 -- ot_ip.c | 199 ------------ ot_ip.h | 31 -- ot_livesync.c | 241 --------------- ot_livesync.h | 75 ----- ot_mutex.c | 70 ----- ot_mutex.h | 21 -- ot_random.c | 51 --- ot_random.h | 17 - ot_rijndael.c | 488 ----------------------------- ot_rijndael.h | 20 -- ot_scan.c | 65 ---- ot_scan.h | 25 -- ot_socket.c | 310 ------------------- ot_socket.h | 56 ---- ot_stats.c | 866 --------------------------------------------------- ot_stats.h | 40 --- ot_tcp.c | 180 ----------- ot_tcp.h | 24 -- ot_udp.c | 226 -------------- ot_udp.h | 17 - ot_urlencoded.c | 168 ---------- ot_urlencoded.h | 53 ---- ot_vector.c | 287 ----------------- ot_vector.h | 39 --- ot_worker.c | 73 ----- ot_worker.h | 37 --- ot_workqueue.c | 241 --------------- ot_workqueue.h | 74 ----- proxy.c | 871 ---------------------------------------------------- src/opentracker.c | 443 ++++++++++++++++++++++++++ src/ot_accesslist.c | 601 ++++++++++++++++++++++++++++++++++++ src/ot_accesslist.h | 77 +++++ src/ot_batch.c | 156 ++++++++++ src/ot_batch.h | 46 +++ src/ot_buf.c | 39 +++ src/ot_buf.h | 25 ++ src/ot_clean.c | 176 +++++++++++ src/ot_clean.h | 28 ++ src/ot_config.c | 586 +++++++++++++++++++++++++++++++++++ src/ot_config.h | 33 ++ src/ot_event.c | 357 +++++++++++++++++++++ src/ot_event.h | 47 +++ src/ot_fdstate.c | 202 ++++++++++++ src/ot_fdstate.h | 62 ++++ src/ot_fmt.c | 40 +++ src/ot_fmt.h | 19 ++ src/ot_fullscrape.c | 455 +++++++++++++++++++++++++++ src/ot_fullscrape.h | 23 ++ src/ot_hex.c | 56 ++++ src/ot_hex.h | 27 ++ src/ot_http.c | 810 ++++++++++++++++++++++++++++++++++++++++++++++++ src/ot_http.h | 53 ++++ src/ot_iovec.c | 87 ++++++ src/ot_iovec.h | 18 ++ src/ot_ip.c | 199 ++++++++++++ src/ot_ip.h | 31 ++ src/ot_livesync.c | 241 +++++++++++++++ src/ot_livesync.h | 75 +++++ src/ot_mutex.c | 70 +++++ src/ot_mutex.h | 21 ++ src/ot_random.c | 51 +++ src/ot_random.h | 17 + src/ot_rijndael.c | 488 +++++++++++++++++++++++++++++ src/ot_rijndael.h | 20 ++ src/ot_scan.c | 65 ++++ src/ot_scan.h | 25 ++ src/ot_socket.c | 310 +++++++++++++++++++ src/ot_socket.h | 56 ++++ src/ot_stats.c | 866 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/ot_stats.h | 40 +++ src/ot_tcp.c | 180 +++++++++++ src/ot_tcp.h | 24 ++ src/ot_udp.c | 226 ++++++++++++++ src/ot_udp.h | 17 + src/ot_urlencoded.c | 168 ++++++++++ src/ot_urlencoded.h | 53 ++++ src/ot_vector.c | 287 +++++++++++++++++ src/ot_vector.h | 39 +++ src/ot_worker.c | 73 +++++ src/ot_worker.h | 37 +++ src/ot_workqueue.c | 241 +++++++++++++++ src/ot_workqueue.h | 74 +++++ src/proxy.c | 871 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/trackerlogic.c | 628 +++++++++++++++++++++++++++++++++++++ src/trackerlogic.h | 283 +++++++++++++++++ trackerlogic.c | 628 ------------------------------------- trackerlogic.h | 283 ----------------- 116 files changed, 10248 insertions(+), 10250 deletions(-) delete mode 100644 opentracker.c delete mode 100644 ot_accesslist.c delete mode 100644 ot_accesslist.h delete mode 100644 ot_batch.c delete mode 100644 ot_batch.h delete mode 100644 ot_buf.c delete mode 100644 ot_buf.h delete mode 100644 ot_clean.c delete mode 100644 ot_clean.h delete mode 100644 ot_config.c delete mode 100644 ot_config.h delete mode 100644 ot_event.c delete mode 100644 ot_event.h delete mode 100644 ot_fdstate.c delete mode 100644 ot_fdstate.h delete mode 100644 ot_fmt.c delete mode 100644 ot_fmt.h delete mode 100644 ot_fullscrape.c delete mode 100644 ot_fullscrape.h delete mode 100644 ot_hex.c delete mode 100644 ot_hex.h delete mode 100644 ot_http.c delete mode 100644 ot_http.h delete mode 100644 ot_iovec.c delete mode 100644 ot_iovec.h delete mode 100644 ot_ip.c delete mode 100644 ot_ip.h delete mode 100644 ot_livesync.c delete mode 100644 ot_livesync.h delete mode 100644 ot_mutex.c delete mode 100644 ot_mutex.h delete mode 100644 ot_random.c delete mode 100644 ot_random.h delete mode 100644 ot_rijndael.c delete mode 100644 ot_rijndael.h delete mode 100644 ot_scan.c delete mode 100644 ot_scan.h delete mode 100644 ot_socket.c delete mode 100644 ot_socket.h delete mode 100644 ot_stats.c delete mode 100644 ot_stats.h delete mode 100644 ot_tcp.c delete mode 100644 ot_tcp.h delete mode 100644 ot_udp.c delete mode 100644 ot_udp.h delete mode 100644 ot_urlencoded.c delete mode 100644 ot_urlencoded.h delete mode 100644 ot_vector.c delete mode 100644 ot_vector.h delete mode 100644 ot_worker.c delete mode 100644 ot_worker.h delete mode 100644 ot_workqueue.c delete mode 100644 ot_workqueue.h delete mode 100644 proxy.c create mode 100644 src/opentracker.c create mode 100644 src/ot_accesslist.c create mode 100644 src/ot_accesslist.h create mode 100644 src/ot_batch.c create mode 100644 src/ot_batch.h create mode 100644 src/ot_buf.c create mode 100644 src/ot_buf.h create mode 100644 src/ot_clean.c create mode 100644 src/ot_clean.h create mode 100644 src/ot_config.c create mode 100644 src/ot_config.h create mode 100644 src/ot_event.c create mode 100644 src/ot_event.h create mode 100644 src/ot_fdstate.c create mode 100644 src/ot_fdstate.h create mode 100644 src/ot_fmt.c create mode 100644 src/ot_fmt.h create mode 100644 src/ot_fullscrape.c create mode 100644 src/ot_fullscrape.h create mode 100644 src/ot_hex.c create mode 100644 src/ot_hex.h create mode 100644 src/ot_http.c create mode 100644 src/ot_http.h create mode 100644 src/ot_iovec.c create mode 100644 src/ot_iovec.h create mode 100644 src/ot_ip.c create mode 100644 src/ot_ip.h create mode 100644 src/ot_livesync.c create mode 100644 src/ot_livesync.h create mode 100644 src/ot_mutex.c create mode 100644 src/ot_mutex.h create mode 100644 src/ot_random.c create mode 100644 src/ot_random.h create mode 100644 src/ot_rijndael.c create mode 100644 src/ot_rijndael.h create mode 100644 src/ot_scan.c create mode 100644 src/ot_scan.h create mode 100644 src/ot_socket.c create mode 100644 src/ot_socket.h create mode 100644 src/ot_stats.c create mode 100644 src/ot_stats.h create mode 100644 src/ot_tcp.c create mode 100644 src/ot_tcp.h create mode 100644 src/ot_udp.c create mode 100644 src/ot_udp.h create mode 100644 src/ot_urlencoded.c create mode 100644 src/ot_urlencoded.h create mode 100644 src/ot_vector.c create mode 100644 src/ot_vector.h create mode 100644 src/ot_worker.c create mode 100644 src/ot_worker.h create mode 100644 src/ot_workqueue.c create mode 100644 src/ot_workqueue.h create mode 100644 src/proxy.c create mode 100644 src/trackerlogic.c create mode 100644 src/trackerlogic.h delete mode 100644 trackerlogic.c delete mode 100644 trackerlogic.h diff --git a/DESIGN b/DESIGN index 5abeb4a..f058fe0 100644 --- a/DESIGN +++ b/DESIGN @@ -114,8 +114,6 @@ A directory for object files. bmake has no pattern rules and the Makefile builds under FreeBSD's make today. Building each binary in one invocation clears the objects out of the tree without giving that up. -A src/ directory. 52 files sharing an ot_ prefix are a module list rather -than clutter, and build output was most of what made the tree look crowded. Platform differences that cost us code diff --git a/Makefile b/Makefile index 993f0ae..d2e0c2d 100644 --- a/Makefile +++ b/Makefile @@ -49,8 +49,8 @@ COMPILE=$(CPPFLAGS) $(OT_CFLAGS) $(FEATURES) LINK=$(LDFLAGS) $(OT_LDFLAGS) $(LDLIBS) $(OT_LDLIBS) BINARY=opentracker -HEADERS=trackerlogic.h ot_urlencoded.h ot_batch.h ot_buf.h ot_event.h ot_fdstate.h ot_fmt.h ot_hex.h ot_ip.h ot_config.h ot_scan.h ot_socket.h ot_tcp.h ot_worker.h ot_workqueue.h ot_mutex.h ot_random.h ot_stats.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h ot_rijndael.h -SOURCES=opentracker.c trackerlogic.c ot_urlencoded.c ot_batch.c ot_buf.c ot_event.c ot_fdstate.c ot_fmt.c ot_hex.c ot_ip.c ot_config.c ot_scan.c ot_socket.c ot_tcp.c ot_worker.c ot_workqueue.c ot_mutex.c ot_random.c ot_stats.c ot_vector.c ot_clean.c ot_udp.c ot_iovec.c ot_fullscrape.c ot_accesslist.c ot_http.c ot_livesync.c ot_rijndael.c +HEADERS=src/trackerlogic.h src/ot_urlencoded.h src/ot_batch.h src/ot_buf.h src/ot_event.h src/ot_fdstate.h src/ot_fmt.h src/ot_hex.h src/ot_ip.h src/ot_config.h src/ot_scan.h src/ot_socket.h src/ot_tcp.h src/ot_worker.h src/ot_workqueue.h src/ot_mutex.h src/ot_random.h src/ot_stats.h src/ot_vector.h src/ot_clean.h src/ot_udp.h src/ot_iovec.h src/ot_fullscrape.h src/ot_accesslist.h src/ot_http.h src/ot_livesync.h src/ot_rijndael.h +SOURCES=src/opentracker.c src/trackerlogic.c src/ot_urlencoded.c src/ot_batch.c src/ot_buf.c src/ot_event.c src/ot_fdstate.c src/ot_fmt.c src/ot_hex.c src/ot_ip.c src/ot_config.c src/ot_scan.c src/ot_socket.c src/ot_tcp.c src/ot_worker.c src/ot_workqueue.c src/ot_mutex.c src/ot_random.c src/ot_stats.c src/ot_vector.c src/ot_clean.c src/ot_udp.c src/ot_iovec.c src/ot_fullscrape.c src/ot_accesslist.c src/ot_http.c src/ot_livesync.c src/ot_rijndael.c BINARIES=$(BINARY) $(BINARY).debug # Everything is built in one go from sources, so there are no object files to @@ -73,7 +73,7 @@ $(BUILDSTAMP): FORCE @echo "$(CFLAGS) $(COMPILE) $(LINK)" | cmp -s - $@ 2>/dev/null || echo "$(CFLAGS) $(COMPILE) $(LINK)" > $@ clean: - rm -rf $(BINARIES) $(BINARIES:=.dSYM) $(BUILDSTAMP) *.o *~ + rm -rf $(BINARIES) $(BINARIES:=.dSYM) $(BUILDSTAMP) *.o *~ src/*.o src/*~ install: $(BINARY) install -d $(DESTDIR)$(BINDIR) $(DESTDIR)$(MANDIR)/man1 $(DESTDIR)$(MANDIR)/man5 diff --git a/Mk/proxy.mk b/Mk/proxy.mk index 2e876e9..64d18dc 100644 --- a/Mk/proxy.mk +++ b/Mk/proxy.mk @@ -7,7 +7,7 @@ # CFLAGS_libowfat=-I$(LIBOWFAT_HEADERS) # LDFLAGS_libowfat=-L$(LIBOWFAT_LIBRARY) -lowfat # -# SOURCES_proxy=proxy.c ot_batch.c ot_fdstate.c ot_vector.c ot_mutex.c ot_random.c ot_scan.c ot_ip.c ot_socket.c +# SOURCES_proxy=src/proxy.c src/ot_batch.c src/ot_fdstate.c src/ot_vector.c src/ot_mutex.c src/ot_random.c src/ot_scan.c src/ot_ip.c src/ot_socket.c # CFLAGS_proxy=$(CFLAGS_production) $(CFLAGS_libowfat) -DWANT_SYNC_LIVE # CFLAGS_proxy_debug=$(CFLAGS_debug) $(CFLAGS_libowfat) -DWANT_SYNC_LIVE # diff --git a/Mk/tests.mk b/Mk/tests.mk index 8e63839..46f308b 100644 --- a/Mk/tests.mk +++ b/Mk/tests.mk @@ -5,13 +5,13 @@ BUILDDEPS+=Mk/tests.mk # the few symbols those modules reference but the tests never reach. TESTBIN=tests/runner TEST_SOURCES=tests/runner.c tests/test.c tests/stubs.c tests/t_urlencoded.c tests/t_iovec.c tests/t_vector.c tests/t_random.c tests/t_hex.c tests/t_accesslist.c tests/t_scan.c tests/t_buf.c tests/t_ip.c tests/t_socket.c tests/t_mutex.c tests/t_trackerlogic.c tests/t_workqueue.c tests/t_batch.c tests/t_fdstate.c tests/t_event.c tests/t_fmt.c -TEST_UNITS=ot_urlencoded.c ot_fmt.c ot_event.c ot_batch.c ot_fdstate.c ot_buf.c ot_ip.c ot_socket.c ot_iovec.c ot_vector.c ot_random.c ot_hex.c ot_scan.c ot_accesslist.c ot_mutex.c ot_workqueue.c trackerlogic.c +TEST_UNITS=src/ot_urlencoded.c src/ot_fmt.c src/ot_event.c src/ot_batch.c src/ot_fdstate.c src/ot_buf.c src/ot_ip.c src/ot_socket.c src/ot_iovec.c src/ot_vector.c src/ot_random.c src/ot_hex.c src/ot_scan.c src/ot_accesslist.c src/ot_mutex.c src/ot_workqueue.c src/trackerlogic.c TEST_HEADERS=tests/test.h $(HEADERS) # A test binary is not a production binary, so it is built with the features # the tests need to reach. FEATURES_test=$(FEATURES) -DWANT_ACCESSLIST_BLACK -DWANT_SYNC_LIVE -CFLAGS_test=-I. -g -O1 $(CPPFLAGS) $(OT_CFLAGS) $(FEATURES_test) +CFLAGS_test=-Isrc -g -O1 $(CPPFLAGS) $(OT_CFLAGS) $(FEATURES_test) # clang over $(CC) because the two toolchains do not check the same things. CC_sanitize?=clang diff --git a/opentracker.c b/opentracker.c deleted file mode 100644 index 14b817e..0000000 --- a/opentracker.c +++ /dev/null @@ -1,443 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. - Some of the stuff below is stolen from Fefes example libowfat httpd. */ - -/* System */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef WANT_SYSLOGS -#include -#endif - -/* Opentracker */ -#include "ot_accesslist.h" -#include "ot_config.h" -#include "ot_hex.h" -#include "ot_http.h" -#include "ot_ip.h" -#include "ot_livesync.h" -#include "ot_mutex.h" -#include "ot_workqueue.h" -#include "ot_random.h" -#include "ot_scan.h" -#include "ot_event.h" -#include "ot_fdstate.h" -#include "ot_socket.h" -#include "ot_stats.h" -#include "ot_tcp.h" -#include "ot_udp.h" -#include "ot_worker.h" -#include "trackerlogic.h" - -/* Globals */ -uint32_t g_tracker_id; -_Atomic int g_opentracker_running = 1; -int g_self_pipe[2]; - -static void panic(const char *routine) __attribute__((noreturn)); -static void panic(const char *routine) { - fprintf(stderr, "%s: %s\n", routine, strerror(errno)); - exit(111); -} - -/* Say we are stopping, wake the loop, and let main do the work on its own - stack. Everything other than SIGINT falls through and does nothing, which - is what OT_SIGNAL_WAKE wants: returning from the read is all it is for. */ -static void signal_handler(int s) { - if (s == SIGINT) { - const char byte = 0; - int saved = errno; - - /* Any new interrupt signal quits the application */ - signal(SIGINT, SIG_DFL); - - /* Tell all other threads to not acquire any new lock on a bucket - but cancel their operations and return */ - g_opentracker_running = 0; - - (void)!write(g_self_pipe[1], &byte, 1); - errno = saved; - } -} - -static void defaul_signal_handlers(void) { - sigset_t signal_mask; - sigemptyset(&signal_mask); - sigaddset(&signal_mask, SIGPIPE); - sigaddset(&signal_mask, SIGHUP); - sigaddset(&signal_mask, SIGINT); - sigaddset(&signal_mask, SIGALRM); - pthread_sigmask(SIG_BLOCK, &signal_mask, NULL); -} - -static void install_signal_handlers(void) { - struct sigaction sa; - sigset_t signal_mask; - sigemptyset(&signal_mask); - - sa.sa_handler = signal_handler; - sigemptyset(&sa.sa_mask); - sa.sa_flags = 0; - if ((sigaction(SIGINT, &sa, NULL) == -1) || (sigaction(SIGALRM, &sa, NULL) == -1) || - (sigaction(OT_SIGNAL_WAKE, &sa, NULL) == -1)) - panic("install_signal_handlers"); - - sigaddset(&signal_mask, SIGINT); - pthread_sigmask(SIG_UNBLOCK, &signal_mask, NULL); -} - - -/* How many ready descriptors one wait hands back at a time */ -#define OT_EVENT_BATCH 512 - -static void *server_mainloop(void *args) { - struct ot_workstruct ws; - time_t next_timeout_check = time_future_seconds(OT_CLIENT_TIMEOUT_CHECKINTERVAL); - struct iovec *iovector; - size_t iovec_entries; - int is_partial; - - (void)args; - - if (workstruct_init(&ws) == -1) - panic("Initializing worker failed"); - - while (g_opentracker_running) { - ot_event ready[OT_EVENT_BATCH]; - int64_t sock; - int count, i; - - count = event_wait(ready, OT_EVENT_BATCH, OT_CLIENT_TIMEOUT_CHECKINTERVAL * 1000); - - for (i = 0; i < count; ++i) { - const void *cookie; - - sock = ready[i].fd; - /* A descriptor closed earlier in this same batch may already have been - handed out again by accept, so an event is only ours if the - generation it was registered under is still the one in place */ - if (fdstate_generation((int)sock) != ready[i].generation) - continue; - cookie = fdstate_cookie((int)sock); - - if ((intptr_t)cookie == FLAG_TCP) - tcp_accept(sock, &ws); - else if ((intptr_t)cookie == FLAG_UDP) - handle_udp6(sock, &ws); - else if ((intptr_t)cookie == FLAG_SELFPIPE) - while (read((int)sock, ws.inbuf, G_INBUF_SIZE) > 0) - ; - else if (fdstate_claim((int)sock, ready[i].generation)) { - if (ready[i].what & EVENT_IN) - tcp_read(sock, &ws); - /* The read may have closed it, so look again before writing */ - if ((ready[i].what & EVENT_OUT) && fdstate_generation((int)sock) == ready[i].generation) - tcp_write(sock); - if (fdstate_generation((int)sock) == ready[i].generation) - fdstate_release((int)sock); - } - } - - while ((sock = workqueue_popresult(&iovec_entries, &iovector, &is_partial)) != -1) - http_sendiovecdata(sock, &ws, iovec_entries, iovector, is_partial); - - if (time_now_seconds() > next_timeout_check) { - int cursor = 0; - - while ((sock = fdstate_next_expired(time_now_seconds(), &cursor)) != -1) - http_close(sock); - next_timeout_check = time_future_seconds(OT_CLIENT_TIMEOUT_CHECKINTERVAL); - } - - livesync_ticker(); - } - - /* This work struct is about to go out of scope, so hand its tally over - before anything can read it there */ - workstruct_deinit(&ws); - return 0; -} - - -/* A group of sockets sharing one address only spreads datagrams if every - member asked for it before binding, the first one included, so whether - this entry will have workers has to be known here. */ -static socket_sharing listen_sharing_for(const ot_listen *entry) { - if (entry->proto != FLAG_MCA && entry->workers > 1 && socket_can_balance()) - return SOCKET_BALANCED; - return entry->shared ? SOCKET_SHARED : SOCKET_EXCLUSIVE; -} - -static void listen_bind_one(ot_listen *entry) { - socket_sharing sharing = listen_sharing_for(entry); - -#ifdef _DEBUG - { - const char *protos[] = {"TCP", "UDP", "UDP mcast"}; - char _debug[512]; - int off = snprintf(_debug, sizeof(_debug), "Binding socket type %s to address [", protos[entry->proto]); - off += ip_format(_debug + off, entry->ip); - snprintf(_debug + off, sizeof(_debug) - off, "]:%d...", entry->port); - fputs(_debug, stderr); - } -#endif - -#ifdef WANT_SYNC_LIVE - if (entry->proto == FLAG_MCA) { - livesync_bind_mcast(entry->ip, entry->port); -#ifdef _DEBUG - fputs(" success.\n", stderr); -#endif - return; - } -#endif - - if (entry->proto == FLAG_TCP) { - if ((entry->sock = socket_tcp_listen(entry->ip, entry->port, sharing)) == -1) - panic("socket_tcp_listen"); - socket_defer_accept((int)entry->sock); - } else { - if ((entry->sock = socket_udp_bind(entry->ip, entry->port, sharing)) == -1) - panic("socket_udp_bind"); - } - - if (event_add((int)entry->sock, EVENT_IN, fdstate_open(entry->sock, (void *)entry->proto, 0)) == -1) - panic("event_add"); - -#ifdef _DEBUG - fputs(" success.\n", stderr); -#endif -} - -/* Bound in the order the entries were given. - Note: On macOS SO_REUSEPORT delivers everything to the socket that bound last. */ -static void listen_bind_all(void) { - size_t count, i; - ot_listen *entries = config_listen_entries(&count); - - for (i = 0; i < count; ++i) - listen_bind_one(entries + i); -} - -/* The socket the event loop was using is passed to the first worker. - If start up tests show that the kernel can spread connections over - a group bound to one address, every worker binds one of its own. - If not, all workers call accept on the first. */ -static void listen_start_tcp_workers(void) { - size_t count, i; - ot_listen *entries = config_listen_entries(&count); - - for (i = 0; i < count; ++i) { - ot_listen *entry = entries + i; - int balance = listen_sharing_for(entry) == SOCKET_BALANCED; - int worker; - - if (entry->proto != FLAG_TCP || entry->workers <= 0) - continue; - - event_del((int)entry->sock); - tcp_worker_install(entry->sock); - - for (worker = 1; worker < entry->workers; ++worker) { - int64_t sock = entry->sock; - - if (balance) { - sock = socket_tcp_listen(entry->ip, entry->port, SOCKET_BALANCED); - if (sock == -1) { - fprintf(stderr, "Warning: could not bind another socket for tcp worker %d: %s. Sharing the first instead.\n", worker, - strerror(errno)); - sock = entry->sock; - } - } - tcp_worker_install(sock); - } - } -} - -static void listen_start_udp_workers(void) { - size_t count, i; - ot_listen *entries = config_listen_entries(&count); - - udp_init(); - - for (i = 0; i < count; ++i) { - ot_listen *entry = entries + i; - int balance = listen_sharing_for(entry) == SOCKET_BALANCED; - int worker; - - if (entry->proto != FLAG_UDP || entry->workers <= 0) - continue; - - event_del((int)entry->sock); - udp_worker_install(entry->sock); - - for (worker = 1; worker < entry->workers; ++worker) { - int64_t sock = entry->sock; - - if (balance) { - sock = socket_udp_bind(entry->ip, entry->port, SOCKET_BALANCED); - if (sock == -1) { - fprintf(stderr, "Warning: could not bind another socket for udp worker %d: %s. Sharing the first instead.\n", worker, strerror(errno)); - sock = entry->sock; - } - } - udp_worker_install(sock); - } - } -} - -void load_state(const char *const state_filename) { - FILE *state_filehandle; - char inbuf[512]; - ot_hash infohash; - uint64_t base, downcount; - int consumed; - - state_filehandle = fopen(state_filename, "r"); - - if (state_filehandle == NULL) { - fprintf(stderr, "Warning: Can't open config file: %s.", state_filename); - return; - } - - /* We do ignore anything that is not of the form "^[:xdigit:]:\d+:\d+" */ - while (fgets(inbuf, sizeof(inbuf), state_filehandle)) { - size_t i; - - if (!hex_to_hash(infohash, inbuf)) - continue; - i = 2 * sizeof(ot_hash); - - if (inbuf[i++] != ':' || !(consumed = scan_uint(inbuf + i, &base, UINT64_MAX))) - continue; - i += consumed; - if (inbuf[i++] != ':' || !(consumed = scan_uint(inbuf + i, &downcount, UINT64_MAX))) - continue; - add_torrent_from_saved_state(infohash, base, downcount); - } - - fclose(state_filehandle); -} - -int drop_privileges(const char *const serveruser, const char *const serverdir) { - struct passwd *pws = NULL; - -#ifdef _DEBUG - if (!geteuid()) - fprintf(stderr, "Dropping to user %s.\n", serveruser); - if (serverdir) - fprintf(stderr, "ch%s'ing to directory %s.\n", geteuid() ? "dir" : "root", serverdir); -#endif - - /* Grab pws entry before chrooting */ - pws = getpwnam(serveruser); - endpwent(); - - if (geteuid() == 0) { - /* Running as root: chroot and drop privileges */ - if (serverdir && chroot(serverdir)) { - fprintf(stderr, "Could not chroot to %s, because: %s\n", serverdir, strerror(errno)); - return -1; - } - - if (chdir("/")) - panic("chdir() failed after chrooting: "); - - /* If we can't find server user, revert to nobody's default uid */ - if (!pws) { - fprintf(stderr, "Warning: Could not get password entry for %s. Reverting to uid -2.\n", serveruser); - if (setegid((gid_t)-2) || setgid((gid_t)-2) || setuid((uid_t)-2) || seteuid((uid_t)-2)) - panic("Could not set uid to value -2"); - } else { - if (setegid(pws->pw_gid) || setgid(pws->pw_gid) || setuid(pws->pw_uid) || seteuid(pws->pw_uid)) - panic("Could not set uid to specified value"); - } - - if (geteuid() == 0 || getegid() == 0) - panic("Still running with root privileges?!"); - } else { - /* Normal user, just chdir() */ - if (serverdir && chdir(serverdir)) { - fprintf(stderr, "Could not chroot to %s, because: %s\n", serverdir, strerror(errno)); - return -1; - } - } - - return 0; -} - -/* Maintain our copy of the clock. time() on BSDs is very expensive. */ -int main(int argc, char **argv) { - - random_init(); - if (fdstate_init() == -1) - panic("fdstate_init"); - if (event_init() == -1) - panic("event_init"); - config_parse(argc, argv); - - listen_bind_all(); - - defaul_signal_handlers(); - -#ifdef WANT_SYSLOGS - openlog("opentracker", 0, LOG_USER); - setlogmask(LOG_UPTO(LOG_INFO)); -#endif - - if (drop_privileges(g_serveruser ? g_serveruser : "nobody", g_serverdir) == -1) - panic("drop_privileges failed, exiting. Last error"); - - - /* Create our self pipe which allows us to interrupt mainloops - wait in case some data is available to send out */ - if (pipe(g_self_pipe) == -1) - panic("selfpipe failed: "); - /* Both ends have to be non blocking: the loop drains the read end until it - is empty, and a worker nudging a full pipe has already made its point */ - socket_set_blocking(g_self_pipe[0], 0); - socket_set_blocking(g_self_pipe[1], 0); - if (event_add(g_self_pipe[0], EVENT_IN, fdstate_open(g_self_pipe[0], (void *)FLAG_SELFPIPE, 0)) == -1) - panic("selfpipe event_add"); - - /* Init all sub systems. This call may fail with an exit() */ - trackerlogic_init(); - -#ifdef _DEBUG_RANDOMTORRENTS - fprintf(stderr, "DEBUG: Generating %d random peers on random torrents. This may take a while. (Setting RANDOMTORRENTS in trackerlogic.h)\n", RANDOMTORRENTS); - trackerlogic_add_random_torrents(RANDOMTORRENTS); - fprintf(stderr, "... done.\n"); -#endif - - if (g_statefile) - load_state(g_statefile); - - install_signal_handlers(); - - listen_start_tcp_workers(); - listen_start_udp_workers(); - - server_mainloop(0); - worker_stop_all(); - - /* No thread is serving any more, so the teardown can run where a debugger - and the sanitizers can follow it */ - trackerlogic_deinit(); -#ifdef WANT_SYSLOGS - closelog(); -#endif - - event_deinit(); - fdstate_deinit(); - return 0; -} diff --git a/ot_accesslist.c b/ot_accesslist.c deleted file mode 100644 index 86f9a7d..0000000 --- a/ot_accesslist.c +++ /dev/null @@ -1,601 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include -#include -#include -#include -#include -#if defined(WANT_ACCESSLIST_BLACK) || defined(WANT_ACCESSLIST_WHITE) -#include -#include -#include -#include -#include -#endif -#ifdef WANT_DYNAMIC_ACCESSLIST -#include -#endif - -/* Opentracker */ -#include "ot_accesslist.h" -#include "ot_hex.h" -#include "ot_ip.h" -#include "ot_vector.h" -#include "trackerlogic.h" - -/* GLOBAL VARIABLES */ -#ifdef WANT_ACCESSLIST -const char *g_accesslist_filename = NULL; -#ifdef WANT_DYNAMIC_ACCESSLIST -const char *g_accesslist_pipe_add = NULL; -const char *g_accesslist_pipe_delete = NULL; -#endif -static pthread_mutex_t g_accesslist_mutex; - -/* Accesslists are lock free linked lists. We can not make them locking, because every announce - would try to acquire the mutex, making it the most contested mutex in the whole of opentracker, - basically creating a central performance choke point. - - The idea is that updating the list heads happens under the g_accesslist_mutex guard and is - done atomically, while consumers might potentially still hold pointers deeper inside the list. - - Consumers (for now only via accesslist_hashisvalid) will always fetch the list head pointer - that is guaranteed to live for at least five minutes. This should be many orders of magnitudes - more than how long it will be needed by the bsearch done on the list. */ -struct ot_accesslist; -typedef struct ot_accesslist ot_accesslist; -struct ot_accesslist { - ot_hash *list; - size_t size; - ot_time base; - ot_accesslist *next; -}; -static ot_accesslist *_Atomic g_accesslist = NULL; -#ifdef WANT_DYNAMIC_ACCESSLIST -static ot_accesslist *_Atomic g_accesslist_add = NULL; -static ot_accesslist *_Atomic g_accesslist_delete = NULL; -#endif - -/* Helpers to work on access lists */ -static int vector_compare_hash(const void *hash1, const void *hash2) { return memcmp(hash1, hash2, OT_HASH_COMPARE_SIZE); } - -static ot_accesslist *accesslist_free(ot_accesslist *accesslist) { - while (accesslist) { - ot_accesslist *this_accesslist = accesslist; - accesslist = this_accesslist->next; - free(this_accesslist->list); - free(this_accesslist); - } - return NULL; -} - -static ot_accesslist *accesslist_make(ot_accesslist *next, size_t size) { - ot_accesslist *accesslist_new = malloc(sizeof(ot_accesslist)); - if (accesslist_new) { - accesslist_new->list = size ? malloc(sizeof(ot_hash) * size) : NULL; - accesslist_new->size = size; - accesslist_new->base = 0; - accesslist_new->next = next; - if (size && !accesslist_new->list) { - free(accesslist_new); - accesslist_new = NULL; - } - } - return accesslist_new; -} - -/* This must be called with g_accesslist_mutex held. - This will never delete head, because that might still be in use. */ -static void accesslist_clean(ot_accesslist *accesslist) { - while (accesslist && accesslist->next) { - if (accesslist->next->base + 5 < time_now_minutes()) - accesslist->next = accesslist_free(accesslist->next); - accesslist = accesslist->next; - } -} - -/* A hash needs forty bytes, and all but a final one are followed by at least - one separator, so n hashes need at least 41n - 1 bytes. (len + 1) / 41 is - therefore an exact upper bound on what a buffer of len bytes can hold. */ -size_t accesslist_parse_hashes(ot_hash *dest, size_t capacity, const char *buf, size_t len) { - const char *end = buf + len; - const char *offs = buf; - size_t count = 0; - - while (count < capacity && end - offs >= 40) { - if (hex_to_hash(dest[count], offs)) { - offs += 40; - - if (offs == end || !hex_isdigit((unsigned char)*offs)) - ++count; - } - - /* Find start of next line */ - while (offs < end && *(offs++) != '\n') {} - } - - return count; -} - -/* Map a file read only. Returns 0 and sets map and maplen on success, -1 if - the file cannot be opened or mapped. */ -static int accesslist_map(const char *filename, const char **map, size_t *maplen) { - struct stat st; - void *mapped; - int fd = open(filename, O_RDONLY); - - if (fd < 0) - return -1; - - if (fstat(fd, &st) < 0 || st.st_size < 0 || (uintmax_t)st.st_size > (uintmax_t)SIZE_MAX) { - close(fd); - return -1; - } - - if (st.st_size == 0) { - close(fd); - *map = NULL; - *maplen = 0; - return 0; - } - - mapped = mmap(NULL, (size_t)st.st_size, PROT_READ, MAP_SHARED, fd, 0); - close(fd); - - if (mapped == MAP_FAILED) - return -1; - - *map = mapped; - *maplen = (size_t)st.st_size; - return 0; -} - -static void accesslist_readfile(void) { - ot_accesslist *accesslist_new; - const char *map; - size_t maplen, capacity, found; - - if (!g_accesslist_filename) - return; - - if (accesslist_map(g_accesslist_filename, &map, &maplen) < 0) { - char *wd = getcwd(NULL, 0); - fprintf(stderr, "Warning: Can't open accesslist file: %s (but will try to create it later, if necessary and possible).\nPWD: %s\n", g_accesslist_filename, wd); - free(wd); - return; - } - - /* Make room for the most info_hashes this file could possibly hold */ - capacity = (maplen + 1) / 41; - accesslist_new = accesslist_make(g_accesslist, capacity); - if (!accesslist_new) { - fprintf(stderr, "Warning: Not enough memory to allocate %zd bytes for accesslist buffer. May succeed later.\n", capacity * sizeof(ot_hash)); - if (map) - munmap((void *)map, maplen); - return; - } - - found = maplen ? accesslist_parse_hashes(accesslist_new->list, capacity, map, maplen) : 0; -#ifdef _DEBUG - fprintf(stderr, "Added %zd info_hashes to accesslist\n", found); -#endif - - if (map) - munmap((void *)map, maplen); - - if (found) - qsort(accesslist_new->list, found, sizeof(ot_hash), vector_compare_hash); - accesslist_new->size = found; - - /* Now exchange the accesslist vector in the least race condition prone way */ - pthread_mutex_lock(&g_accesslist_mutex); - - /* Mark old accesslist as "needs to live another five minutes" */ - if (g_accesslist) - g_accesslist->base = time_now_minutes(); - accesslist_new->next = g_accesslist; - g_accesslist = accesslist_new; /* Only now set a new list */ - -#ifdef WANT_DYNAMIC_ACCESSLIST - /* If we have dynamic accesslists, reloading a new one will always void the add/delete lists. - Insert empty ones at the list head */ - if (g_accesslist_add && (accesslist_new = accesslist_make(g_accesslist_add, 0)) != NULL) { - g_accesslist_add->base = time_now_minutes(); - g_accesslist_add = accesslist_new; - } - if (g_accesslist_delete && (accesslist_new = accesslist_make(g_accesslist_delete, 0)) != NULL) { - g_accesslist_delete->base = time_now_minutes(); - g_accesslist_delete = accesslist_new; - } -#endif - - accesslist_clean(g_accesslist); - - pthread_mutex_unlock(&g_accesslist_mutex); -} - -int accesslist_hashisvalid(const ot_hash hash) { - /* Get working copy of current access list */ - ot_accesslist *accesslist = g_accesslist; -#ifdef WANT_DYNAMIC_ACCESSLIST - ot_accesslist *accesslist_add, *accesslist_delete; -#endif - void *exactmatch = NULL; - - if (accesslist && accesslist->size) - exactmatch = bsearch(hash, accesslist->list, accesslist->size, OT_HASH_COMPARE_SIZE, vector_compare_hash); - -#ifdef WANT_DYNAMIC_ACCESSLIST - /* If we had no match on the main list, scan the list of dynamically added hashes */ - accesslist_add = g_accesslist_add; - if ((exactmatch == NULL) && accesslist_add && accesslist_add->size) - exactmatch = bsearch(hash, accesslist_add->list, accesslist_add->size, OT_HASH_COMPARE_SIZE, vector_compare_hash); - - /* If we found a matching hash on the main list, scan the list of dynamically deleted hashes */ - accesslist_delete = g_accesslist_delete; - if ((exactmatch != NULL) && accesslist_delete && accesslist_delete->size && bsearch(hash, accesslist_delete->list, accesslist_delete->size, OT_HASH_COMPARE_SIZE, vector_compare_hash)) - exactmatch = NULL; -#endif - -#ifdef WANT_ACCESSLIST_BLACK - return exactmatch == NULL; -#else - return exactmatch != NULL; -#endif -} - -static void *accesslist_worker(void *args) { - int sig; - sigset_t signal_mask; - - sigemptyset(&signal_mask); - sigaddset(&signal_mask, SIGHUP); - - (void)args; - - while (1) { - if (!g_opentracker_running) - return NULL; - - /* Initial attempt to read accesslist */ - accesslist_readfile(); - - /* Wait for signals */ - while (sigwait(&signal_mask, &sig) != 0 && sig != SIGHUP) - ; - } - return NULL; -} - -#ifdef WANT_DYNAMIC_ACCESSLIST -static pthread_t thread_adder_id, thread_deleter_id; -static _Atomic int adder_left, deleter_left; -static void *accesslist_adddel_worker(const char *fifoname, ot_accesslist *_Atomic *adding_to, ot_accesslist *_Atomic *removing_from) { - struct stat st; - - if (!stat(fifoname, &st)) { - if (!S_ISFIFO(st.st_mode)) { - fprintf(stderr, "Error when starting dynamic accesslists: Found Non-FIFO file at %s.\nPlease remove it and restart opentracker.\n", fifoname); - return NULL; - } - } else { - int error = mkfifo(fifoname, 0755); - if (error && error != EEXIST) { - fprintf(stderr, "Error when starting dynamic accesslists: Couldn't create FIFO at %s, error: %s\n", fifoname, strerror(errno)); - return NULL; - } - } - - while (g_opentracker_running) { - FILE *fifo = fopen(fifoname, "r"); - char *line = NULL; - size_t linecap = 0; - ssize_t linelen; - - if (!fifo) { - /* Woken on purpose: the loop condition above decides what happens next */ - if (errno == EINTR) - continue; - fprintf(stderr, "Error when reading dynamic accesslists: Couldn't open FIFO at %s, error: %s\n", fifoname, strerror(errno)); - return NULL; - } - - while ((linelen = getline(&line, &linecap, fifo)) > 0) { - ot_hash info_hash; - - printf("Got line %*s", (int)linelen, line); - /* We do ignore anything that is not of the form "^[:xdigit:]{40}[^:xdigit:].*" - If there's not enough characters for an info_hash in the line, skip it. */ - if (linelen < 41) - continue; - - if (!hex_to_hash(info_hash, line)) - continue; - - /* From now on we modify g_accesslist_add and g_accesslist_delete, so prevent the - other worker threads from doing the same */ - pthread_mutex_lock(&g_accesslist_mutex); - - /* If the info hash is in the removing_from list, create a new head without that entry */ - if (*removing_from && (*removing_from)->list) { - ot_hash *exactmatch = bsearch(info_hash, (*removing_from)->list, (*removing_from)->size, OT_HASH_COMPARE_SIZE, vector_compare_hash); - if (exactmatch) { - ptrdiff_t off = exactmatch - (*removing_from)->list; - ot_accesslist *accesslist_new = accesslist_make(*removing_from, (*removing_from)->size - 1); - if (accesslist_new) { - size_t after = (*removing_from)->size - off - 1; - - /* Removing the only entry leaves a list with no room at all, and - memcpy may not be handed that even for nothing */ - if (off) - memcpy(accesslist_new->list, (*removing_from)->list, sizeof(ot_hash) * off); - if (after) - memcpy(accesslist_new->list + off, (*removing_from)->list + off + 1, sizeof(ot_hash) * after); - (*removing_from)->base = time_now_minutes(); - *removing_from = accesslist_new; - } - } - } - - /* Simple case: there's no adding_to list yet, create one with one member */ - if (!*adding_to) { - ot_accesslist *accesslist_new = accesslist_make(NULL, 1); - if (accesslist_new) { - memcpy(accesslist_new->list, info_hash, sizeof(ot_hash)); - *adding_to = accesslist_new; - } - } else { - int exactmatch = 0; - ot_hash *insert_point = vector_binary_search(info_hash, (*adding_to)->list, (*adding_to)->size, sizeof(ot_hash), OT_HASH_COMPARE_SIZE, &exactmatch); - - /* Only if the info hash is not in the adding_to list, create a new head with that entry */ - if (!exactmatch) { - ot_accesslist *accesslist_new = accesslist_make(*adding_to, (*adding_to)->size + 1); - ptrdiff_t off = insert_point - (*adding_to)->list; - if (accesslist_new) { - memcpy(accesslist_new->list, (*adding_to)->list, sizeof(ot_hash) * off); - memcpy(accesslist_new->list + off, info_hash, sizeof(info_hash)); - memcpy(accesslist_new->list + off + 1, (*adding_to)->list + off, sizeof(ot_hash) * ((*adding_to)->size - off)); - (*adding_to)->base = time_now_minutes(); - *adding_to = accesslist_new; - } - } - } - - pthread_mutex_unlock(&g_accesslist_mutex); - } - - free(line); - fclose(fifo); - } - return NULL; -} - -static void *accesslist_adder_worker(void *args) { - (void)args; - accesslist_adddel_worker(g_accesslist_pipe_add, &g_accesslist_add, &g_accesslist_delete); - atomic_store(&adder_left, 1); - return NULL; -} -static void *accesslist_deleter_worker(void *args) { - (void)args; - accesslist_adddel_worker(g_accesslist_pipe_delete, &g_accesslist_delete, &g_accesslist_add); - atomic_store(&deleter_left, 1); - return NULL; -} - -/* These sit in a blocking open or read on a FIFO, so they are woken the way - the other workers are, and the signal is repeated because it only - interrupts a call already under way. The pipe name is what says whether - the thread was ever started. */ -static void accesslist_stop_fifo_worker(pthread_t thread, _Atomic int *left, const char *pipe_name) { - if (!pipe_name) - return; - - while (!atomic_load(left)) { - pthread_kill(thread, OT_SIGNAL_WAKE); - usleep(1000); - } - pthread_join(thread, NULL); -} -#endif - -static pthread_t thread_id; -void accesslist_init(void) { - pthread_mutex_init(&g_accesslist_mutex, NULL); - pthread_create(&thread_id, NULL, accesslist_worker, NULL); -#ifdef WANT_DYNAMIC_ACCESSLIST - if (g_accesslist_pipe_add) - pthread_create(&thread_adder_id, NULL, accesslist_adder_worker, NULL); - if (g_accesslist_pipe_delete) - pthread_create(&thread_deleter_id, NULL, accesslist_deleter_worker, NULL); -#endif -} - -void accesslist_deinit(void) { - /* Wake the worker and wait for it to notice that the tracker is stopping. */ - pthread_kill(thread_id, SIGHUP); - pthread_join(thread_id, NULL); - -#ifdef WANT_DYNAMIC_ACCESSLIST - /* Before the lists they are reading are freed below */ - accesslist_stop_fifo_worker(thread_adder_id, &adder_left, g_accesslist_pipe_add); - accesslist_stop_fifo_worker(thread_deleter_id, &deleter_left, g_accesslist_pipe_delete); -#endif - - pthread_mutex_lock(&g_accesslist_mutex); - - g_accesslist = accesslist_free(g_accesslist); - -#ifdef WANT_DYNAMIC_ACCESSLIST - g_accesslist_add = accesslist_free(g_accesslist_add); - g_accesslist_delete = accesslist_free(g_accesslist_delete); -#endif - - pthread_mutex_unlock(&g_accesslist_mutex); - pthread_mutex_destroy(&g_accesslist_mutex); -} - -void accesslist_cleanup(void) { - pthread_mutex_lock(&g_accesslist_mutex); - - accesslist_clean(g_accesslist); -#if WANT_DYNAMIC_ACCESSLIST - accesslist_clean(g_accesslist_add); - accesslist_clean(g_accesslist_delete); -#endif - - pthread_mutex_unlock(&g_accesslist_mutex); -} -#endif - -int accesslist_address_in_net(const ot_ip6 address, const ot_net *net) { - int bits = net->bits, checkbits = (0x7f00 >> (bits & 7)); - int result = memcmp(address, &net->address, bits >> 3); - if (!result && (bits & 7)) - result = (checkbits & address[bits >> 3]) - (checkbits & net->address[bits >> 3]); - return result == 0; -} - -void *accesslist_set_value_for_net(const ot_net *net, ot_vector *vector, const void *value, const size_t member_size) { - size_t i; - int exactmatch; - - /* Caller must have a concept of ot_net in it's member */ - if (member_size < sizeof(ot_net)) - return 0; - - /* Check each net in vector for overlap */ - uint8_t *member = ((uint8_t *)vector->data); - for (i = 0; i < vector->size; ++i) { - if (accesslist_address_in_net(*(ot_ip6 *)member, net) || accesslist_address_in_net(net->address, (ot_net *)member)) - return 0; - member += member_size; - } - - member = vector_find_or_insert(vector, (void *)net, member_size, sizeof(ot_net), &exactmatch); - if (member) { - memcpy(member, net, sizeof(ot_net)); - memcpy(member + sizeof(ot_net), value, member_size - sizeof(ot_net)); - } - - return member; -} - -/* Takes a vector filled with { ot_net net, uint8_t[x] value }; - Returns value associated with the net, or NULL if not found */ -void *accesslist_get_value_for_net(const ot_ip6 address, const ot_vector *vector, const size_t member_size) { - int exactmatch; - /* This binary search will return a pointer to the first non-containing network... */ - ot_net *net = vector_binary_search(address, vector->data, vector->size, member_size, sizeof(ot_ip6), &exactmatch); - if (!net) - return NULL; - /* ... so we'll need to move back one step unless we've exactly hit the first address in network */ - if (!exactmatch && ((void *)net > vector->data)) - --net; - if (!accesslist_address_in_net(address, net)) - return NULL; - return (void *)net; -} - -#ifdef WANT_IP_FROM_PROXY -typedef struct { - ot_net *proxy; - ot_vector networks; -} ot_proxymap; - -static ot_vector g_proxies_list; -static pthread_mutex_t g_proxies_list_mutex = PTHREAD_MUTEX_INITIALIZER; - -int proxylist_add_network(const ot_net *proxy, const ot_net *net) { - ot_proxymap *map; - int exactmatch, result = 1; - pthread_mutex_lock(&g_proxies_list_mutex); - - /* If we have a direct hit, use and extend the vector there */ - map = vector_binary_search(proxy, g_proxies_list.data, g_proxies_list.size, sizeof(ot_proxymap), sizeof(ot_net), &exactmatch); - - if (!map || !exactmatch) { - /* else see, if we've got overlapping networks - and get a new empty vector if not */ - ot_vector empty; - memset(&empty, 0, sizeof(ot_vector)); - map = accesslist_set_value_for_net(proxy, &g_proxies_list, &empty, sizeof(ot_proxymap)); - } - - if (map && accesslist_set_value_for_net(net, &map->networks, NULL, sizeof(ot_net))) - result = 1; - - pthread_mutex_unlock(&g_proxies_list_mutex); - return result; -} - -int proxylist_check_proxy(const ot_ip6 proxy, const ot_ip6 address) { - int result = 0; - ot_proxymap *map; - - pthread_mutex_lock(&g_proxies_list_mutex); - - if ((map = accesslist_get_value_for_net(proxy, &g_proxies_list, sizeof(ot_proxymap)))) - if (!address || accesslist_get_value_for_net(address, &map->networks, sizeof(ot_net))) - result = 1; - - pthread_mutex_unlock(&g_proxies_list_mutex); - return result; -} - -#endif - -static ot_net g_admin_nets[OT_ADMINIP_MAX]; -static ot_permissions g_admin_nets_permissions[OT_ADMINIP_MAX]; -static size_t g_admin_nets_count = 0; - -int accesslist_bless_net(ot_net *net, ot_permissions permissions) { - if (g_admin_nets_count >= OT_ADMINIP_MAX) - return -1; - - memcpy(g_admin_nets + g_admin_nets_count, net, sizeof(ot_net)); - g_admin_nets_permissions[g_admin_nets_count++] = permissions; - -#ifdef _DEBUG - { - char _debug[512]; - int off = snprintf(_debug, sizeof(_debug), "Blessing ip net "); - off += ip_format(_debug + off, net->address); - if (net->bits < 128) { - _debug[off++] = '/'; - if (ip_is_v4mapped(net->address)) - off += snprintf(_debug + off, sizeof(_debug) - off, "%d", net->bits - 96); - else - off += snprintf(_debug + off, sizeof(_debug) - off, "%d", net->bits); - } - - if (permissions & OT_PERMISSION_MAY_STAT) - off += snprintf(_debug + off, 512 - off, " may_fetch_stats"); - if (permissions & OT_PERMISSION_MAY_LIVESYNC) - off += snprintf(_debug + off, 512 - off, " may_sync_live"); - if (permissions & OT_PERMISSION_MAY_FULLSCRAPE) - off += snprintf(_debug + off, 512 - off, " may_fetch_fullscrapes"); - if (permissions & OT_PERMISSION_MAY_PROXY) - off += snprintf(_debug + off, 512 - off, " may_proxy"); - if (!permissions) - off += snprintf(_debug + off, sizeof(_debug) - off, " nothing"); - _debug[off++] = '.'; - _debug[off++] = '\n'; - (void)write(2, _debug, off); - } -#endif - - return 0; -} - -int accesslist_is_blessed(ot_ip6 ip, ot_permissions permissions) { - size_t i; - for (i = 0; i < g_admin_nets_count; ++i) - if (accesslist_address_in_net(ip, g_admin_nets + i) && (g_admin_nets_permissions[i] & permissions)) - return 1; - return 0; -} diff --git a/ot_accesslist.h b/ot_accesslist.h deleted file mode 100644 index 68c88dc..0000000 --- a/ot_accesslist.h +++ /dev/null @@ -1,77 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_ACCESSLIST_H__ -#define OT_ACCESSLIST_H__ - -#include "trackerlogic.h" - -#if defined(WANT_ACCESSLIST_BLACK) && defined(WANT_ACCESSLIST_WHITE) -#error WANT_ACCESSLIST_BLACK and WANT_ACCESSLIST_WHITE are exclusive. -#endif - -#if defined(WANT_ACCESSLIST_BLACK) || defined(WANT_ACCESSLIST_WHITE) -#define WANT_ACCESSLIST -/* Starts a thread that rereads the list when sent SIGHUP, and is stopped by - being sent one, so the caller blocks SIGHUP process wide beforehand */ -void accesslist_init(void); -void accesslist_deinit(void); -int accesslist_hashisvalid(const ot_hash hash); -void accesslist_cleanup(void); - -/* Scan buf for up to capacity info_hashes in hexadecimal notation, one per - line, and return how many were written to dest */ -size_t accesslist_parse_hashes(ot_hash *dest, size_t capacity, const char *buf, size_t len); - -extern const char *g_accesslist_filename; -#ifdef WANT_DYNAMIC_ACCESSLIST -extern const char *g_accesslist_pipe_add; -extern const char *g_accesslist_pipe_delete; -#endif - -#else -#ifdef WANT_DYNAMIC_ACCESSLIST -#error WANT_DYNAMIC_ACCESSLIST needs either WANT_ACCESSLIST_BLACK or WANT_ACCESSLIST_WHITE -#endif - -#define accesslist_init(accesslist_filename) -#define accesslist_deinit() -#define accesslist_hashisvalid(hash) 1 -#endif - -/* Test if an address is subset of an ot_net, return value is considered a bool */ -int accesslist_address_in_net(const ot_ip6 address, const ot_net *net); - -/* Store a value into a vector of struct { ot_net net, uint8_t[x] value } member; - returns NULL - if member_size is too small, or - if one of the nets inside the vector are a subnet of _net_, or - if _net_ is a subnet of one of the nets inside the vector, or - if the vector could not be resized - returns pointer to new member in vector for success - member_size can be sizeof(ot_net) to reduce the lookup to a boolean mapping -*/ -void *accesslist_set_value_for_net(const ot_net *net, ot_vector *vector, const void *value, const size_t member_size); - -/* Takes a vector filled with struct { ot_net net, uint8_t[x] value } member; - Returns pointer to _member_ associated with the net, or NULL if not found - member_size can be sizeof(ot_net) to reduce the lookup to a boolean mapping -*/ -void *accesslist_get_value_for_net(const ot_ip6 address, const ot_vector *vector, const size_t member_size); - -#ifdef WANT_IP_FROM_PROXY -int proxylist_add_network(const ot_net *proxy, const ot_net *net); -int proxylist_check_network(const ot_ip6 *proxy, const ot_ip6 address /* can be NULL to only check proxy */); -#endif - -typedef enum { - OT_PERMISSION_MAY_FULLSCRAPE = 0x1, - OT_PERMISSION_MAY_STAT = 0x2, - OT_PERMISSION_MAY_LIVESYNC = 0x4, - OT_PERMISSION_MAY_PROXY = 0x8 -} ot_permissions; - -int accesslist_bless_net(ot_net *net, ot_permissions permissions); -int accesslist_is_blessed(ot_ip6 ip, ot_permissions permissions); - -#endif diff --git a/ot_batch.c b/ot_batch.c deleted file mode 100644 index e4dda0b..0000000 --- a/ot_batch.c +++ /dev/null @@ -1,156 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include -#include -#include -#include -#include -#include - -/* Opentracker */ -#include "ot_batch.h" - -#ifndef IOV_MAX -#define IOV_MAX 1024 -#endif - -#define OT_BATCH_MIN_ENTRIES 16 - -static int batch_room_for(ot_batch *batch, size_t wanted) { - size_t space = batch->space ? batch->space : OT_BATCH_MIN_ENTRIES; - struct iovec *grown; - - if (batch->count + wanted <= batch->space) - return 0; - - /* During a chunked transfer entries are appended while earlier ones are - still going out, so the front of the array frees up as we go. Slide the - rest down before asking for more. */ - if (batch->first && batch->count - batch->first + wanted <= batch->space) { - memmove(batch->entries, batch->entries + batch->first, (batch->count - batch->first) * sizeof(struct iovec)); - batch->count -= batch->first; - batch->first = 0; - return 0; - } - - while (space < batch->count + wanted) - space *= 2; - - if (!(grown = realloc(batch->entries, space * sizeof(struct iovec)))) - return -1; - - batch->entries = grown; - batch->space = space; - return 0; -} - -int batch_add(ot_batch *batch, void *buf, size_t len) { - if (batch_room_for(batch, 1)) { - free(buf); - return -1; - } - - batch->entries[batch->count].iov_base = buf; - batch->entries[batch->count].iov_len = len; - ++batch->count; - return 0; -} - -int batch_add_vector(ot_batch *batch, const struct iovec *entries, size_t count) { - if (batch_room_for(batch, count)) { - while (count--) - free(entries[count].iov_base); - return -1; - } - - memcpy(batch->entries + batch->count, entries, count * sizeof(struct iovec)); - batch->count += count; - return 0; -} - -/* Drop everything the kernel accepted, freeing each buffer as it goes */ -static void batch_consume(ot_batch *batch, size_t written) { - while (written) { - struct iovec *entry = batch->entries + batch->first; - size_t left = entry->iov_len - batch->offset; - - if (written < left) { - batch->offset += written; - return; - } - - free(entry->iov_base); - written -= left; - batch->offset = 0; - ++batch->first; - } - - /* Nothing is outstanding, so the array can start over from the front */ - if (batch->first == batch->count) - batch->first = batch->count = 0; -} - -/* A peer that has gone away must come back as an error rather than as a - signal that kills the tracker, so ask the kernel not to raise one where it - can be asked. macOS cannot, and relies on SIGPIPE being blocked. */ -static ssize_t batch_writev(int s, struct iovec *entries, int count) { -#ifdef MSG_NOSIGNAL - struct msghdr msg; - - memset(&msg, 0, sizeof(msg)); - msg.msg_iov = entries; - msg.msg_iovlen = count; - return sendmsg(s, &msg, MSG_NOSIGNAL); -#else - return writev(s, entries, count); -#endif -} - -ssize_t batch_send(int s, ot_batch *batch) { - size_t entries = batch->count - batch->first; - ssize_t written; - - if (!batch_pending(batch)) - return BATCH_DONE; - - if (entries > IOV_MAX) - entries = IOV_MAX; - - /* The first entry may be half way out already */ - batch->entries[batch->first].iov_base = (char *)batch->entries[batch->first].iov_base + batch->offset; - batch->entries[batch->first].iov_len -= batch->offset; - written = batch_writev(s, batch->entries + batch->first, (int)entries); - batch->entries[batch->first].iov_base = (char *)batch->entries[batch->first].iov_base - batch->offset; - batch->entries[batch->first].iov_len += batch->offset; - - if (written < 0) - return (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) ? BATCH_AGAIN : BATCH_FAILED; - if (!written) - return BATCH_FAILED; - - batch_consume(batch, (size_t)written); - return written; -} - -void batch_reset(ot_batch *batch) { - while (batch->first < batch->count) - free(batch->entries[batch->first++].iov_base); - - free(batch->entries); - batch->entries = NULL; - batch->count = batch->first = batch->offset = batch->space = 0; -} - -/* For the few constants the chunked encoding needs, which the batch frees - along with everything else */ -int batch_add_copy(ot_batch *batch, const void *buf, size_t len) { - void *copy = malloc(len); - - if (!copy) - return -1; - memcpy(copy, buf, len); - return batch_add(batch, copy, len); -} diff --git a/ot_batch.h b/ot_batch.h deleted file mode 100644 index 910bea7..0000000 --- a/ot_batch.h +++ /dev/null @@ -1,46 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_BATCH_H__ -#define OT_BATCH_H__ - -#include -#include -#include - -/* A queue of buffers on their way to one socket. The batch owns every buffer - handed to it and frees each one as soon as it has left, so a client that - reads slowly costs memory only for what it has not received yet. - - A zeroed ot_batch is ready to use. */ -typedef struct { - struct iovec *entries; - size_t space; - size_t count; - size_t first; /* entries before this one are sent and freed */ - size_t offset; /* bytes of entries[first] already sent */ -} ot_batch; - -static inline int batch_pending(const ot_batch *batch) { return batch->first != batch->count; } - -/* What batch_send reports instead of a byte count */ -enum { BATCH_DONE = 0, BATCH_AGAIN = -1, BATCH_FAILED = -2 }; - -/* Queue buf and take ownership of it. Returns 0, or -1 having freed buf. */ -int batch_add(ot_batch *batch, void *buf, size_t len); - -/* Write as much as the socket takes. Returns the number of bytes written, or - BATCH_DONE, BATCH_AGAIN or BATCH_FAILED. */ -ssize_t batch_send(int s, ot_batch *batch); - -/* Queue every entry of a vector, taking ownership of each buffer but not of - the vector itself. Returns 0, or -1 having freed all of them. */ -int batch_add_vector(ot_batch *batch, const struct iovec *entries, size_t count); - -/* Queue a copy of buf, which the batch then owns. Returns 0, or -1. */ -int batch_add_copy(ot_batch *batch, const void *buf, size_t len); - -/* Free whatever has not left yet, along with the batch's own array */ -void batch_reset(ot_batch *batch); - -#endif diff --git a/ot_buf.c b/ot_buf.c deleted file mode 100644 index ecd3f8d..0000000 --- a/ot_buf.c +++ /dev/null @@ -1,39 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include - -/* Opentracker */ -#include "ot_buf.h" - -int buf_cat(ot_buf *buf, const void *src, size_t len) { - char *data; - - if (!len) - return 0; - - if (buf->size + len < buf->size) - return -1; - - if (!(data = realloc(buf->data, buf->size + len))) - return -1; - - buf->data = data; - memcpy(data + buf->size, src, len); - buf->size += len; - - return 0; -} - -void buf_truncate(ot_buf *buf, size_t len) { - if (len < buf->size) - buf->size = len; -} - -void buf_reset(ot_buf *buf) { - free(buf->data); - buf->data = NULL; - buf->size = 0; -} diff --git a/ot_buf.h b/ot_buf.h deleted file mode 100644 index 7fb944e..0000000 --- a/ot_buf.h +++ /dev/null @@ -1,25 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_BUF_H__ -#define OT_BUF_H__ - -#include - -/* A growable byte buffer, holding an HTTP request that did not arrive in a - single packet. Requests that arrive whole are served straight from the - worker's input buffer and never come near this. */ -typedef struct { - char *data; - size_t size; -} ot_buf; - -/* Append len bytes from src. Returns 0, or -1 if the memory could not be - had, in which case the buffer keeps whatever it already held. */ -int buf_cat(ot_buf *buf, const void *src, size_t len); - -void buf_truncate(ot_buf *buf, size_t len); - -void buf_reset(ot_buf *buf); - -#endif diff --git a/ot_clean.c b/ot_clean.c deleted file mode 100644 index f181e83..0000000 --- a/ot_clean.c +++ /dev/null @@ -1,176 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include -#include -#include - -/* Opentracker */ -#include "ot_accesslist.h" -#include "ot_clean.h" -#include "ot_mutex.h" -#include "ot_stats.h" -#include "ot_vector.h" -#include "trackerlogic.h" - -/* Returns amount of removed peers */ -static ssize_t clean_single_bucket(ot_peer *peers, size_t peer_count, size_t peer_size, time_t timedout, size_t *removed_seeders) { - ot_peer *last_peer = peers + peer_count * peer_size, *insert_point; - - /* Two scan modes: unless there is one peer removed, just increase ot_peertime */ - while (peers < last_peer) { - time_t timediff = timedout + OT_PEERTIME(peers, peer_size); - if (timediff >= OT_PEER_TIMEOUT) - break; - OT_PEERTIME(peers, peer_size) = timediff; - peers += peer_size; - } - - /* If we at least remove one peer, we have to copy */ - for (insert_point = peers; peers < last_peer; peers += peer_size) { - time_t timediff = timedout + OT_PEERTIME(peers, peer_size); - - if (timediff < OT_PEER_TIMEOUT) { - OT_PEERTIME(peers, peer_size) = timediff; - memcpy(insert_point, peers, peer_size); - insert_point += peer_size; - } else if (OT_PEERFLAG_D(peers, peer_size) & PEER_FLAG_SEEDING) - (*removed_seeders)++; - } - - return (peers - insert_point) / peer_size; -} - -int clean_single_peer_list(ot_peerlist *peer_list, size_t peer_size) { - ot_vector *peer_vector = &peer_list->peers; - time_t timedout = (time_t)(time_now_minutes() - peer_list->base); - size_t num_buckets = 1, removed_seeders = 0; - - /* No need to clean empty torrent */ - if (!timedout) - return 0; - - /* Torrent has idled out */ - if (timedout > OT_TORRENT_TIMEOUT) - return 1; - - /* Nothing to be cleaned here? Test if torrent is worth keeping */ - if (timedout > OT_PEER_TIMEOUT) { - if (!peer_list->peer_count) - return peer_list->down_count ? 0 : 1; - timedout = OT_PEER_TIMEOUT; - } - - if (OT_PEERLIST_HASBUCKETS(peer_list)) { - num_buckets = peer_vector->size; - peer_vector = (ot_vector *)peer_vector->data; - } - - while (num_buckets--) { - size_t removed_peers = clean_single_bucket(peer_vector->data, peer_vector->size, peer_size, timedout, &removed_seeders); - peer_list->peer_count -= removed_peers; - peer_vector->size -= removed_peers; - if (removed_peers) - vector_fixup_peers(peer_vector, peer_size); - - /* Skip to next bucket, a vector containing peers */ - ++peer_vector; - } - - peer_list->seed_count -= removed_seeders; - - /* See if we need to convert a torrent from simple vector to bucket list */ - if ((peer_list->peer_count > OT_PEER_BUCKET_MINCOUNT) || OT_PEERLIST_HASBUCKETS(peer_list)) - vector_redistribute_buckets(peer_list, peer_size); - - if (peer_list->peer_count) - peer_list->base = time_now_minutes(); - else { - /* When we got here, the last time that torrent - has been touched is OT_PEER_TIMEOUT Minutes before */ - peer_list->base = time_now_minutes() - OT_PEER_TIMEOUT; - } - return 0; -} - -/* Clean a single torrent - return 1 if torrent timed out -*/ -int clean_single_torrent(ot_torrent *torrent) { - return clean_single_peer_list(torrent->peer_list6, OT_PEER_SIZE6) * clean_single_peer_list(torrent->peer_list4, OT_PEER_SIZE4); -} - -static pthread_mutex_t g_clean_mutex = PTHREAD_MUTEX_INITIALIZER; -static pthread_cond_t g_clean_wakeup = PTHREAD_COND_INITIALIZER; - -static void clean_sleep(void) { - struct timespec deadline; - - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OT_CLEAN_SLEEP / 1000000; - deadline.tv_nsec += (OT_CLEAN_SLEEP % 1000000) * 1000; - if (deadline.tv_nsec >= 1000000000) { - ++deadline.tv_sec; - deadline.tv_nsec -= 1000000000; - } - - /* Both sides hold the mutex, so the flag cannot be cleared and signalled - in the window between testing it and waiting on it */ - pthread_mutex_lock(&g_clean_mutex); - while (g_opentracker_running) - if (pthread_cond_timedwait(&g_clean_wakeup, &g_clean_mutex, &deadline) == ETIMEDOUT) - break; - pthread_mutex_unlock(&g_clean_mutex); -} - -/* Clean up all peers in one bucket, remove timedout pools and torrents */ -static void clean_bucket(size_t bucket) { - ot_vector *torrents_list = mutex_bucket_lock((int)bucket); - size_t toffs; - int delta_torrentcount = 0; - - for (toffs = 0; toffs < torrents_list->size; ++toffs) { - ot_torrent *torrent = ((ot_torrent *)(torrents_list->data)) + toffs; - if (clean_single_torrent(torrent)) { - vector_remove_torrent(torrents_list, torrent); - --delta_torrentcount; - --toffs; - } - } - mutex_bucket_unlock((int)bucket, delta_torrentcount); -} - -static void *clean_worker(void *args) { - (void)args; - while (1) { - size_t bucket = OT_BUCKET_COUNT; - while (bucket) { - size_t todo = bucket < OT_CLEAN_BUCKETS_PER_WAKE ? bucket : OT_CLEAN_BUCKETS_PER_WAKE; - - /* One bucket is locked at a time, however many a wake gets through */ - while (todo--) - clean_bucket(--bucket); - - if (!g_opentracker_running) - return NULL; - clean_sleep(); - } - stats_cleanup(); -#ifdef WANT_ACCESSLIST - accesslist_cleanup(); -#endif - } - return NULL; -} - -static pthread_t thread_id; -void clean_init(void) { pthread_create(&thread_id, NULL, clean_worker, NULL); } - -void clean_deinit(void) { - pthread_mutex_lock(&g_clean_mutex); - pthread_cond_signal(&g_clean_wakeup); - pthread_mutex_unlock(&g_clean_mutex); - pthread_join(thread_id, NULL); -} diff --git a/ot_clean.h b/ot_clean.h deleted file mode 100644 index 0592791..0000000 --- a/ot_clean.h +++ /dev/null @@ -1,28 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_CLEAN_H__ -#define OT_CLEAN_H__ - -#include "trackerlogic.h" - -/* The amount of time a clean cycle should take */ -#define OT_CLEAN_INTERVAL_MINUTES 2 - -/* How often the cleaner may wake. A full sweep takes OT_CLEAN_INTERVAL_MINUTES. - The clean operations per wake is goverend by OT_BUCKET_COUNT_BITS, - set by the Makefile. - At the default of 1024 buckets this works out at one bucket every - 117 ms. */ -#define OT_CLEAN_MAX_WAKES_PER_SECOND 10 - -#define OT_CLEAN_WAKE_BUDGET ((OT_CLEAN_INTERVAL_MINUTES) * 60 * (OT_CLEAN_MAX_WAKES_PER_SECOND)) -#define OT_CLEAN_BUCKETS_PER_WAKE (((OT_BUCKET_COUNT) + (OT_CLEAN_WAKE_BUDGET) - 1) / (OT_CLEAN_WAKE_BUDGET)) -#define OT_CLEAN_WAKES ((OT_BUCKET_COUNT) / (OT_CLEAN_BUCKETS_PER_WAKE)) -#define OT_CLEAN_SLEEP (((OT_CLEAN_INTERVAL_MINUTES) * 60 * 1000000) / (OT_CLEAN_WAKES)) - -void clean_init(void); -void clean_deinit(void); -int clean_single_torrent(ot_torrent *torrent); - -#endif diff --git a/ot_config.c b/ot_config.c deleted file mode 100644 index 95136b5..0000000 --- a/ot_config.c +++ /dev/null @@ -1,586 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include -#include -#include -#include -#include -#include - -/* Opentracker */ -#include "ot_accesslist.h" -#include "ot_event.h" -#include "ot_config.h" -#include "ot_http.h" -#include "ot_ip.h" -#include "ot_livesync.h" -#include "ot_scan.h" -#include "ot_stats.h" -#include "trackerlogic.h" - -const char *g_redirecturl; -const char *g_serverdir; -const char *g_serveruser; -const char *g_statefile; - -#define OT_LISTEN_INHERIT (-1) - -static ot_listen *g_listen; -static size_t g_listen_count; -static int g_listen_shared; -static int g_listen_workers_udp = 1; -static int g_listen_workers_tcp = 0; - -static void panic(const char *routine) __attribute__((noreturn)); -static void panic(const char *routine) { - fprintf(stderr, "%s: %s\n", routine, strerror(errno)); - exit(111); -} - -static void listen_append(const ot_ip6 ip, uint16_t port, PROTO_FLAG proto, int shared, int workers) { - ot_listen *grown = realloc(g_listen, (g_listen_count + 1) * sizeof(ot_listen)); - ot_listen *entry; - - if (!grown) - panic("listen_append"); - g_listen = grown; - - entry = g_listen + g_listen_count++; - memcpy(entry->ip, ip, sizeof(ot_ip6)); - entry->port = port; - entry->proto = proto; - entry->shared = shared; - entry->workers = workers; - entry->sock = -1; -} - -typedef enum { LISTEN_OPTION_UNKNOWN, LISTEN_OPTION_SHARED, LISTEN_OPTION_WORKERS } listen_option_kind; - -typedef struct { - const char *name; - size_t len; - listen_option_kind kind; - int takes_value; -} listen_option_key; - -#define LISTEN_OPTION_KEY(s, k, v) {s, sizeof(s) - 1, k, v} - -static const listen_option_key listen_option_keys[] = { - LISTEN_OPTION_KEY("shared", LISTEN_OPTION_SHARED, 0), - LISTEN_OPTION_KEY("workers", LISTEN_OPTION_WORKERS, 1), -}; -static const size_t listen_option_count = sizeof(listen_option_keys) / sizeof(listen_option_keys[0]); - -static listen_option_kind parse_listen_option(char **p) { - size_t i; - - for (i = 0; i < listen_option_count; ++i) { - const listen_option_key *key = &listen_option_keys[i]; - char delimiter = (*p)[key->len]; - - if (strncmp(*p, key->name, key->len)) - continue; - if (key->takes_value ? delimiter != '=' : (delimiter && !isspace((unsigned char)delimiter))) - continue; - - *p += key->len + (key->takes_value ? 1 : 0); - return key->kind; - } - return LISTEN_OPTION_UNKNOWN; -} - -/* Options trailing an address in a listen statement, in any order */ -static int listen_scan_options(char *p, int *shared, int *workers) { - uint64_t value; - - for (;;) { - while (isspace((unsigned char)*p)) - ++p; - if (!*p) - return 1; - - switch (parse_listen_option(&p)) { - case LISTEN_OPTION_SHARED: - *shared = 1; - break; - case LISTEN_OPTION_WORKERS: { - size_t off = scan_uint(p, &value, UINT_MAX); - - if (!off) - return 0; - *workers = (int)value; - p += off; - break; - } - default: - return 0; - } - } -} - - -static const char *set_config_option(const char **option, char *value) { -#ifdef _DEBUG - fprintf(stderr, "Setting config option: %s\n", value); -#endif - while (isspace(*value)) - ++value; - free((char *)*option); - return *option = strdup(value); -} - -static int scan_ip6_port(const char *src, ot_ip6 ip, uint16_t *port) { - const char *s = src; - uint64_t value; - int off, bracket = 0; - while (isspace(*s)) - ++s; - if (*s == '[') - ++s, ++bracket; /* for v6 style notation */ - if (!(off = ip_parse(s, ip))) - return 0; - s += off; - if (bracket && *s == ']') - ++s; - if (*s == 0 || isspace(*s)) - return s - src; - if (!ip_is_v4mapped(ip)) { - if (*s != ':' && *s != '.') - return 0; - if (!bracket && *(s) == ':') - return 0; - s++; - } else { - if (*(s++) != ':') - return 0; - } - if (!(off = scan_uint(s, &value, 65535))) - return 0; - *port = (uint16_t)value; - return off + s - src; -} - -static int scan_ip6_net(const char *src, ot_net *net) { - const char *s = src; - uint64_t value; - int off; - while (isspace(*s)) - ++s; - if (!(off = ip_parse(s, net->address))) - return 0; - s += off; - if (*s != '/') - net->bits = 128; - else { - s++; - if (!(off = scan_uint(s, &value, 128))) - return 0; - net->bits = (int)value; - if (ip_is_v4mapped(net->address)) - net->bits += 96; - if (net->bits > 128) - return 0; - s += off; - } - return off + s - src; -} - -typedef enum { - CONFIG_UNKNOWN, - CONFIG_TRACKER_ROOTDIR, - CONFIG_TRACKER_USER, - CONFIG_LISTEN_TCP_UDP, - CONFIG_LISTEN_TCP, - CONFIG_LISTEN_UDP, - CONFIG_LISTEN_UDP_WORKERS, - CONFIG_LISTEN_TCP_WORKERS, - CONFIG_LISTEN_SHARED, - CONFIG_ACCESS_WHITELIST, - CONFIG_ACCESS_BLACKLIST, - CONFIG_ACCESS_FIFO_ADD, - CONFIG_ACCESS_FIFO_DELETE, - CONFIG_ACCESS_STATS, - CONFIG_ACCESS_STATS_PATH, - CONFIG_ACCESS_PROXY, - CONFIG_TRACKER_REDIRECT_URL, - CONFIG_LIVESYNC_CLUSTER_NODE_IP, - CONFIG_LIVESYNC_CLUSTER_LISTEN, -} config_kind; - -typedef struct { - const char *name; - size_t len; - config_kind kind; -} config_key; - -#define CONFIG_KEY(s, k) { s, sizeof(s) - 1, k } - -static const config_key config_keys[] = { - CONFIG_KEY("tracker.rootdir", CONFIG_TRACKER_ROOTDIR), - CONFIG_KEY("tracker.user", CONFIG_TRACKER_USER), - CONFIG_KEY("listen.tcp_udp", CONFIG_LISTEN_TCP_UDP), - CONFIG_KEY("listen.tcp", CONFIG_LISTEN_TCP), - CONFIG_KEY("listen.udp", CONFIG_LISTEN_UDP), - CONFIG_KEY("listen.udp.workers", CONFIG_LISTEN_UDP_WORKERS), - CONFIG_KEY("listen.tcp.workers", CONFIG_LISTEN_TCP_WORKERS), - CONFIG_KEY("listen.shared", CONFIG_LISTEN_SHARED), -#ifdef WANT_ACCESSLIST_WHITE - CONFIG_KEY("access.whitelist", CONFIG_ACCESS_WHITELIST), -#elif defined(WANT_ACCESSLIST_BLACK) - CONFIG_KEY("access.blacklist", CONFIG_ACCESS_BLACKLIST), -#endif -#ifdef WANT_DYNAMIC_ACCESSLIST - CONFIG_KEY("access.fifo_add", CONFIG_ACCESS_FIFO_ADD), - CONFIG_KEY("access.fifo_delete", CONFIG_ACCESS_FIFO_DELETE), -#endif -#ifdef WANT_RESTRICT_STATS - CONFIG_KEY("access.stats", CONFIG_ACCESS_STATS), -#endif - CONFIG_KEY("access.stats_path", CONFIG_ACCESS_STATS_PATH), -#ifdef WANT_IP_FROM_PROXY - CONFIG_KEY("access.proxy", CONFIG_ACCESS_PROXY), -#endif - CONFIG_KEY("tracker.redirect_url", CONFIG_TRACKER_REDIRECT_URL), -#ifdef WANT_SYNC_LIVE - CONFIG_KEY("livesync.cluster.node_ip", CONFIG_LIVESYNC_CLUSTER_NODE_IP), - CONFIG_KEY("livesync.cluster.listen", CONFIG_LIVESYNC_CLUSTER_LISTEN), -#endif -}; -static const size_t config_entry_count = sizeof(config_keys) / sizeof(config_keys[0]); - -static config_kind parse_config_key(char **p) { - size_t i; - for (i = 0; i < config_entry_count; ++i) { - const config_key *key = &config_keys[i]; - if (strncmp(*p, key->name, key->len) == 0 && isspace((unsigned char)(*p)[key->len])) { - *p += key->len + 1; - /* Skip white spaces */ - while (isspace((unsigned char)**p)) - ++(*p); - return key->kind; - } - } - return CONFIG_UNKNOWN; -} - -static int parse_configfile(char *config_filename) { - FILE *config_file_handle; - char inbuf[512]; - ot_ip6 tmpip; -#if defined(WANT_RESTRICT_STATS) || defined(WANT_IP_FROM_PROXY) || defined(WANT_SYNC_LIVE) - ot_net tmpnet; -#endif - int bound = 0; - - config_file_handle = fopen(config_filename, "r"); - - if (config_file_handle == NULL) { - fprintf(stderr, "Warning: Can't open config file: %s.", config_filename); - return 0; - } - - while (fgets(inbuf, sizeof(inbuf), config_file_handle)) { - char *p = inbuf, *end; - uint64_t value; - uint16_t tmpport = 6969; - config_kind kind; - - /* Skip white spaces */ - while (isspace((unsigned char)*p)) - ++p; - - /* Ignore comments and empty lines */ - if ((*p == '#') || (*p == '\n') || (*p == 0)) - continue; - - /* consume trailing new lines and spaces */ - end = p + strlen(p); - while (end > p && isspace((unsigned char)end[-1])) - *--end = 0; - - /* Scan for commands */ - kind = parse_config_key(&p); - switch (kind) { - case CONFIG_TRACKER_ROOTDIR: - set_config_option(&g_serverdir, p); - break; - case CONFIG_TRACKER_USER: - set_config_option(&g_serveruser, p); - break; - case CONFIG_LISTEN_TCP_UDP: - case CONFIG_LISTEN_TCP: - case CONFIG_LISTEN_UDP: { - int shared = OT_LISTEN_INHERIT; - int workers = OT_LISTEN_INHERIT; - int off = scan_ip6_port(p, tmpip, &tmpport); - - if (!off || !listen_scan_options(p + off, &shared, &workers)) - goto parse_error; - if (kind != CONFIG_LISTEN_UDP) - listen_append(tmpip, tmpport, FLAG_TCP, shared, workers), ++bound; - if (kind != CONFIG_LISTEN_TCP) - listen_append(tmpip, tmpport, FLAG_UDP, shared, workers), ++bound; - break; - } - case CONFIG_LISTEN_UDP_WORKERS: - if (!scan_uint(p, &value, UINT_MAX)) - goto parse_error; - g_listen_workers_udp = (int)value; - break; - case CONFIG_LISTEN_TCP_WORKERS: - if (!scan_uint(p, &value, UINT_MAX)) - goto parse_error; - g_listen_workers_tcp = (int)value; - break; - case CONFIG_LISTEN_SHARED: - if (!scan_uint(p, &value, 1)) - goto parse_error; - g_listen_shared = (int)value; - break; -#ifdef WANT_ACCESSLIST_WHITE - case CONFIG_ACCESS_WHITELIST: - set_config_option(&g_accesslist_filename, p); - break; -#elif defined(WANT_ACCESSLIST_BLACK) - case CONFIG_ACCESS_BLACKLIST: - set_config_option(&g_accesslist_filename, p); - break; -#endif -#ifdef WANT_DYNAMIC_ACCESSLIST - case CONFIG_ACCESS_FIFO_ADD: - set_config_option(&g_accesslist_pipe_add, p); - break; - case CONFIG_ACCESS_FIFO_DELETE: - set_config_option(&g_accesslist_pipe_delete, p); - break; -#endif -#ifdef WANT_RESTRICT_STATS - case CONFIG_ACCESS_STATS: - if (!scan_ip6_net(p, &tmpnet)) - goto parse_error; - accesslist_bless_net(&tmpnet, OT_PERMISSION_MAY_STAT); - break; -#endif - case CONFIG_ACCESS_STATS_PATH: - set_config_option(&g_stats_path, p); - break; -#ifdef WANT_IP_FROM_PROXY - case CONFIG_ACCESS_PROXY: - if (!scan_ip6_net(p, &tmpnet)) - goto parse_error; - accesslist_bless_net(&tmpnet, OT_PERMISSION_MAY_PROXY); - break; -#endif - case CONFIG_TRACKER_REDIRECT_URL: - set_config_option(&g_redirecturl, p); - break; -#ifdef WANT_SYNC_LIVE - case CONFIG_LIVESYNC_CLUSTER_NODE_IP: - if (!scan_ip6_net(p, &tmpnet)) - goto parse_error; - accesslist_bless_net(&tmpnet, OT_PERMISSION_MAY_LIVESYNC); - break; - case CONFIG_LIVESYNC_CLUSTER_LISTEN: - tmpport = LIVESYNC_PORT; - { - int shared = OT_LISTEN_INHERIT, dummy = OT_LISTEN_INHERIT; - int off = scan_ip6_port(p, tmpip, &tmpport); - - if (!off || !listen_scan_options(p + off, &shared, &dummy)) - goto parse_error; - listen_append(tmpip, tmpport, FLAG_MCA, shared, OT_LISTEN_INHERIT); - } - break; -#endif - default: - fprintf(stderr, "Unhandled line in config file: %s\n", inbuf); - break; - parse_error: - fprintf(stderr, "Parse error in config file: %s\n", inbuf); - } - } - fclose(config_file_handle); - return bound; -} - -static void usage(char *name) { - fprintf(stderr, - "Usage: %s [-i ip] [-p port] [-P port] [-r redirect] [-d dir] [-u user] [-A ip[/bits]] [-f config] [-s livesyncport]" -#ifdef WANT_ACCESSLIST_BLACK - " [-b blacklistfile]" -#elif defined(WANT_ACCESSLIST_WHITE) - " [-w whitelistfile]" -#endif - "\n", - name); -} - -#define HELPLINE(opt, desc) fprintf(stderr, "\t%-10s%s\n", opt, desc) -static void help(char *name) { - usage(name); - - HELPLINE("-f config", "include and execute the config file"); - HELPLINE("-i ip", "specify ip to bind to with next -[pP] (default: any, overrides preceeding ones)"); - HELPLINE("-p port", "do bind to tcp port (default: 6969, you may specify more than one)"); - HELPLINE("-P port", "do bind to udp port (default: 6969, you may specify more than one)"); - HELPLINE("-r redirecturl", "specify url where / should be redirected to (default none)"); - HELPLINE("-d dir", "specify directory to try to chroot to (default: \".\")"); - HELPLINE("-u user", "specify user under whose privileges opentracker should run (default: \"nobody\")"); - HELPLINE("-A ip[/bits]", "bless an ip address or net as admin address (e.g. to allow syncs from this address)"); - HELPLINE("-v", "report version, event mechanism and the feature flags opentracker was built with"); -#ifdef WANT_ACCESSLIST_BLACK - HELPLINE("-b file", "specify blacklist file."); -#elif defined(WANT_ACCESSLIST_WHITE) - HELPLINE("-w file", "specify whitelist file."); -#endif - - fprintf(stderr, "\nExample: ./opentracker -i 127.0.0.1 -p 6969 -P 6969 -f ./opentracker.conf -i 10.1.1.23 -p 2710 -p 80\n"); - fprintf(stderr, " Here -i 127.0.0.1 selects the ip address for the next -p 6969 and -P 6969.\n"); - fprintf(stderr, " If no port is bound from config file or command line, the last address given\n"); - fprintf(stderr, " (or ::1 if none is set) will be used on port 6969.\n"); -} -#undef HELPLINE - - -static void listen_resolve_defaults(void) { - size_t i; - - for (i = 0; i < g_listen_count; ++i) { - ot_listen *entry = g_listen + i; - - if (entry->shared == OT_LISTEN_INHERIT) - entry->shared = g_listen_shared; - if (entry->workers == OT_LISTEN_INHERIT) - entry->workers = entry->proto == FLAG_TCP ? g_listen_workers_tcp : g_listen_workers_udp; - - /* A tcp worker hands a connection it cannot finish to the event loop, - which only one thread may touch where the set is ours rather than the - kernel's */ - if (entry->proto == FLAG_TCP && entry->workers > 1 && !event_can_share()) { - fprintf(stderr, "Warning: Multiple workers impossible on this platform with event mechanism %s chosen.\n", - event_mechanism()); - entry->workers = 1; - } - } -} - -ot_listen *config_listen_entries(size_t *count) { - *count = g_listen_count; - return g_listen; -} - -void config_parse(int argc, char **argv) { - ot_ip6 serverip; - ot_net tmpnet; - int bound = 0, scanon = 1; - uint64_t value; - - memset(serverip, 0, sizeof(ot_ip6)); -#ifdef WANT_V4_ONLY - serverip[10] = serverip[11] = -1; -#endif - - while (scanon) { - switch (getopt(argc, argv, - ":i:p:A:P:d:u:r:s:f:l:v" -#ifdef WANT_ACCESSLIST_BLACK - "b:" -#elif defined(WANT_ACCESSLIST_WHITE) - "w:" -#endif - "h")) { - case -1: - scanon = 0; - break; - case 'i': - if (!ip_parse(optarg, serverip)) { - usage(argv[0]); - exit(1); - } - break; -#ifdef WANT_ACCESSLIST_BLACK - case 'b': - set_config_option(&g_accesslist_filename, optarg); - break; -#elif defined(WANT_ACCESSLIST_WHITE) - case 'w': - set_config_option(&g_accesslist_filename, optarg); - break; -#endif - case 'p': - if (!scan_uint(optarg, &value, 65535)) { - usage(argv[0]); - exit(1); - } - listen_append(serverip, (uint16_t)value, FLAG_TCP, OT_LISTEN_INHERIT, OT_LISTEN_INHERIT); - bound++; - break; - case 'P': - if (!scan_uint(optarg, &value, 65535)) { - usage(argv[0]); - exit(1); - } - listen_append(serverip, (uint16_t)value, FLAG_UDP, OT_LISTEN_INHERIT, OT_LISTEN_INHERIT); - bound++; - break; -#ifdef WANT_SYNC_LIVE - case 's': - if (!scan_uint(optarg, &value, 65535)) { - usage(argv[0]); - exit(1); - } - listen_append(serverip, (uint16_t)value, FLAG_MCA, OT_LISTEN_INHERIT, OT_LISTEN_INHERIT); - break; -#endif - case 'd': - set_config_option(&g_serverdir, optarg); - break; - case 'u': - set_config_option(&g_serveruser, optarg); - break; - case 'r': - set_config_option(&g_redirecturl, optarg); - break; - case 'l': - g_statefile = optarg; - break; - case 'A': - if (!scan_ip6_net(optarg, &tmpnet)) { - usage(argv[0]); - exit(1); - } - accesslist_bless_net(&tmpnet, 0xffff); /* Allow everything for now */ - break; - case 'f': - bound += parse_configfile(optarg); - break; - case 'h': - help(argv[0]); - exit(0); - case 'v': { - char buffer[8192]; - char *cursor = buffer; - - stats_return_tracker_version(&cursor, buffer + sizeof(buffer) - 1); - stats_return_tracker_features(&cursor, buffer + sizeof(buffer) - 1); - *cursor = 0; - fputs(buffer, stderr); - exit(0); - } - default: - case '?': - usage(argv[0]); - exit(1); - } - } - - if (!bound) { - listen_append(serverip, 6969, FLAG_TCP, OT_LISTEN_INHERIT, OT_LISTEN_INHERIT); - listen_append(serverip, 6969, FLAG_UDP, OT_LISTEN_INHERIT, OT_LISTEN_INHERIT); - } - - listen_resolve_defaults(); -} diff --git a/ot_config.h b/ot_config.h deleted file mode 100644 index 79187ee..0000000 --- a/ot_config.h +++ /dev/null @@ -1,33 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_CONFIG_H__ -#define OT_CONFIG_H__ - -#include - -#include "trackerlogic.h" - -/* An address opentracker was asked to listen on. Options the statement did - not name have been filled in from the file wide defaults by the time these - are handed out. sock is filled in by whoever binds them. */ -typedef struct { - ot_ip6 ip; - uint16_t port; - PROTO_FLAG proto; - int shared; - int workers; - int64_t sock; -} ot_listen; - -extern const char *g_redirecturl; -extern const char *g_serverdir; -extern const char *g_serveruser; -extern const char *g_statefile; - -/* Reads the command line and every config file it names. Exits on error. */ -void config_parse(int argc, char **argv); - -ot_listen *config_listen_entries(size_t *count); - -#endif diff --git a/ot_event.c b/ot_event.c deleted file mode 100644 index 6062f70..0000000 --- a/ot_event.c +++ /dev/null @@ -1,357 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include -#include -#include - -/* Opentracker */ -#include "ot_event.h" - -/* One caller waits, so the scratch the mechanism needs is kept here rather - than allocated per call. */ - -/* poll works everywhere and is what the BSDs get until kqueue is written. - Defining OT_EVENT_FORCE_POLL picks it on a platform that would otherwise - use something better, which is how both are exercised by the tests. */ -#if defined(OT_EVENT_FORCE_POLL) -#define OT_EVENT_POLL -#elif defined(OT_EVENT_FORCE_KQUEUE) -#define OT_EVENT_KQUEUE -#elif defined(__linux__) -#define OT_EVENT_EPOLL -#elif defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__DragonFly__) || defined(__APPLE__) -#define OT_EVENT_KQUEUE -#else -#define OT_EVENT_POLL -#endif - -#ifdef OT_EVENT_EPOLL -/* ------------------------------------------------------------- epoll */ - -#include - -static int g_epoll = -1; -static struct epoll_event *g_ready; -static int g_ready_space; - -const char *event_mechanism(void) { return "epoll"; } -/* The set lives in the kernel, and epoll_ctl during another thread's - epoll_wait is what it is for */ -int event_can_share(void) { return 1; } - -int event_init(void) { - g_ready = NULL; - g_ready_space = 0; - return (g_epoll = epoll_create1(EPOLL_CLOEXEC)) == -1 ? -1 : 0; -} - -void event_deinit(void) { - if (g_epoll != -1) - close(g_epoll); - free(g_ready); - g_epoll = -1; - g_ready = NULL; - g_ready_space = 0; -} - -static int event_ctl(int op, int fd, int what, uint32_t generation) { - struct epoll_event ev; - - memset(&ev, 0, sizeof(ev)); - ev.events = (what & EVENT_IN ? (uint32_t)EPOLLIN : 0) | (what & EVENT_OUT ? (uint32_t)EPOLLOUT : 0); - /* The generation rides along inside the kernel, so an event says which - connection it belongs to without anybody looking it up */ - ev.data.u64 = ((uint64_t)generation << 32) | (uint32_t)fd; - return epoll_ctl(g_epoll, op, fd, &ev); -} - -int event_add(int fd, int what, uint32_t generation) { return event_ctl(EPOLL_CTL_ADD, fd, what, generation); } -int event_modify(int fd, int what, uint32_t generation) { return event_ctl(EPOLL_CTL_MOD, fd, what, generation); } -int event_del(int fd) { return epoll_ctl(g_epoll, EPOLL_CTL_DEL, fd, NULL); } - -/* A descriptor leaves the interest list when the last reference to its open - file description is closed, and the tracker never duplicates one */ -int event_close(int fd) { return close(fd); } - -int event_wait(ot_event *out, int max, int timeout_ms) { - int count, i; - - if (max > g_ready_space) { - struct epoll_event *grown = realloc(g_ready, (size_t)max * sizeof(struct epoll_event)); - - if (!grown) - return -1; - g_ready = grown; - g_ready_space = max; - } - - /* An interrupted wait is reported as no events rather than retried here, - so that the caller gets to look at whatever the signal was about */ - if ((count = epoll_wait(g_epoll, g_ready, max, timeout_ms)) == -1) - return errno == EINTR ? 0 : -1; - for (i = 0; i < count; ++i) { - uint32_t events = g_ready[i].events; - - out[i].fd = (int)(uint32_t)g_ready[i].data.u64; - out[i].generation = (uint32_t)(g_ready[i].data.u64 >> 32); - /* A hangup or an error is reported as both, so whichever side the caller - was waiting on gets to notice */ - out[i].what = (events & (EPOLLIN | EPOLLHUP | EPOLLERR) ? EVENT_IN : 0) | - (events & (EPOLLOUT | EPOLLHUP | EPOLLERR) ? EVENT_OUT : 0); - } - return count; -} - -#endif /* OT_EVENT_EPOLL */ - -#ifdef OT_EVENT_KQUEUE -/* ------------------------------------------------------------ kqueue */ - -#include -#include -#include - -static int g_kqueue = -1; -static struct kevent *g_ready; -static int g_ready_space; - -const char *event_mechanism(void) { return "kqueue"; } -/* The set lives in the kernel, and kevent may be called from any thread */ -int event_can_share(void) { return 1; } - -int event_init(void) { - g_ready = NULL; - g_ready_space = 0; - return (g_kqueue = kqueue()) == -1 ? -1 : 0; -} - -void event_deinit(void) { - if (g_kqueue != -1) - close(g_kqueue); - free(g_ready); - g_kqueue = -1; - g_ready = NULL; - g_ready_space = 0; -} - -/* kqueue keeps one registration per filter, so reading and writing are two - entries. A change of interest is an add of the one wanted and a delete - of the one not. The generation travels in udata, which the kernel passes - back untouched. */ -static int event_change(int fd, int what, uint32_t generation, int deleting) { - struct kevent change[2], result[2]; - void *udata = (void *)(uintptr_t)(((uint64_t)generation << 32) | (uint32_t)fd); - int succeeded = 0, count, i; - - EV_SET(&change[0], (uintptr_t)fd, EVFILT_READ, - EV_RECEIPT | ((deleting || !(what & EVENT_IN)) ? EV_DELETE : EV_ADD), 0, 0, udata); - EV_SET(&change[1], (uintptr_t)fd, EVFILT_WRITE, - EV_RECEIPT | ((deleting || !(what & EVENT_OUT)) ? EV_DELETE : EV_ADD), 0, 0, udata); - - if ((count = kevent(g_kqueue, change, 2, result, 2, NULL)) == -1) - return -1; - - for (i = 0; i < count && i < 2; ++i) { - if (!result[i].data) { - ++succeeded; - continue; - } - /* Dropping interest we never registered is not a failure */ - if ((change[i].flags & EV_DELETE) && result[i].data == ENOENT) - continue; - errno = (int)result[i].data; - return -1; - } - - /* Nothing at all was registered or removed, so the descriptor is unknown */ - if (!succeeded) { - errno = ENOENT; - return -1; - } - return 0; -} - -int event_add(int fd, int what, uint32_t generation) { return what ? event_change(fd, what, generation, 0) : 0; } -int event_modify(int fd, int what, uint32_t generation) { return event_change(fd, what, generation, 0); } -int event_del(int fd) { return event_change(fd, 0, 0, 1); } - -/* Closing a descriptor removes every kevent that references it */ -int event_close(int fd) { return close(fd); } - -int event_wait(ot_event *out, int max, int timeout_ms) { - struct timespec timeout; - struct timespec *deadline = NULL; - int count, i; - - if (max > g_ready_space) { - struct kevent *grown = realloc(g_ready, (size_t)max * sizeof(struct kevent)); - - if (!grown) - return -1; - g_ready = grown; - g_ready_space = max; - } - - if (timeout_ms >= 0) { - timeout.tv_sec = timeout_ms / 1000; - timeout.tv_nsec = (long)(timeout_ms % 1000) * 1000000; - deadline = &timeout; - } - - /* An interrupted wait is reported as no events rather than retried here, - so that the caller gets to look at whatever the signal was about */ - if ((count = kevent(g_kqueue, NULL, 0, g_ready, max, deadline)) == -1) - return errno == EINTR ? 0 : -1; - for (i = 0; i < count; ++i) { - uint64_t udata = (uint64_t)(uintptr_t)g_ready[i].udata; - int eof = (g_ready[i].flags & (EV_EOF | EV_ERROR)) != 0; - - out[i].fd = (int)(uint32_t)udata; - out[i].generation = (uint32_t)(udata >> 32); - out[i].what = ((g_ready[i].filter == EVFILT_READ || eof) ? EVENT_IN : 0) | - ((g_ready[i].filter == EVFILT_WRITE || eof) ? EVENT_OUT : 0); - } - return count; -} - -#endif /* OT_EVENT_KQUEUE */ - -#ifdef OT_EVENT_POLL -/* -------------------------------------------------------------- poll */ - -#include - -/* poll wants its descriptors packed, so the registered ones live in one - array and a second maps a descriptor number to its place in it. */ -static struct pollfd *g_pfds; -static uint32_t *g_generation; -static int *g_where; -static int g_count, g_space, g_where_space; - -const char *event_mechanism(void) { return "poll"; } -/* The arrays below are ours and nothing guards them */ -int event_can_share(void) { return 0; } - -int event_init(void) { - g_pfds = NULL, g_generation = NULL, g_where = NULL; - g_count = g_space = g_where_space = 0; - return 0; -} - -void event_deinit(void) { - free(g_pfds), free(g_generation), free(g_where); - g_pfds = NULL, g_generation = NULL, g_where = NULL; - g_count = g_space = g_where_space = 0; -} - -static short event_to_poll(int what) { - return (short)((what & EVENT_IN ? POLLIN : 0) | (what & EVENT_OUT ? POLLOUT : 0)); -} - -static int event_room_for(int fd) { - if (fd >= g_where_space) { - int space = g_where_space ? g_where_space : 64; - int *grown; - - while (space <= fd) - space *= 2; - if (!(grown = realloc(g_where, (size_t)space * sizeof(int)))) - return -1; - while (g_where_space < space) - grown[g_where_space++] = -1; - g_where = grown; - } - - if (g_count == g_space) { - int space = g_space ? g_space * 2 : 64; - struct pollfd *pfds = realloc(g_pfds, (size_t)space * sizeof(struct pollfd)); - uint32_t *gens; - - if (!pfds) - return -1; - g_pfds = pfds; - if (!(gens = realloc(g_generation, (size_t)space * sizeof(uint32_t)))) - return -1; - g_generation = gens; - g_space = space; - } - return 0; -} - -int event_add(int fd, int what, uint32_t generation) { - if (fd < 0 || event_room_for(fd) == -1) - return -1; - if (g_where[fd] != -1) { - errno = EEXIST; - return -1; - } - - g_pfds[g_count].fd = fd; - g_pfds[g_count].events = event_to_poll(what); - g_pfds[g_count].revents = 0; - g_generation[g_count] = generation; - g_where[fd] = g_count++; - return 0; -} - -int event_modify(int fd, int what, uint32_t generation) { - if (fd < 0 || fd >= g_where_space || g_where[fd] == -1) { - errno = ENOENT; - return -1; - } - g_pfds[g_where[fd]].events = event_to_poll(what); - g_generation[g_where[fd]] = generation; - return 0; -} - -int event_del(int fd) { - int at; - - if (fd < 0 || fd >= g_where_space || (at = g_where[fd]) == -1) { - errno = ENOENT; - return -1; - } - - /* The last one moves into the gap, so the array stays packed */ - g_pfds[at] = g_pfds[--g_count]; - g_generation[at] = g_generation[g_count]; - g_where[g_pfds[at].fd] = at; - g_where[fd] = -1; - return 0; -} - -/* Nothing here lives in the kernel, so the entry has to come out by hand - before the descriptor goes, or poll reports POLLNVAL for it forever */ -int event_close(int fd) { - event_del(fd); - return close(fd); -} - -int event_wait(ot_event *out, int max, int timeout_ms) { - int ready, found = 0, i; - - /* An interrupted wait is reported as no events rather than retried here, - so that the caller gets to look at whatever the signal was about */ - if ((ready = poll(g_pfds, (nfds_t)g_count, timeout_ms)) == -1) - return errno == EINTR ? 0 : -1; - if (ready <= 0) - return ready; - - for (i = 0; i < g_count && found < max; ++i) { - short revents = g_pfds[i].revents; - - if (!revents) - continue; - out[found].fd = g_pfds[i].fd; - out[found].generation = g_generation[i]; - out[found].what = (revents & (POLLIN | POLLHUP | POLLERR | POLLNVAL) ? EVENT_IN : 0) | - (revents & (POLLOUT | POLLHUP | POLLERR | POLLNVAL) ? EVENT_OUT : 0); - ++found; - } - return found; -} - -#endif /* OT_EVENT_POLL */ diff --git a/ot_event.h b/ot_event.h deleted file mode 100644 index cb84ca8..0000000 --- a/ot_event.h +++ /dev/null @@ -1,47 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_EVENT_H__ -#define OT_EVENT_H__ - -#include - -/* Which descriptors have become readable or writable, asked of whichever - mechanism the kernel offers. The generation a descriptor was registered - with is handed back with every event, so a caller can tell an event for - the connection it is holding from one for a connection that has since been - closed and had its number reused. */ - -enum { EVENT_IN = 1, EVENT_OUT = 2 }; - -typedef struct { - int fd; - uint32_t generation; - int what; -} ot_event; - -int event_init(void); -void event_deinit(void); - -int event_add(int fd, int what, uint32_t generation); -int event_modify(int fd, int what, uint32_t generation); -int event_del(int fd); - -/* Stop watching a descriptor and close it. Where the kernel drops a closed - descriptor from its set by itself, this is one system call where doing the - two separately would be two. */ -int event_close(int fd); - -/* Waits at most timeout_ms, or indefinitely when that is negative, and fills - out with up to max ready descriptors. Returns how many, or -1. */ -int event_wait(ot_event *out, int max, int timeout_ms); - -/* Whether the registered set may be changed from another thread while this - one is waiting on it. True where the kernel keeps the set, false where - this module keeps it in memory of its own. */ -int event_can_share(void); - -/* Which mechanism was compiled in, for the version string */ -const char *event_mechanism(void); - -#endif diff --git a/ot_fdstate.c b/ot_fdstate.c deleted file mode 100644 index 8e4bd89..0000000 --- a/ot_fdstate.c +++ /dev/null @@ -1,202 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include -#include -#include -#include -#include - -/* Opentracker */ -#include "ot_fdstate.h" - -enum { FD_FREE, FD_IDLE, FD_BUSY, FD_CLOSING }; - -/* Generation and state share one word so that taking the state is also a - check that the generation has not moved */ -#define FD_GEN(w) ((uint32_t)((w) >> 32)) -#define FD_ST(w) ((uint32_t)(((w) >> 24) & 0xff)) -#define FD_WORD(g, s) (((uint64_t)(g) << 32) | ((uint64_t)(s) << 24)) -#define FD_SET_ST(w, s) (((w) & ~(uint64_t)0xff000000) | ((uint64_t)(s) << 24)) - -typedef struct { - _Atomic uint64_t state; - _Atomic(void *) cookie; - _Atomic ot_time expiry; -} ot_fdslot; - -/* A descriptor the kernel can hand out is below the limit it was handed out - under, but the limit can be raised after we sized the table, so an out of - range descriptor is rejected */ -#define FD_SLOTS_MAX (1u << 20) - -static ot_fdslot *g_slots; -static size_t g_slot_count; -/* Descriptors are handed out lowest first, so a sweep only has to walk as - far as the highest one ever seen rather than the whole table. */ -static _Atomic size_t g_slot_highest; - -int fdstate_init(void) { - struct rlimit limit; - size_t count = FD_SLOTS_MAX; - - if (getrlimit(RLIMIT_NOFILE, &limit) == 0 && limit.rlim_cur != RLIM_INFINITY && limit.rlim_cur < FD_SLOTS_MAX) - count = (size_t)limit.rlim_cur; - - g_slots = mmap(NULL, count * sizeof(ot_fdslot), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (g_slots == MAP_FAILED) { - g_slots = NULL; - return -1; - } - - g_slot_count = count; - atomic_store(&g_slot_highest, 0); - return 0; -} - -void fdstate_deinit(void) { - if (g_slots) - munmap(g_slots, g_slot_count * sizeof(ot_fdslot)); - g_slots = NULL; - g_slot_count = 0; - atomic_store(&g_slot_highest, 0); -} - -size_t fdstate_slots(void) { return g_slot_count; } - -static ot_fdslot *fdstate_slot(int fd) { - if (fd < 0 || (size_t)fd >= g_slot_count) - return NULL; - return g_slots + fd; -} - -uint32_t fdstate_open(int fd, void *cookie, ot_time expiry) { - ot_fdslot *slot = fdstate_slot(fd); - uint64_t word; - - if (!slot) - return 0; - - { - size_t highest = atomic_load_explicit(&g_slot_highest, memory_order_relaxed); - - while ((size_t)fd > highest && - !atomic_compare_exchange_weak_explicit(&g_slot_highest, &highest, (size_t)fd, memory_order_relaxed, - memory_order_relaxed)) - ; - } - - word = atomic_load_explicit(&slot->state, memory_order_relaxed); - atomic_store_explicit(&slot->cookie, cookie, memory_order_release); - atomic_store_explicit(&slot->expiry, expiry, memory_order_relaxed); - /* Publishes the two above to whoever claims the slot next */ - atomic_store_explicit(&slot->state, FD_WORD(FD_GEN(word), FD_IDLE), memory_order_release); - - return FD_GEN(word); -} - -uint32_t fdstate_generation(int fd) { - ot_fdslot *slot = fdstate_slot(fd); - - return slot ? FD_GEN(atomic_load_explicit(&slot->state, memory_order_relaxed)) : 0; -} - -int fdstate_claim(int fd, uint32_t generation) { - ot_fdslot *slot = fdstate_slot(fd); - uint64_t word; - - if (!slot) - return 0; - - word = atomic_load_explicit(&slot->state, memory_order_acquire); - for (;;) { - if (FD_GEN(word) != generation || FD_ST(word) != FD_IDLE) - return 0; - if (atomic_compare_exchange_weak_explicit(&slot->state, &word, FD_SET_ST(word, FD_BUSY), memory_order_acquire, - memory_order_relaxed)) - return 1; - /* word now holds what was really there, so the test above runs again */ - } -} - -void fdstate_release(int fd) { - ot_fdslot *slot = fdstate_slot(fd); - uint64_t word; - - if (!slot) - return; - - word = atomic_load_explicit(&slot->state, memory_order_relaxed); - atomic_store_explicit(&slot->state, FD_SET_ST(word, FD_IDLE), memory_order_release); -} - -int fdstate_claim_expired(int fd, ot_time now) { - ot_fdslot *slot = fdstate_slot(fd); - uint64_t word; - ot_time expiry; - - if (!slot) - return 0; - - /* Reading before asking for the slot keeps the common case, a descriptor - that has not expired, off the bus entirely */ - word = atomic_load_explicit(&slot->state, memory_order_relaxed); - if (FD_ST(word) != FD_IDLE) - return 0; - expiry = atomic_load_explicit(&slot->expiry, memory_order_relaxed); - if (!expiry || expiry > now) - return 0; - - /* Nothing can move the expiry while the slot sits at FD_IDLE, so winning - the slot is the last word */ - return atomic_compare_exchange_strong_explicit(&slot->state, &word, FD_SET_ST(word, FD_CLOSING), memory_order_acquire, - memory_order_relaxed); -} - -void fdstate_close(int fd) { - ot_fdslot *slot = fdstate_slot(fd); - uint64_t word; - - if (!slot) - return; - - word = atomic_load_explicit(&slot->state, memory_order_relaxed); - atomic_store_explicit(&slot->cookie, NULL, memory_order_relaxed); - atomic_store_explicit(&slot->expiry, 0, memory_order_relaxed); - atomic_store_explicit(&slot->state, FD_WORD(FD_GEN(word) + 1, FD_FREE), memory_order_release); -} - -int fdstate_next_expired(ot_time now, int *cursor) { - size_t highest = atomic_load_explicit(&g_slot_highest, memory_order_relaxed); - - for (; (size_t)*cursor <= highest; ++*cursor) - if (fdstate_claim_expired(*cursor, now)) - return (*cursor)++; - - return -1; -} - -void *fdstate_cookie(int fd) { - ot_fdslot *slot = fdstate_slot(fd); - - /* Pairs with the release in fdstate_set_cookie and fdstate_open */ - return slot ? atomic_load_explicit(&slot->cookie, memory_order_acquire) : NULL; -} - -void fdstate_set_cookie(int fd, void *cookie) { - ot_fdslot *slot = fdstate_slot(fd); - - /* Publishes what the cookie points at to whoever reads it next, which may - be another thread entirely */ - if (slot) - atomic_store_explicit(&slot->cookie, cookie, memory_order_release); -} - -void fdstate_set_expiry(int fd, ot_time expiry) { - ot_fdslot *slot = fdstate_slot(fd); - - if (slot) - atomic_store_explicit(&slot->expiry, expiry, memory_order_relaxed); -} diff --git a/ot_fdstate.h b/ot_fdstate.h deleted file mode 100644 index 60fe070..0000000 --- a/ot_fdstate.h +++ /dev/null @@ -1,62 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_FDSTATE_H__ -#define OT_FDSTATE_H__ - -#include - -#include "trackerlogic.h" - -/* What the tracker knows about a file descriptor: who owns it, when it - should be given up on, and whether anyone is working on it right now. - - A descriptor number says nothing about identity, since the kernel hands - the same number out again as soon as it is closed. Every slot therefore - carries a generation that is bumped when the descriptor is given up, and - every caller that was told about a descriptor has to say which generation - it was told about. A caller holding a stale generation is turned away - instead of being handed somebody else's connection. - - The table is sized once from RLIMIT_NOFILE and never moves, so looking a - descriptor up needs no synchronisation at all. */ - -int fdstate_init(void); -void fdstate_deinit(void); -size_t fdstate_slots(void); - -/* Take a freshly accepted descriptor into the table and return the - generation to quote when asking for it back. expiry of 0 never expires. */ -uint32_t fdstate_open(int fd, void *cookie, ot_time expiry); - -/* Which generation a descriptor currently carries, to check an event that - was handed out earlier against the connection living there now. */ -uint32_t fdstate_generation(int fd); - -/* Ask for exclusive use. Returns 0 if the descriptor has been recycled since - generation was handed out, or if somebody else is using it. */ -int fdstate_claim(int fd, uint32_t generation); -void fdstate_release(int fd); - -/* The cleaner's side of the same thing: takes the descriptor only if nobody - is working on it and it really has expired. Never waits. */ -int fdstate_claim_expired(int fd, ot_time now); - -/* Give the descriptor up. The generation is bumped here, which has to happen - before close(2) so that the kernel cannot reissue the number while events - quoting the old generation are still in flight. */ -void fdstate_close(int fd); - -/* Walks the table for descriptors that have expired, handing back one at a - time until it returns -1. The cursor belongs to the caller, so several - sweepers can run without sharing anything. */ -int fdstate_next_expired(ot_time now, int *cursor); - -/* Both only valid while the caller holds the descriptor. */ -void *fdstate_cookie(int fd); - -/* Hand the slot a different cookie, for a connection whose state has moved */ -void fdstate_set_cookie(int fd, void *cookie); -void fdstate_set_expiry(int fd, ot_time expiry); - -#endif diff --git a/ot_fmt.c b/ot_fmt.c deleted file mode 100644 index 362c9b5..0000000 --- a/ot_fmt.c +++ /dev/null @@ -1,40 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include -#include - -/* Opentracker */ -#include "ot_fmt.h" - -void fmt_append(char **cursor, const char *end, const char *format, ...) { - va_list arguments; - size_t room; - int written; - - if (*cursor >= end) { - *cursor = (char *)end; - return; - } - - room = (size_t)(end - *cursor); - va_start(arguments, format); - written = vsnprintf(*cursor, room, format, arguments); - va_end(arguments); - - if (written < 0) - return; - *cursor = (size_t)written >= room ? (char *)end : *cursor + written; -} - -void fmt_bytes(char **cursor, const char *end, const void *bytes, size_t length) { - if (*cursor >= end || (size_t)(end - *cursor) < length) { - *cursor = (char *)end; - return; - } - - memcpy(*cursor, bytes, length); - *cursor += length; -} diff --git a/ot_fmt.h b/ot_fmt.h deleted file mode 100644 index 214cbc4..0000000 --- a/ot_fmt.h +++ /dev/null @@ -1,19 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_FMT_H__ -#define OT_FMT_H__ - -#include - -/* Appending to a cursor that walks a fixed buffer. Both take the cursor by - reference and advance it, so a reply is written as a run of statements and - no caller does pointer arithmetic of its own. */ - -void fmt_append(char **cursor, const char *end, const char *format, ...) __attribute__((format(printf, 3, 4))); - -/* Append raw bytes for the binary stretches of a bencoded reply. - Either everything is appended or nothing at all */ -void fmt_bytes(char **cursor, const char *end, const void *bytes, size_t length); - -#endif diff --git a/ot_fullscrape.c b/ot_fullscrape.c deleted file mode 100644 index 9afad92..0000000 --- a/ot_fullscrape.c +++ /dev/null @@ -1,455 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifdef WANT_FULLSCRAPE - -/* System */ -#include -#include -#include -#include -#ifdef WANT_COMPRESSION_GZIP -#include -#endif -#ifdef WANT_COMPRESSION_ZSTD -#include -#endif - -/* Opentracker */ -#include "ot_fmt.h" -#include "ot_hex.h" -#include "ot_fullscrape.h" -#include "ot_iovec.h" -#include "ot_mutex.h" -#include "ot_workqueue.h" -#include "ot_urlencoded.h" -#include "trackerlogic.h" - -/* Fetch full scrape info for all torrents - Full scrapes usually are huge and one does not want to - allocate more memory. So lets get them in 512k units -*/ -#define OT_SCRAPE_CHUNK_SIZE (1024 * 1024) - -/* "d8:completei%zde10:downloadedi%zde10:incompletei%zdee" */ -#define OT_SCRAPE_MAXENTRYLEN 256 - -/* Forward declaration */ -static void fullscrape_make(int taskid, ot_tasktype mode); -#ifdef WANT_COMPRESSION_GZIP -static void fullscrape_make_gzip(int taskid, ot_tasktype mode); -#endif -#ifdef WANT_COMPRESSION_ZSTD -static void fullscrape_make_zstd(int taskid, ot_tasktype mode); -#endif - -/* Converter function from memory to human readable hex strings - XXX - Duplicated from ot_stats. Needs fix. */ -/* This is the entry point into this worker thread - It grabs tasks from mutex_tasklist and delivers results back -*/ -static void *fullscrape_worker(void *args) { - (void)args; - - while (g_opentracker_running) { - ot_tasktype tasktype = TASK_FULLSCRAPE; - ot_taskid taskid = workqueue_poptask(&tasktype); - if (!taskid) - break; -#ifdef WANT_COMPRESSION_ZSTD - if (tasktype & TASK_FLAG_ZSTD) - fullscrape_make_zstd(taskid, tasktype); - else -#endif -#ifdef WANT_COMPRESSION_GZIP - if (tasktype & TASK_FLAG_GZIP) - fullscrape_make_gzip(taskid, tasktype); - else -#endif - fullscrape_make(taskid, tasktype); - workqueue_pushchunked(taskid, NULL); - } - return NULL; -} - -static pthread_t thread_id; -void fullscrape_init(void) { - pthread_create( &thread_id, NULL, fullscrape_worker, NULL ); -} - -void fullscrape_deinit(void) { - workqueue_wakeall(); - pthread_join( thread_id, NULL ); -} - -void fullscrape_deliver( int64_t sock, ot_tasktype tasktype ) { - workqueue_pushtask( sock, TASK_FULLSCRAPE | tasktype ); -} - -static void fullscrape_write_one( ot_tasktype mode, char **cursor, ot_torrent *torrent, ot_hash *hash ) { - /* Callers guarantee this much room: the chunk writer keeps its low - watermark that far from the end, the compressors hand over a buffer of - exactly that size */ - char *r = *cursor; - const char *r_end = r + OT_SCRAPE_MAXENTRYLEN; - size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; - size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; - size_t down_count = torrent->peer_list6->down_count + torrent->peer_list4->down_count; - - switch (mode & TASK_TASK_MASK) { - case TASK_FULLSCRAPE: - default: - /* push hash as bencoded string */ - *r++ = '2'; - *r++ = '0'; - *r++ = ':'; - memcpy(r, hash, sizeof(ot_hash)); - r += sizeof(ot_hash); - /* push rest of the scrape string */ - fmt_append(&r, r_end, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", seed_count, down_count, peer_count - seed_count); - - break; - case TASK_FULLSCRAPE_TPB_ASCII: - hex_from_hash(r, *hash); - r += 2 * sizeof(ot_hash); - fmt_append(&r, r_end, ":%zd:%zd\n", seed_count, peer_count - seed_count); - break; - case TASK_FULLSCRAPE_TPB_ASCII_PLUS: - hex_from_hash(r, *hash); - r += 2 * sizeof(ot_hash); - fmt_append(&r, r_end, ":%zd:%zd:%zd\n", seed_count, peer_count - seed_count, down_count); - break; - case TASK_FULLSCRAPE_TPB_BINARY: - memcpy(r, *hash, sizeof(ot_hash)); - r += sizeof(ot_hash); - *(uint32_t *)(r + 0) = htonl((uint32_t)seed_count); - *(uint32_t *)(r + 4) = htonl((uint32_t)(peer_count - seed_count)); - r += 8; - break; - case TASK_FULLSCRAPE_TPB_URLENCODED: - r += urlencoded_format(r, *hash, sizeof(ot_hash)); - fmt_append(&r, r_end, ":%zd:%zd\n", seed_count, peer_count - seed_count); - break; - case TASK_FULLSCRAPE_TRACKERSTATE: - hex_from_hash(r, *hash); - r += 2 * sizeof(ot_hash); - fmt_append(&r, r_end, ":%zd:%zd\n", torrent->peer_list6->base, down_count); - break; - } - *cursor = r; -} - -static void fullscrape_make(int taskid, ot_tasktype mode) { - size_t bucket; - char *r, *re; - struct iovec iovector = {NULL, 0}; - - /* Setup return vector... */ - r = iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); - if (!r) - return; - - /* re points to low watermark */ - re = r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN; - - if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) - fmt_append(&r, (char *)iovector.iov_base + OT_SCRAPE_CHUNK_SIZE, "d5:filesd"); - - /* For each bucket... */ - for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { - /* Get exclusive access to that bucket */ - ot_vector *torrents_list = mutex_bucket_lock(bucket); - ot_torrent *torrents = (ot_torrent *)(torrents_list->data); - size_t i; - - /* For each torrent in this bucket.. */ - for (i = 0; i < torrents_list->size; ++i) { - fullscrape_write_one(mode, &r, torrents + i, &torrents[i].hash); - - if (r > re) { - iovector.iov_len = r - (char *)iovector.iov_base; - - if (workqueue_pushchunked(taskid, &iovector)) { - free(iovector.iov_base); - mutex_bucket_unlock(bucket, 0); - return; - } - /* Allocate a fresh output buffer */ - r = iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); - if (!r) { - mutex_bucket_unlock(bucket, 0); - return; - } - - /* re points to low watermark */ - re = r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN; - } - } - - /* All torrents done: release lock on current bucket */ - mutex_bucket_unlock(bucket, 0); - - /* Parent thread died? */ - if (!g_opentracker_running) - return; - } - - if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) - fmt_append(&r, (char *)iovector.iov_base + OT_SCRAPE_CHUNK_SIZE, "ee"); - - /* Send rest of data */ - iovector.iov_len = r - (char *)iovector.iov_base; - if (workqueue_pushchunked(taskid, &iovector)) - free(iovector.iov_base); -} - -#ifdef WANT_COMPRESSION_GZIP - -static void fullscrape_make_gzip(int taskid, ot_tasktype mode) { - size_t bucket; - char *r; - struct iovec iovector = {NULL, 0}; - int zres; - z_stream strm; - /* Setup return vector... */ - iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); - if (!iovector.iov_base) - return; - - memset(&strm, 0, sizeof(strm)); - strm.next_out = (uint8_t *)iovector.iov_base; - strm.avail_out = OT_SCRAPE_CHUNK_SIZE; - if (deflateInit2(&strm, 7, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY) != Z_OK) - fprintf(stderr, "not ok.\n"); - - if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) { - strm.next_in = (uint8_t *)"d5:filesd"; - strm.avail_in = strlen("d5:filesd"); - zres = deflate(&strm, Z_NO_FLUSH); - } - - /* For each bucket... */ - for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { - /* Get exclusive access to that bucket */ - ot_vector *torrents_list = mutex_bucket_lock(bucket); - ot_torrent *torrents = (ot_torrent *)(torrents_list->data); - size_t i; - - /* For each torrent in this bucket.. */ - for (i = 0; i < torrents_list->size; ++i) { - char compress_buffer[OT_SCRAPE_MAXENTRYLEN]; - r = compress_buffer; - fullscrape_write_one(mode, &r, torrents + i, &torrents[i].hash); - strm.next_in = (uint8_t *)compress_buffer; - strm.avail_in = r - compress_buffer; - zres = deflate(&strm, Z_NO_FLUSH); - if ((zres < Z_OK) && (zres != Z_BUF_ERROR)) - fprintf(stderr, "deflate() failed while in fullscrape_make().\n"); - - /* Check if there still is enough buffer left */ - while (!strm.avail_out) { - iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base; - - if (workqueue_pushchunked(taskid, &iovector)) { - free(iovector.iov_base); - mutex_bucket_unlock(bucket, 0); - return; - } - /* Allocate a fresh output buffer */ - iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); - if (!iovector.iov_base) { - fprintf(stderr, "Out of memory trying to claim ouput buffer\n"); - deflateEnd(&strm); - mutex_bucket_unlock(bucket, 0); - return; - } - strm.next_out = (uint8_t *)iovector.iov_base; - strm.avail_out = OT_SCRAPE_CHUNK_SIZE; - zres = deflate(&strm, Z_NO_FLUSH); - if ((zres < Z_OK) && (zres != Z_BUF_ERROR)) - fprintf(stderr, "deflate() failed while in fullscrape_make().\n"); - } - } - - /* All torrents done: release lock on current bucket */ - mutex_bucket_unlock(bucket, 0); - - /* Parent thread died? */ - if (!g_opentracker_running) { - deflateEnd(&strm); - return; - } - } - - if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) { - strm.next_in = (uint8_t *)"ee"; - strm.avail_in = strlen("ee"); - } - - if (deflate(&strm, Z_FINISH) < Z_OK) - fprintf(stderr, "deflate() failed while in fullscrape_make()'s endgame.\n"); - - iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base; - if (workqueue_pushchunked(taskid, &iovector)) { - free(iovector.iov_base); - deflateEnd(&strm); - return; - } - - /* Check if there's a last batch of data in the zlib buffer */ - if (!strm.avail_out) { - /* Allocate a fresh output buffer */ - iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); - - if (!iovector.iov_base) { - fprintf(stderr, "Problem with iovec_next_chunk\n"); - deflateEnd(&strm); - return; - } - strm.next_out = iovector.iov_base; - strm.avail_out = OT_SCRAPE_CHUNK_SIZE; - if (deflate(&strm, Z_FINISH) < Z_OK) - fprintf(stderr, "deflate() failed while in fullscrape_make()'s endgame.\n"); - - /* Only pass the new buffer if there actually was some data left in the buffer */ - iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base; - if (!iovector.iov_len || workqueue_pushchunked(taskid, &iovector)) - free(iovector.iov_base); - } - - deflateEnd(&strm); -} -/* WANT_COMPRESSION_GZIP */ -#endif - -#ifdef WANT_COMPRESSION_ZSTD - -static void fullscrape_make_zstd(int taskid, ot_tasktype mode) { - int bucket; - char *r; - struct iovec iovector = {NULL, 0}; - ZSTD_CCtx *zstream = ZSTD_createCCtx(); - ZSTD_inBuffer inbuf; - ZSTD_outBuffer outbuf; - size_t more_bytes; - - if (!zstream) - return; - - /* Setup return vector... */ - iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); - if (!iovector.iov_base) { - ZSTD_freeCCtx(zstream); - return; - } - - /* Working with a compression level 6 is half as fast as level 3, but - seems to be the last reasonable bump that's worth extra cpu */ - ZSTD_CCtx_setParameter(zstream, ZSTD_c_compressionLevel, 6); - - outbuf.dst = iovector.iov_base; - outbuf.size = OT_SCRAPE_CHUNK_SIZE; - outbuf.pos = 0; - - if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) { - inbuf.src = (const void *)"d5:filesd"; - inbuf.size = strlen("d5:filesd"); - inbuf.pos = 0; - ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue); - } - - /* For each bucket... */ - for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { - /* Get exclusive access to that bucket */ - ot_vector *torrents_list = mutex_bucket_lock(bucket); - ot_torrent *torrents = (ot_torrent *)(torrents_list->data); - size_t i; - - /* For each torrent in this bucket.. */ - for (i = 0; i < torrents_list->size; ++i) { - char compress_buffer[OT_SCRAPE_MAXENTRYLEN]; - r = compress_buffer; - fullscrape_write_one(mode, &r, torrents + i, &torrents[i].hash); - inbuf.src = compress_buffer; - inbuf.size = r - compress_buffer; - inbuf.pos = 0; - ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue); - - /* Check if there still is enough buffer left */ - while (outbuf.pos + OT_SCRAPE_MAXENTRYLEN > outbuf.size) { - iovector.iov_len = outbuf.size; - - if (workqueue_pushchunked(taskid, &iovector)) { - free(iovector.iov_base); - ZSTD_freeCCtx(zstream); - return mutex_bucket_unlock(bucket, 0); - } - /* Allocate a fresh output buffer */ - iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); - if (!iovector.iov_base) { - fprintf(stderr, "Out of memory trying to claim ouput buffer\n"); - ZSTD_freeCCtx(zstream); - return mutex_bucket_unlock(bucket, 0); - } - - outbuf.dst = iovector.iov_base; - outbuf.size = OT_SCRAPE_CHUNK_SIZE; - outbuf.pos = 0; - - ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue); - } - } - - /* All torrents done: release lock on current bucket */ - mutex_bucket_unlock(bucket, 0); - - /* Parent thread died? */ - if (!g_opentracker_running) - return; - } - - if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) { - inbuf.src = (const void *)"ee"; - inbuf.size = strlen("ee"); - inbuf.pos = 0; - } - - more_bytes = ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_end); - - iovector.iov_len = outbuf.pos; - if (workqueue_pushchunked(taskid, &iovector)) { - free(iovector.iov_base); - ZSTD_freeCCtx(zstream); - return; - } - - /* Check if there's a last batch of data in the zlib buffer */ - if (more_bytes) { - /* Allocate a fresh output buffer */ - iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); - - if (!iovector.iov_base) { - fprintf(stderr, "Problem with iovec_next_chunk\n"); - ZSTD_freeCCtx(zstream); - return; - } - - outbuf.dst = iovector.iov_base; - outbuf.size = OT_SCRAPE_CHUNK_SIZE; - outbuf.pos = 0; - - ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_end); - - /* Only pass the new buffer if there actually was some data left in the buffer */ - iovector.iov_len = outbuf.pos; - if (!iovector.iov_len || workqueue_pushchunked(taskid, &iovector)) - free(iovector.iov_base); - } - - ZSTD_freeCCtx(zstream); -} -/* WANT_COMPRESSION_ZSTD */ -#endif - -/* WANT_FULLSCRAPE */ -#endif diff --git a/ot_fullscrape.h b/ot_fullscrape.h deleted file mode 100644 index 8d11cd3..0000000 --- a/ot_fullscrape.h +++ /dev/null @@ -1,23 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_FULLSCRAPE_H__ -#define OT_FULLSCRAPE_H__ - -#ifdef WANT_FULLSCRAPE - -#include "ot_mutex.h" -#include "ot_workqueue.h" - -void fullscrape_init(void); -void fullscrape_deinit(void); -void fullscrape_deliver(int64_t sock, ot_tasktype tasktype); - -#else - -#define fullscrape_init() -#define fullscrape_deinit() - -#endif - -#endif diff --git a/ot_hex.c b/ot_hex.c deleted file mode 100644 index b537094..0000000 --- a/ot_hex.c +++ /dev/null @@ -1,56 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include - -/* Opentracker */ -#include "ot_hex.h" - -int hex_value(int c) { - unsigned char u = (unsigned char)c; - - if (u >= '0' && u <= '9') - return u - '0'; - u |= 0x20; /* fold upper case onto lower, digits are already handled */ - if (u >= 'a' && u <= 'f') - return u - 'a' + 10; - return -1; -} - -int hex_isdigit(int c) { return hex_value(c) >= 0; } - -char *hex_from_hash(char *dest, const uint8_t *hash) { - static const char digits[] = "0123456789ABCDEF"; - char *d = dest; - size_t i; - - for (i = 0; i < sizeof(ot_hash); ++i) { - *d++ = digits[hash[i] >> 4]; - *d++ = digits[hash[i] & 15]; - } - *d = 0; - return dest; -} - -int hex_to_hash(ot_hash hash, const char *src) { - uint8_t decoded[sizeof(ot_hash)]; - size_t i; - - for (i = 0; i < sizeof(ot_hash); ++i) { - int high, low; - - /* Tested one character at a time so that a string ending early stops us - at its terminator rather than one byte past it */ - if ((high = hex_value((unsigned char)src[2 * i])) < 0) - return 0; - if ((low = hex_value((unsigned char)src[2 * i + 1])) < 0) - return 0; - - decoded[i] = (uint8_t)((high << 4) | low); - } - - memcpy(hash, decoded, sizeof(decoded)); - return 1; -} diff --git a/ot_hex.h b/ot_hex.h deleted file mode 100644 index ddbbb44..0000000 --- a/ot_hex.h +++ /dev/null @@ -1,27 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_HEX_H__ -#define OT_HEX_H__ - -#include - -#include "trackerlogic.h" - -/* Write hash as forty upper case hex digits followed by a terminating zero. - dest must have room for forty one bytes. Returns dest. */ -char *hex_from_hash(char *dest, const uint8_t *hash); - -/* Read forty hex digits from src into hash. Returns non-zero on success. - On failure hash is left untouched and no byte of src beyond the offending - character is read, so src need not be forty characters long. */ -int hex_to_hash(ot_hash hash, const char *src); - -/* Non-zero if c is a hexadecimal digit, in either case. */ -int hex_isdigit(int c); - -/* The value of a hexadecimal digit, or -1 if c is not one. Shared with the - address scanner so that both agree on what a digit is. */ -int hex_value(int c); - -#endif diff --git a/ot_http.c b/ot_http.c deleted file mode 100644 index 96b8f4e..0000000 --- a/ot_http.c +++ /dev/null @@ -1,810 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* Opentracker */ -#include "ot_accesslist.h" -#include "ot_fullscrape.h" -#include "ot_http.h" -#include "ot_iovec.h" -#include "ot_ip.h" -#include "ot_mutex.h" -#include "ot_workqueue.h" -#include "ot_scan.h" -#include "ot_event.h" -#include "ot_fdstate.h" -#include "ot_vector.h" -#include "ot_fmt.h" -#include "ot_stats.h" -#include "ot_urlencoded.h" -#include "trackerlogic.h" - -#define OT_MAXMULTISCRAPE_COUNT 64 -extern const char *g_redirecturl; - -const char *g_stats_path; -ssize_t g_stats_path_len; - -enum { SUCCESS_HTTP_HEADER_LENGTH = 80, SUCCESS_HTTP_SIZE_OFF = 17 }; - -struct http_data *http_adopt(const int64_t sock) { - struct http_data *cookie = fdstate_cookie(sock), *heap; - int interest; - - if (!cookie || !(cookie->flag & STRUCT_HTTP_FLAG_ONSTACK)) - return cookie; - - if (!(heap = malloc(sizeof(*heap)))) - return NULL; - - *heap = *cookie; - heap->flag &= ~STRUCT_HTTP_FLAG_ONSTACK; - - /* Publishing the cookie is what lets another thread reach this connection, - so nothing may touch it afterwards, not even to read what it asked for */ - interest = heap->interest; - fdstate_set_cookie((int)sock, heap); - - if (event_add((int)sock, interest, fdstate_generation((int)sock)) == -1) { - fdstate_set_cookie((int)sock, cookie); - free(heap); - return NULL; - } - return heap; -} - -/* Give up a connection: the one place that knows everything hanging off it. */ -void http_close(const int64_t sock) { - struct http_data *cookie = fdstate_cookie(sock); - - fdstate_close((int)sock); - - if (cookie) { - if (cookie->batch) - batch_reset(cookie->batch); - free(cookie->batch); - buf_reset(&cookie->request); - if (cookie->flag & (STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER)) - workqueue_canceltask(sock); - if (!(cookie->flag & STRUCT_HTTP_FLAG_ONSTACK)) - free(cookie); - } - - event_close((int)sock); -} - -static void http_senddata(const int64_t sock, struct ot_workstruct *ws) { - struct http_data *cookie = fdstate_cookie(sock); - ssize_t written_size; - - if (!cookie) { - http_close(sock); - return; - } - - /* whoever sends data is not interested in its input buffer */ - if (ws->keep_alive && ws->header_size != ws->request_size) { - size_t rest = ws->request_size - ws->header_size; - if (cookie->request.data) { - memmove(cookie->request.data, ws->request + ws->header_size, rest); - buf_truncate(&cookie->request, rest); - } else if (buf_cat(&cookie->request, ws->request + ws->header_size, rest)) { - /* Without room for what the client pipelined behind this request, answer - this one and let the connection close, rather than keeping it open - around a request we have silently dropped */ - buf_reset(&cookie->request); - ws->keep_alive = 0; - } - } else - buf_reset(&cookie->request); - - written_size = write(sock, ws->reply, ws->reply_size); - /* A send buffer with no room in it is not a failure: nothing went out, so - the whole reply goes into the batch below and leaves when the socket - drains. Only a real error ends the connection here. */ - if (written_size < 0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) - written_size = 0; - - if ((written_size < 0) || ((written_size == ws->reply_size) && !ws->keep_alive)) { - http_close(sock); - return; - } - - if (written_size < ws->reply_size) { - char *outbuf; - - if (!(outbuf = malloc(ws->reply_size - written_size))) { - http_close(sock); - return; - } - - memcpy(outbuf, ws->reply + written_size, ws->reply_size - written_size); - if (!cookie->batch) { - cookie->batch = calloc(1, sizeof(ot_batch)); - if (!cookie->batch) { - free(outbuf); - http_close(sock); - return; - } - } - - batch_add(cookie->batch, outbuf, ws->reply_size - written_size); - - /* writeable short data sockets just have a tcp timeout */ - if (!ws->keep_alive) - fdstate_set_expiry(sock, 0); - http_interest_set(sock, cookie, EVENT_OUT); - } -} - -#define HTTPERROR_302 return http_issue_error(sock, ws, CODE_HTTPERROR_302) -#define HTTPERROR_400 return http_issue_error(sock, ws, CODE_HTTPERROR_400) -#define HTTPERROR_400_PARAM return http_issue_error(sock, ws, CODE_HTTPERROR_400_PARAM) -#define HTTPERROR_400_COMPACT return http_issue_error(sock, ws, CODE_HTTPERROR_400_COMPACT) -#define HTTPERROR_400_DOUBLEHASH return http_issue_error(sock, ws, CODE_HTTPERROR_400_PARAM) -#define HTTPERROR_402_NOTMODEST return http_issue_error(sock, ws, CODE_HTTPERROR_402_NOTMODEST) -#define HTTPERROR_403_IP return http_issue_error(sock, ws, CODE_HTTPERROR_403_IP) -#define HTTPERROR_404 return http_issue_error(sock, ws, CODE_HTTPERROR_404) -#define HTTPERROR_500 return http_issue_error(sock, ws, CODE_HTTPERROR_500) -/* A connection is either being read or being written, never both, so what it - waits for is set outright rather than added to and taken away again. The - event layer only hears about it when the answer changes. */ -void http_interest_set(const int64_t sock, struct http_data *cookie, int what) { - if (cookie->interest == what) - return; - cookie->interest = what; - /* Still on a stack, so not in the event loop yet: whoever adopts this - connection registers whatever it asked for last */ - if (cookie->flag & STRUCT_HTTP_FLAG_ONSTACK) - return; - event_modify((int)sock, what, fdstate_generation((int)sock)); -} - -ssize_t http_issue_error(const int64_t sock, struct ot_workstruct *ws, int code) { - const char *error_code[] = {"302 Found", "400 Invalid Request", "400 Invalid Request", "400 Invalid Request", "402 Payment Required", - "403 Not Modest", "403 Access Denied", "404 Not Found", "500 Internal Server Error"}; - const char *title = error_code[code]; - - ws->reply = ws->outbuf; - if (code == CODE_HTTPERROR_302) - ws->reply_size = snprintf(ws->reply, G_OUTBUF_SIZE, "HTTP/1.0 302 Found\r\nContent-Length: 0\r\nLocation: %s\r\n\r\n", g_redirecturl); - else - ws->reply_size = snprintf(ws->reply, G_OUTBUF_SIZE, "HTTP/1.0 %s\r\nContent-Type: text/html\r\nContent-Length: %zd\r\n\r\n%s\n", title, - strlen(title) + 16 - 4, title + 4); - -#ifdef _DEBUG_HTTPERROR - fprintf(stderr, "DEBUG: invalid request was: %s\n", ws->debugbuf); -#endif - stats_issue_event(ws, EVENT_FAILED, FLAG_TCP, code); - http_senddata(sock, ws); - return ws->reply_size = -2; -} - -ssize_t http_sendiovecdata(const int64_t sock, struct ot_workstruct *ws, size_t iovec_entries, struct iovec *iovector, int is_partial) { - struct http_data *cookie = fdstate_cookie(sock); - char *header; - const char *encoding = ""; - size_t header_size, size = iovec_length(&iovec_entries, (const struct iovec **)&iovector); - - /* No cookie? Bad socket. Leave. */ - if (!cookie) { - iovec_free(&iovec_entries, &iovector); - HTTPERROR_500; - } - - /* Descriptor numbers get recycled, so a result can only belong to this - connection if this connection is in fact waiting for one. */ - if (!(cookie->flag & (STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER))) { - iovec_free(&iovec_entries, &iovector); - return 0; - } - - /* If this socket collected request in a buffer, free it now */ - buf_reset(&cookie->request); - - /* If we came here, wait for the answer is over */ - if (cookie->flag & STRUCT_HTTP_FLAG_WAITINGFORTASK) { - http_interest_set(sock, cookie, 0); - cookie->flag &= ~STRUCT_HTTP_FLAG_WAITINGFORTASK; - } - - if (iovec_entries) { - - if (cookie->flag & STRUCT_HTTP_FLAG_ZSTD) - encoding = "Content-Encoding: zstd\r\n"; - else if (cookie->flag & STRUCT_HTTP_FLAG_GZIP) - encoding = "Content-Encoding: gzip\r\n"; - else if (cookie->flag & STRUCT_HTTP_FLAG_BZIP2) - encoding = "Content-Encoding: bzip2\r\n"; - - if (!(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED)) - header_size = asprintf(&header, "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n%sContent-Length: %zd\r\n\r\n", encoding, size); - else { - if (!(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER)) { - header_size = - asprintf(&header, "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n%sTransfer-Encoding: chunked\r\n\r\n%zx\r\n", encoding, size); - cookie->flag |= STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER; - } else - header_size = asprintf(&header, "%zx\r\n", size); - } - if (!header) { - iovec_free(&iovec_entries, &iovector); - HTTPERROR_500; - } - - if (!cookie->batch) { - cookie->batch = calloc(1, sizeof(ot_batch)); - if (!cookie->batch) { - free(header); - iovec_free(&iovec_entries, &iovector); - HTTPERROR_500; - } - } - - batch_add(cookie->batch, header, header_size); - batch_add_vector(cookie->batch, iovector, iovec_entries); - free(iovector); - - if (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) - batch_add_copy(cookie->batch, "\r\n", 2); - } - - if ((cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) && cookie->batch && !is_partial) { - batch_add_copy(cookie->batch, "0\r\n\r\n", 5); - cookie->flag &= ~STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER; - } - - fdstate_set_expiry(sock, time_future_seconds(OT_CLIENT_TIMEOUT_SEND)); - http_interest_set(sock, cookie, EVENT_OUT); - return 0; -} - -static ssize_t http_handle_stats(const int64_t sock, struct ot_workstruct *ws, char *read_ptr) { - static const ot_keywords keywords_main[] = {{"mode", 1}, {"format", 2}, {"info_hash", 3}, {NULL, -3}}; - static const ot_keywords keywords_mode[] = {{"peer", TASK_STATS_PEERS}, - {"conn", TASK_STATS_CONNS}, - {"scrp", TASK_STATS_SCRAPE}, - {"udp4", TASK_STATS_UDP}, - {"tcp4", TASK_STATS_TCP}, - {"torr", TASK_STATS_TORRENTS}, - {"fscr", TASK_STATS_FULLSCRAPE}, - {"s24s", TASK_STATS_SLASH24S}, - {"tpbs", TASK_STATS_TPB}, - {"herr", TASK_STATS_HTTPERRORS}, - {"completed", TASK_STATS_COMPLETED}, - {"top100", TASK_STATS_TOP100}, - {"top10", TASK_STATS_TOP10}, - {"renew", TASK_STATS_RENEW}, - {"syncs", TASK_STATS_SYNCS}, - {"everything", TASK_STATS_EVERYTHING}, - {"statedump", TASK_FULLSCRAPE_TRACKERSTATE}, - {"woodpeckers", TASK_STATS_WOODPECKERS}, -#ifdef WANT_LOG_NUMWANT - {"numwants", TASK_STATS_NUMWANTS}, -#endif - {NULL, -3}}; - static const ot_keywords keywords_format[] = {{"bin", TASK_FULLSCRAPE_TPB_BINARY}, {"ben", TASK_FULLSCRAPE}, - {"url", TASK_FULLSCRAPE_TPB_URLENCODED}, {"txt", TASK_FULLSCRAPE_TPB_ASCII}, - {"txtp", TASK_FULLSCRAPE_TPB_ASCII_PLUS}, {NULL, -3}}; - - struct http_data *cookie = fdstate_cookie(sock); - char *reply; - int mode = TASK_STATS_PEERS, scanon = 1, format = 0; - - /* A socket without one is a dead socket, which is what http_sendiovecdata - already answers 500 to. */ - if (!cookie) - HTTPERROR_500; - -#ifdef WANT_RESTRICT_STATS - if (!accesslist_is_blessed(cookie->ip, OT_PERMISSION_MAY_STAT)) - HTTPERROR_403_IP; -#endif - - while (scanon) { - switch (urlencoded_match_keyword(keywords_main, &read_ptr, SCAN_SEARCHPATH_PARAM)) { - case -2: - scanon = 0; - break; /* TERMINATOR */ - case -1: - HTTPERROR_400_PARAM; /* PARSE ERROR */ - case -3: - urlencoded_skip_value(&read_ptr); - break; - case 1: /* matched "mode" */ - if ((mode = urlencoded_match_keyword(keywords_mode, &read_ptr, SCAN_SEARCHPATH_VALUE)) <= 0) - HTTPERROR_400_PARAM; - break; - case 2: /* matched "format" */ - if ((format = urlencoded_match_keyword(keywords_format, &read_ptr, SCAN_SEARCHPATH_VALUE)) <= 0) - HTTPERROR_400_PARAM; - break; - case 3: - HTTPERROR_400_PARAM; /* If the stats URL was mistakenly added as announce URL, return a 400 */ - } - } - -#ifdef WANT_FULLSCRAPE - if (mode == TASK_FULLSCRAPE_TRACKERSTATE) { - format = mode; - mode = TASK_STATS_TPB; - } - - if (mode == TASK_STATS_TPB) { -#ifdef WANT_COMPRESSION_GZIP - ws->request[ws->request_size - 1] = 0; -#ifndef WANT_COMPRESSION_GZIP_ALWAYS - if (strstr(read_ptr - 1, "gzip")) { -#endif - cookie->flag |= STRUCT_HTTP_FLAG_GZIP; - format |= TASK_FLAG_GZIP; -#ifndef WANT_COMPRESSION_GZIP_ALWAYS - } -#endif -#endif - cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED; - - /* Clients waiting for us should not easily timeout */ - fdstate_set_expiry(sock, 0); - http_interest_set(sock, cookie, 0); - - /* Everything this connection needs has been written, so it can be given - a home of its own and only then be made reachable by a worker thread */ - if (!http_adopt(sock)) - HTTPERROR_500; - fullscrape_deliver(sock, format); - return ws->reply_size = -2; - } -#endif - - /* default format for now */ - if ((mode & TASK_CLASS_MASK) == TASK_STATS) { - - cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK; - - /* Complex stats also include expensive memory debugging tools */ - fdstate_set_expiry(sock, 0); - - if (!http_adopt(sock)) - HTTPERROR_500; - stats_deliver(sock, mode); - return ws->reply_size = -2; - } - - /* Simple stats can be answerred immediately */ - reply = ws->reply; - return_stats_for_tracker(&reply, ws->outbuf + G_OUTBUF_SIZE, mode, 0); - return ws->reply_size = reply - ws->reply; -} - -#ifdef WANT_MODEST_FULLSCRAPES -static pthread_mutex_t g_modest_fullscrape_mutex = PTHREAD_MUTEX_INITIALIZER; -static ot_vector g_modest_fullscrape_timeouts; -typedef struct { - ot_ip6 ip; - ot_time last_fullscrape; -} ot_scrape_log; -#endif - -#ifdef WANT_FULLSCRAPE -static ssize_t http_handle_fullscrape(const int64_t sock, struct ot_workstruct *ws) { - struct http_data *cookie = fdstate_cookie(sock); - int format = 0; - -#ifdef WANT_MODEST_FULLSCRAPES - { - ot_scrape_log this_peer, *new_peer; - int exactmatch; - memcpy(this_peer.ip, cookie->ip, sizeof(ot_ip6)); - this_peer.last_fullscrape = time_now_seconds(); - pthread_mutex_lock(&g_modest_fullscrape_mutex); - new_peer = vector_find_or_insert(&g_modest_fullscrape_timeouts, &this_peer, sizeof(ot_scrape_log), sizeof(ot_ip6), &exactmatch); - if (!new_peer) { - pthread_mutex_unlock(&g_modest_fullscrape_mutex); - HTTPERROR_500; - } - if (exactmatch && (this_peer.last_fullscrape - new_peer->last_fullscrape) < OT_MODEST_PEER_TIMEOUT) { - pthread_mutex_unlock(&g_modest_fullscrape_mutex); - HTTPERROR_402_NOTMODEST; - } - memcpy(new_peer, &this_peer, sizeof(ot_scrape_log)); - pthread_mutex_unlock(&g_modest_fullscrape_mutex); - } -#endif - - -#if defined(WANT_COMPRESSION_GZIP) || defined(WANT_COMPRESSION_ZSTD) - ws->request[ws->request_size - 1] = 0; -#ifdef WANT_COMPRESSION_GZIP - if (strstr(ws->request, "gzip")) { - cookie->flag |= STRUCT_HTTP_FLAG_GZIP; - format |= TASK_FLAG_GZIP; - } -#endif -#ifdef WANT_COMPRESSION_ZSTD - if (strstr(ws->request, "zstd")) { - cookie->flag |= STRUCT_HTTP_FLAG_ZSTD; - format |= TASK_FLAG_ZSTD; - } -#endif - -#if defined(WANT_COMPRESSION_ZSTD) && defined(WANT_COMPRESSION_ZSTD_ALWAYS) - cookie->flag |= STRUCT_HTTP_FLAG_ZSTD; - format |= TASK_FLAG_ZSTD; -#endif - -#if defined(WANT_COMPRESSION_GZIP) && defined(WANT_COMPRESSION_GZIP_ALWAYS) - cookie->flag |= STRUCT_HTTP_FLAG_GZIP; - format |= TASK_FLAG_GZIP; -#endif -#endif - - stats_issue_event(ws, EVENT_FULLSCRAPE_REQUEST, 0, (uintptr_t)cookie->ip); - -#ifdef _DEBUG_HTTPERROR - fprintf(stderr, "%s", ws->debugbuf); -#endif - - /* Pass this task to the worker thread */ - cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED; - /* Clients waiting for us should not easily timeout */ - fdstate_set_expiry(sock, 0); - http_interest_set(sock, cookie, 0); - - if (!http_adopt(sock)) - HTTPERROR_500; - fullscrape_deliver(sock, format); - return ws->reply_size = -2; -} -#endif - -static int compare_hash(const void *left, const void *right) { return memcmp(left, right, sizeof(ot_hash)); } - -static ssize_t http_handle_scrape(const int64_t sock, struct ot_workstruct *ws, char *read_ptr) { - static const ot_keywords keywords_scrape[] = {{"info_hash", 1}, {NULL, -3}}; - - ot_hash *multiscrape_buf = (ot_hash *)ws->request; - char *reply = ws->reply; - int scanon = 1; - size_t i, unique, numwant = 0; - - /* This is to hack around stupid clients that send "scrape ?info_hash" */ - if (read_ptr[-1] != '?') { - while ((*read_ptr != '?') && (*read_ptr != '\n')) - ++read_ptr; - if (*read_ptr == '\n') - HTTPERROR_400_PARAM; - ++read_ptr; - } - - while (scanon) { - switch (urlencoded_match_keyword(keywords_scrape, &read_ptr, SCAN_SEARCHPATH_PARAM)) { - case -2: - scanon = 0; - break; /* TERMINATOR */ - default: - HTTPERROR_400_PARAM; /* PARSE ERROR */ - case -3: - urlencoded_skip_value(&read_ptr); - break; - case 1: /* matched "info_hash" */ - /* ignore this, when we have less than 20 bytes */ - if (urlencoded_parse(&read_ptr, (char *)(multiscrape_buf + numwant++), SCAN_SEARCHPATH_VALUE) != (ssize_t)sizeof(ot_hash)) - HTTPERROR_400_PARAM; - break; - } - } - - /* No info_hash found? Inform user */ - if (!numwant) - HTTPERROR_400_PARAM; - - /* Limit number of hashes to process */ - if (numwant > OT_MAXMULTISCRAPE_COUNT) - numwant = OT_MAXMULTISCRAPE_COUNT; - - /* A bencoded dictionary has its keys in order and names each one once, so - the answer cannot just follow the order the client asked in */ - qsort(multiscrape_buf, numwant, sizeof(ot_hash), compare_hash); - for (i = unique = 1; i < numwant; ++i) - if (memcmp(multiscrape_buf[i], multiscrape_buf[unique - 1], sizeof(ot_hash))) - memcpy(multiscrape_buf[unique++], multiscrape_buf[i], sizeof(ot_hash)); - numwant = unique; - - /* Enough for http header + whole scrape string */ - return_tcp_scrape_for_torrent((const ot_hash *)multiscrape_buf, numwant, &reply, ws->outbuf + G_OUTBUF_SIZE); - ws->reply_size = reply - ws->reply; - stats_issue_event(ws, EVENT_SCRAPE, FLAG_TCP, ws->reply_size); - return ws->reply_size; -} - -#ifdef WANT_LOG_NUMWANT -unsigned long long numwants[201]; -#endif - -#if defined(WANT_KEEPALIVE) || defined(WANT_IP_FROM_PROXY) -static char *http_header(char *data, size_t byte_count, const char *header) { - size_t i; - long sl = strlen(header); - for (i = 0; i + sl + 2 < byte_count; ++i) { - if (data[i] != '\n' || data[i + sl + 1] != ':') - continue; - if (strncasecmp(data + i + 1, header, sl)) - continue; - data += i + sl + 2; - while (*data == ' ' || *data == '\t') - ++data; - return data; - } - return 0; -} -#endif - -static ot_keywords keywords_announce[] = {{"port", 1}, {"left", 2}, {"event", 3}, {"numwant", 4}, {"compact", 5}, {"compact6", 5}, {"info_hash", 6}, -#ifdef WANT_IP_FROM_QUERY_STRING - {"ip", 7}, -#endif - {"peer_id", 8}, {NULL, -3}}; -static ot_keywords keywords_announce_event[] = {{"completed", 1}, {"stopped", 2}, {NULL, -3}}; - -/* For the replies that are one fixed string and nothing else, where the only - thing the caller wants back is how long it turned out */ -static ssize_t reply_literal(struct ot_workstruct *ws, const char *text) { - char *reply = ws->reply; - - fmt_bytes(&reply, ws->outbuf + G_OUTBUF_SIZE, text, strlen(text)); - return reply - ws->reply; -} - -static ssize_t http_handle_announce(const int64_t sock, struct ot_workstruct *ws, char *read_ptr) { - size_t numwant; - int64_t tmp; - int scanon; - unsigned short port = 0; - char *write_ptr; - ssize_t len; - struct http_data *cookie = fdstate_cookie(sock); - - /* This is to hack around stupid clients that send "announce ?info_hash" */ - if (read_ptr[-1] != '?') { - while ((*read_ptr != '?') && (*read_ptr != '\n')) - ++read_ptr; - if (*read_ptr == '\n') - HTTPERROR_400_PARAM; - ++read_ptr; - } - -#ifdef WANT_IP_FROM_PROXY - if (accesslist_is_blessed(cookie->ip, OT_PERMISSION_MAY_PROXY)) { - ot_ip6 proxied_ip; - char *fwd = http_header(ws->request, ws->header_size, "x-forwarded-for"); - if (fwd && ip_parse(fwd, proxied_ip)) { - OT_SETIP(ws->peer, proxied_ip); - } else - OT_SETIP(ws->peer, cookie->ip); - } else -#endif - OT_SETIP(ws->peer, cookie->ip); - - ws->peer_id = NULL; - ws->hash = NULL; - - OT_SETPORT(ws->peer, &port); - OT_PEERFLAG(ws->peer) = 0; - numwant = 50; - scanon = 1; - - while (scanon) { - switch (urlencoded_match_keyword(keywords_announce, &read_ptr, SCAN_SEARCHPATH_PARAM)) { - case -2: - scanon = 0; - break; /* TERMINATOR */ - case -1: - HTTPERROR_400_PARAM; /* PARSE ERROR */ - case -3: - urlencoded_skip_value(&read_ptr); - break; - case 1: /* matched "port" */ - len = urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE); - if ((len <= 0) || scan_int(write_ptr, len, &tmp) || (tmp > 0xffff)) - HTTPERROR_400_PARAM; - port = htons((uint16_t)tmp); - OT_SETPORT(&ws->peer, &port); - break; - case 2: /* matched "left" */ - if ((len = urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE)) <= 0) - HTTPERROR_400_PARAM; - if (scan_int(write_ptr, len, &tmp)) - tmp = 0; - if (!tmp) - OT_PEERFLAG(&ws->peer) |= PEER_FLAG_SEEDING; - break; - case 3: /* matched "event" */ - switch (urlencoded_match_keyword(keywords_announce_event, &read_ptr, SCAN_SEARCHPATH_VALUE)) { - case -1: - HTTPERROR_400_PARAM; - case 1: /* matched "completed" */ - OT_PEERFLAG(&ws->peer) |= PEER_FLAG_COMPLETED; - break; - case 2: /* matched "stopped" */ - OT_PEERFLAG(&ws->peer) |= PEER_FLAG_STOPPED; - break; - default: - break; - } - break; - case 4: /* matched "numwant" */ - len = urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE); - if ((len <= 0) || scan_int(write_ptr, len, &tmp)) - HTTPERROR_400_PARAM; - if (tmp < 0) - tmp = 50; - if (tmp > 200) - tmp = 200; - numwant = (size_t)tmp; - break; - case 5: /* matched "compact" */ - len = urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE); - if ((len <= 0) || scan_int(write_ptr, len, &tmp)) - HTTPERROR_400_PARAM; - if (!tmp) - HTTPERROR_400_COMPACT; - break; - case 6: /* matched "info_hash" */ - if (ws->hash) - HTTPERROR_400_DOUBLEHASH; - /* ignore this, when we have less than 20 bytes */ - if (urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE) != 20) - HTTPERROR_400_PARAM; - ws->hash = (ot_hash *)write_ptr; - break; -#ifdef WANT_IP_FROM_QUERY_STRING - case 7: /* matched "ip" */ - { - char *tmp_buf1 = ws->reply, *tmp_buf2 = ws->reply + 16; - len = urlencoded_parse(&read_ptr, tmp_buf2, SCAN_SEARCHPATH_VALUE); - tmp_buf2[len] = 0; - if ((len <= 0) || ip_parse(tmp_buf2, tmp_buf1) != (size_t)len) - HTTPERROR_400_PARAM; - OT_SETIP(&ws->peer, tmp_buf1); - } break; -#endif - case 8: /* matched "peer_id" */ - /* ignore this, when we have less than 20 bytes */ - if (urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE) != 20) - HTTPERROR_400_PARAM; - ws->peer_id = write_ptr; - break; - } - } - -#ifdef WANT_LOG_NUMWANT - numwants[numwant]++; -#endif - - /* XXX DEBUG - stats_issue_event(ws, EVENT_ACCEPT, FLAG_TCP, (uintptr_t)ws->reply ); - */ - - /* Scanned whole query string */ - if (!ws->hash) - return ws->reply_size = reply_literal(ws, "d14:failure reason80:Your client forgot to send your torrent's info_hash. Please upgrade your client.e"); - - if (OT_PEERFLAG(&ws->peer) & PEER_FLAG_STOPPED) - ws->reply_size = remove_peer_from_torrent(FLAG_TCP, ws); - else - ws->reply_size = add_peer_to_torrent_and_return_peers(FLAG_TCP, ws, numwant); - - stats_issue_event(ws, EVENT_ANNOUNCE, FLAG_TCP, ws->reply_size); - return ws->reply_size; -} - -ssize_t http_handle_request(const int64_t sock, struct ot_workstruct *ws) { - ssize_t reply_off, len; - char *read_ptr = ws->request, *write_ptr; - -#ifdef _DEBUG_HTTPERROR - reply_off = ws->request_size; - if (ws->request_size >= G_DEBUGBUF_SIZE) - reply_off = G_DEBUGBUF_SIZE - 1; - memcpy(ws->debugbuf, ws->request, reply_off); - ws->debugbuf[reply_off] = 0; -#endif - - /* Tell subroutines where to put reply data */ - ws->reply = ws->outbuf + SUCCESS_HTTP_HEADER_LENGTH; - - /* Find out if the client wants to keep this connection alive, before any - subroutine gets to answer: an error reply finishes with the connection - on its way out, and ws outlives the connection it was serving */ - ws->keep_alive = 0; -#ifdef WANT_KEEPALIVE - if ((write_ptr = http_header(ws->request, ws->header_size, "connection")) && (*write_ptr == 'K' || *write_ptr == 'k')) - ws->keep_alive = 1; -#endif - { - struct http_data *connection = fdstate_cookie(sock); - if (connection) { - if (ws->keep_alive) - connection->flag |= STRUCT_HTTP_FLAG_KEEPALIVE; - else - connection->flag &= ~STRUCT_HTTP_FLAG_KEEPALIVE; - } - } - - /* This one implicitely tests strlen < 5, too -- remember, it is \n terminated */ - if (memcmp(read_ptr, "GET /", 5)) - HTTPERROR_400; - - /* Skip leading '/' */ - for (read_ptr += 4; *read_ptr == '/'; ++read_ptr) - ; - - /* Try to parse the request. - In reality we abandoned requiring the url to be correct. This now - only decodes url encoded characters, we check for announces and - scrapes by looking for "a*" or "sc" */ - len = urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_PATH); - - /* If parsing returned an error, leave with not found */ - if (g_redirecturl && (len == -2)) - HTTPERROR_302; - if (len <= 0) - HTTPERROR_404; - - /* This is the hardcore match for announce*/ - if ((*write_ptr == 'a') || (*write_ptr == '?')) - http_handle_announce(sock, ws, read_ptr); -#ifdef WANT_FULLSCRAPE - else if (!memcmp(write_ptr, "scrape HTTP/", 12)) - http_handle_fullscrape(sock, ws); -#endif - /* This is the hardcore match for scrape */ - else if (!memcmp(write_ptr, "sc", 2)) - http_handle_scrape(sock, ws, read_ptr); - /* All the rest is matched the standard way */ - else if (len == g_stats_path_len && !memcmp(write_ptr, g_stats_path, len)) - http_handle_stats(sock, ws, read_ptr); - else - HTTPERROR_404; - - /* If routines handled sending themselves, just return */ - if (ws->reply_size == -2) - return 0; - /* If routine failed, let http error take over */ - if (ws->reply_size <= 0) - HTTPERROR_500; - - /* This one is rather ugly, so I take you step by step through it. - - 1. In order to avoid having two buffers, one for header and one for content, we allow all above functions from trackerlogic to - write to a fixed location, leaving SUCCESS_HTTP_HEADER_LENGTH bytes in our work buffer, which is enough for the static string - plus dynamic space needed to expand our Content-Length value. We reserve SUCCESS_HTTP_SIZE_OFF for its expansion and calculate - the space NOT needed to expand in reply_off - */ - reply_off = SUCCESS_HTTP_SIZE_OFF - snprintf(ws->outbuf, 0, "%zd", ws->reply_size); - ws->reply = ws->outbuf + reply_off; - - /* 2. Now we write the header so that its terminating '\0' lands exactly one byte before the content starts. The bound is the - space step 1 reserved, so the '\0' falls where step 3 expects it. Complete packet size is increased by size of header plus - one byte '\n', we will copy over '\0' in next step */ - ws->reply_size += 1 + snprintf(ws->reply, (size_t)(SUCCESS_HTTP_HEADER_LENGTH - reply_off), - "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: %zd\r\n\r", ws->reply_size); - - /* 3. Finally we join both blocks neatly */ - ws->outbuf[SUCCESS_HTTP_HEADER_LENGTH - 1] = '\n'; - - http_senddata(sock, ws); - return ws->reply_size; -} diff --git a/ot_http.h b/ot_http.h deleted file mode 100644 index fb56c64..0000000 --- a/ot_http.h +++ /dev/null @@ -1,53 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_HTTP_H__ -#define OT_HTTP_H__ - -#include -#include -#include - -#include "ot_batch.h" -#include "ot_event.h" -#include "ot_buf.h" -#include "trackerlogic.h" - -typedef enum { - STRUCT_HTTP_FLAG_WAITINGFORTASK = 1, - STRUCT_HTTP_FLAG_GZIP = 2, - STRUCT_HTTP_FLAG_BZIP2 = 4, - STRUCT_HTTP_FLAG_ZSTD = 8, - STRUCT_HTTP_FLAG_CHUNKED = 16, - STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER = 32, - STRUCT_HTTP_FLAG_KEEPALIVE = 64, - STRUCT_HTTP_FLAG_ONSTACK = 128 -} STRUCT_HTTP_FLAG; - -struct http_data { - ot_buf request; - int interest; /* EVENT_IN and EVENT_OUT as asked for */ - ot_batch *batch; - ot_ip6 ip; - STRUCT_HTTP_FLAG flag; -}; - -/* What this connection is waiting for: EVENT_IN, EVENT_OUT, or nothing */ -void http_interest_set(const int64_t s, struct http_data *cookie, int what); - -/* Close a connection and release everything that belongs to it */ -void http_close(const int64_t s); - -/* Promote a stack cookie to a heap one, for persistence and for handing it - over to the event loop or to another thread. Does nothing to one that is - already on the heap. Returns the connection's state, or NULL. */ -struct http_data *http_adopt(const int64_t s); - -ssize_t http_handle_request(const int64_t s, struct ot_workstruct *ws); -ssize_t http_sendiovecdata(const int64_t s, struct ot_workstruct *ws, size_t iovec_entries, struct iovec *iovector, int is_partial); -ssize_t http_issue_error(const int64_t s, struct ot_workstruct *ws, int code); - -extern const char *g_stats_path; -extern ssize_t g_stats_path_len; - -#endif diff --git a/ot_iovec.c b/ot_iovec.c deleted file mode 100644 index 328f160..0000000 --- a/ot_iovec.c +++ /dev/null @@ -1,87 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include - -/* Opentracker */ -#include "ot_iovec.h" - -void *iovec_new_chunk(size_t *iovec_entries, struct iovec **iovector, size_t new_alloc) { - void *new_data; - size_t new_entries = 1 + *iovec_entries; - struct iovec *new_vec = realloc(*iovector, new_entries * sizeof(struct iovec)); - - if (!new_vec) - return NULL; - - *iovector = new_vec; - - /* Only allocate after we have a place to store the pointer */ - new_data = malloc(new_alloc); - if (!new_data) - return NULL; - - new_vec[new_entries - 1].iov_base = new_data; - new_vec[new_entries - 1].iov_len = new_alloc; - - ++*iovec_entries; - return new_data; -} - -void *iovec_append(size_t *iovec_entries, struct iovec **iovector, struct iovec *append_iovector) { - size_t new_entries = *iovec_entries + 1; - struct iovec *new_vec = realloc(*iovector, new_entries * sizeof(struct iovec)); - if (!new_vec) - return NULL; - - /* Take over data from appended iovec */ - new_vec[*iovec_entries].iov_base = append_iovector->iov_base; - new_vec[*iovec_entries].iov_len = append_iovector->iov_len; - - append_iovector->iov_base = NULL; - append_iovector->iov_len = 0; - - *iovector = new_vec; - *iovec_entries = new_entries; - - return new_vec; -} - -void iovec_free(size_t *iovec_entries, struct iovec **iovector) { - size_t i; - for (i = 0; i < *iovec_entries; ++i) - free(((*iovector)[i]).iov_base); - free(*iovector); - *iovector = NULL; - *iovec_entries = 0; -} - -void iovec_trim_last(size_t *iovec_entries, struct iovec **iovector, void *last_ptr) { - if (*iovec_entries) { - char *base = (char *)((*iovector)[*iovec_entries - 1]).iov_base; - size_t new_alloc = ((char *)last_ptr) - base; - - ((*iovector)[*iovec_entries - 1]).iov_base = realloc(base, new_alloc); - ((*iovector)[*iovec_entries - 1]).iov_len = new_alloc; - } -} - -void *iovec_next_chunk(size_t *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc) { - void *new_data; - - iovec_trim_last(iovec_entries, iovector, last_ptr); - - if (!(new_data = iovec_new_chunk(iovec_entries, iovector, new_alloc))) - iovec_free(iovec_entries, iovector); - - return new_data; -} - -size_t iovec_length(const size_t *iovec_entries, const struct iovec **iovector) { - size_t i, length = 0; - for (i = 0; i < *iovec_entries; ++i) - length += ((*iovector)[i]).iov_len; - return length; -} diff --git a/ot_iovec.h b/ot_iovec.h deleted file mode 100644 index 4a8abbb..0000000 --- a/ot_iovec.h +++ /dev/null @@ -1,18 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_IOVEC_H__ -#define OT_IOVEC_H__ - -#include - -void *iovec_new_chunk(size_t *iovec_entries, struct iovec **iovector, size_t new_alloc); -void *iovec_append(size_t *iovec_entries, struct iovec **iovector, struct iovec *append_iovector); -void iovec_trim_last(size_t *iovec_entries, struct iovec **iovector, void *last_ptr); -void iovec_free(size_t *iovec_entries, struct iovec **iovector); - -size_t iovec_length(const size_t *iovec_entries, const struct iovec **iovector); - -void *iovec_next_chunk(size_t *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc); - -#endif diff --git a/ot_ip.c b/ot_ip.c deleted file mode 100644 index 9c76290..0000000 --- a/ot_ip.c +++ /dev/null @@ -1,199 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include - -/* Opentracker */ -#include "ot_hex.h" -#include "ot_ip.h" - -const char ip_v4mapped_prefix[12] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff}; - -/* The dotted quad is only ever reached through the v4 mapped form, so both - halves of it stay in here */ - -static size_t ip4_format(char *dest, const uint8_t *ip) { - char *d = dest; - int i; - - for (i = 0; i < 4; ++i) { - unsigned int octet = ip[i]; - - if (i) - *d++ = '.'; - if (octet >= 100) - *d++ = (char)('0' + octet / 100); - if (octet >= 10) - *d++ = (char)('0' + (octet / 10) % 10); - *d++ = (char)('0' + octet % 10); - } - - return (size_t)(d - dest); -} - -static size_t ip4_scan(const char *src, uint8_t ip[4]) { - uint8_t octets[4]; - const char *s = src; - int i; - - for (i = 0; i < 4; ++i) { - unsigned int value = 0; - int digits = 0; - - if (i && *s++ != '.') - return 0; - - /* Leading zeros are tolerated, a value beyond one octet is not */ - while (*s >= '0' && *s <= '9' && digits < 4) { - value = value * 10 + (unsigned int)(*s++ - '0'); - ++digits; - } - if (!digits || value > 255) - return 0; - - octets[i] = (uint8_t)value; - } - - memcpy(ip, octets, sizeof(octets)); - return (size_t)(s - src); -} - -static unsigned int group_at(const char *ip, int n) { return ((unsigned int)(uint8_t)ip[2 * n] << 8) | (uint8_t)ip[2 * n + 1]; } - -size_t ip_format(char *dest, const ot_ip6 ip) { - static const char hexdigit[] = "0123456789abcdef"; - char *d = dest; - int best_start = 0, best_len = 0, i = 0; - - if (ip_is_v4mapped(ip)) - return ip4_format(dest, (const uint8_t *)ip + sizeof(ip_v4mapped_prefix)); - - /* RFC 5952 4.2: the longest run of zero groups becomes "::", never a run - of one, and the first of two equal runs wins */ - while (i < 8) { - int j = i; - - if (group_at(ip, i)) { - ++i; - continue; - } - while (j < 8 && !group_at(ip, j)) - ++j; - if (j - i > best_len) { - best_len = j - i; - best_start = i; - } - i = j; - } - if (best_len < 2) - best_len = 0; - - for (i = 0; i < 8; ++i) { - unsigned int group; - int shift = 12; - - if (best_len && i == best_start) { - *d++ = ':'; - if (i + best_len == 8) - *d++ = ':'; /* nothing follows, so close the run */ - i += best_len - 1; - continue; - } - - if (i) - *d++ = ':'; - - group = group_at(ip, i); - while (shift && !(group >> shift)) - shift -= 4; /* RFC 5952 4.1: no leading zeros */ - for (;; shift -= 4) { - *d++ = hexdigit[(group >> shift) & 15]; - if (!shift) - break; - } - } - - return (size_t)(d - dest); -} - -size_t ip_parse(const char *src, ot_ip6 ip) { - uint8_t prefix[16], suffix[16]; - size_t prefixlen = 0, suffixlen = 0; - const char *s = src; - int have_gap = 0; - int need_colon = 0; - - /* A plain IPv4 address is an address too, stored v4 mapped */ - { - size_t consumed = ip4_scan(src, prefix); - if (consumed) { - memcpy(ip, ip_v4mapped_prefix, sizeof(ip_v4mapped_prefix)); - memcpy(ip + sizeof(ip_v4mapped_prefix), prefix, 4); - return consumed; - } - } - - for (;;) { - unsigned int value = 0; - int digits = 0, nibble; - uint8_t *into = have_gap ? suffix : prefix; - size_t *fill = have_gap ? &suffixlen : &prefixlen; - - /* RFC 4291 2.2: one run of zero groups may be written "::" */ - if (s[0] == ':' && s[1] == ':') { - if (have_gap) - return 0; - have_gap = 1; - need_colon = 0; - s += 2; - continue; - } - - if (need_colon) { - if (*s != ':') - break; - ++s; - } - - /* RFC 4291 2.2 form 3: the final two groups may be a dotted quad */ - if (prefixlen + suffixlen + 4 <= 16) { - size_t consumed = ip4_scan(s, into + *fill); - if (consumed) { - *fill += 4; - s += consumed; - break; - } - } - - while (digits < 4 && (nibble = hex_value(*s)) >= 0) { - value = value * 16 + (unsigned int)nibble; - ++digits; - ++s; - } - if (!digits) { - if (need_colon) - --s; /* leave the separator we ate for the caller */ - break; - } - - if (prefixlen + suffixlen + 2 > 16) - return 0; - - into[(*fill)++] = (uint8_t)(value >> 8); - into[(*fill)++] = (uint8_t)value; - need_colon = 1; - } - - if (!prefixlen && !suffixlen && !have_gap) - return 0; - if (!have_gap && prefixlen + suffixlen != 16) - return 0; /* without "::" every group has to be spelled out */ - - memset(ip, 0, 16); - memcpy(ip, prefix, prefixlen); - memcpy(ip + 16 - suffixlen, suffix, suffixlen); - - return (size_t)(s - src); -} diff --git a/ot_ip.h b/ot_ip.h deleted file mode 100644 index 533403e..0000000 --- a/ot_ip.h +++ /dev/null @@ -1,31 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_IP_H__ -#define OT_IP_H__ - -#include -#include - -#include "trackerlogic.h" - -/* opentracker holds every address as sixteen bytes, writing IPv4 as the v4 - mapped form of RFC 4291 section 2.5.5.2. */ -extern const char ip_v4mapped_prefix[12]; - -static inline int ip_is_v4mapped(const void *ip) { return !memcmp(ip, ip_v4mapped_prefix, sizeof(ip_v4mapped_prefix)); } - -/* Read an address and return how many characters it took, or zero if src - does not start with one. A plain IPv4 address is accepted and stored in - its v4 mapped form. - - The count lets the caller carry on where the address ended, which is how - "address:port" and "address/bits" are read. */ -size_t ip_parse(const char *src, ot_ip6 ip); - -/* Write an address and return how many characters it took. Nothing is - terminated, so a caller may keep appending. A v4 mapped address is written - as the plain dotted quad rather than "::ffff:1.2.3.4". */ -size_t ip_format(char *dest, const ot_ip6 ip); - -#endif diff --git a/ot_livesync.c b/ot_livesync.c deleted file mode 100644 index 083fe1a..0000000 --- a/ot_livesync.c +++ /dev/null @@ -1,241 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include -#include -#include -#include -#include -#include - -/* Opentracker */ -#include "ot_accesslist.h" -#include "ot_ip.h" -#include "ot_socket.h" -#include "ot_livesync.h" -#include "ot_mutex.h" -#include "ot_stats.h" -#include "trackerlogic.h" - -#ifdef WANT_SYNC_LIVE - -ot_ip6 groupip_1 = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 224, 0, 23, 5}; - -#define LIVESYNC_INCOMING_BUFFSIZE (256 * 256) - -#define LIVESYNC_OUTGOING_BUFFSIZE_PEERS 1480 -#define LIVESYNC_OUTGOING_WATERMARK_PEERS (sizeof(ot_peer) + sizeof(ot_hash)) - -#define LIVESYNC_MAXDELAY 15 /* seconds */ - -enum { OT_SYNC_PEER4, OT_SYNC_PEER6 }; - -/* Forward declaration */ -static void *livesync_worker(void *args); - -/* For incoming packets */ -static int g_socket_in = -1; - -/* For outgoing packets */ -static int g_socket_out = -1; - -static pthread_mutex_t g_outbuf_mutex = PTHREAD_MUTEX_INITIALIZER; -typedef struct { - uint8_t data[LIVESYNC_OUTGOING_BUFFSIZE_PEERS]; - size_t fill; - ot_time next_packet_time; -} sync_buffer; - -static sync_buffer g_v6_buf; -static sync_buffer g_v4_buf; - -static pthread_t thread_id; -static int thread_started; -static _Atomic int thread_left; -void livesync_init(void) { - uint32_t packet_type6 = htonl(OT_SYNC_PEER6); - uint32_t packet_type4 = htonl(OT_SYNC_PEER4); - - if (g_socket_in == -1) - exerr("No socket address for live sync specified."); - - /* Prepare outgoing peers buffer */ - memcpy(g_v6_buf.data, &g_tracker_id, sizeof(g_tracker_id)); - memcpy(g_v4_buf.data, &g_tracker_id, sizeof(g_tracker_id)); - - memcpy(g_v6_buf.data + sizeof(g_tracker_id), &packet_type6, sizeof(packet_type6)); - memcpy(g_v4_buf.data + sizeof(g_tracker_id), &packet_type4, sizeof(packet_type4)); - - g_v6_buf.fill = sizeof(g_tracker_id) + sizeof(uint32_t); - g_v4_buf.fill = sizeof(g_tracker_id) + sizeof(uint32_t); - - g_v6_buf.next_packet_time = time_future_seconds(LIVESYNC_MAXDELAY); - g_v4_buf.next_packet_time = time_future_seconds(LIVESYNC_MAXDELAY); - - thread_started = !pthread_create(&thread_id, NULL, livesync_worker, NULL); -} - -void livesync_deinit(void) { - if (thread_started) { - /* Repeats so it interrupts a read already under way */ - while (!atomic_load(&thread_left)) { - pthread_kill(thread_id, OT_SIGNAL_WAKE); - usleep(1000); - } - pthread_join(thread_id, NULL); - thread_started = 0; - } - - if (g_socket_in != -1) - close(g_socket_in); - if (g_socket_out != -1) - close(g_socket_out); -} - -void livesync_bind_mcast(ot_ip6 ip, uint16_t port) { - if (!ip_is_v4mapped(ip)) - exerr("v6 mcast support not yet available."); - - if (g_socket_in != -1) - exerr("Error: Livesync listen ip specified twice."); - - if ((g_socket_in = socket_mcast_join(groupip_1, ip, port)) == -1) - exerr("Error: Cant bind live sync incoming socket."); - - if ((g_socket_out = socket_mcast_bind(ip, port, 0)) == -1) - exerr("Error: Cant bind live sync outgoing socket."); -} - -/* Caller MUST hold g_outbuf_mutex. Returns with g_outbuf_mutex unlocked */ -static void livesync_issue_peersync(sync_buffer *buf) { - char mycopy[LIVESYNC_OUTGOING_BUFFSIZE_PEERS]; - size_t fill = buf->fill; - - memcpy(mycopy, buf->data, fill); - buf->fill = sizeof(g_tracker_id) + sizeof(uint32_t); - buf->next_packet_time = time_future_seconds(LIVESYNC_MAXDELAY); - - /* From now this thread has a local copy of the buffer and - has modified the protected element */ - pthread_mutex_unlock(&g_outbuf_mutex); - - socket_mcast_send(g_socket_out, mycopy, fill, groupip_1, LIVESYNC_PORT); -} - -static void livesync_handle_peersync(struct ot_workstruct *ws, size_t peer_size) { - size_t off = sizeof(g_tracker_id) + sizeof(uint32_t); - - /* Prepare peer record for v4mapped expansion */ - if (peer_size == OT_PEER_SIZE4) - memcpy(&ws->peer, ip_v4mapped_prefix, sizeof(ip_v4mapped_prefix)); - - /* Now basic sanity checks have been done on the live sync packet - We might add more testing and logging. */ - while ((ssize_t)(off + sizeof(ot_hash) + peer_size) <= ws->request_size) { - char *peer_ip = ws->request + off + sizeof(ot_hash); - if (peer_size == OT_PEER_SIZE4) - memcpy(&ws->peer[0] + sizeof(ip_v4mapped_prefix), peer_ip, peer_size); - else - memcpy(&ws->peer, peer_ip, peer_size); - ws->hash = (ot_hash *)(ws->request + off); - - if (!g_opentracker_running) - return; - - if (OT_PEERFLAG(ws->peer) & PEER_FLAG_STOPPED) - remove_peer_from_torrent(FLAG_MCA, ws); - else - add_peer_to_torrent_and_return_peers(FLAG_MCA, ws, /* amount = */ 0); - - off += sizeof(ot_hash) + peer_size; - } - - stats_issue_event(ws, EVENT_SYNC, 0, (ws->request_size - sizeof(g_tracker_id) - sizeof(uint32_t)) / ((ssize_t)sizeof(ot_hash) + peer_size)); -} - -/* Tickle the live sync module from time to time, so no events get - stuck when there's not enough traffic to fill udp packets fast - enough */ -void livesync_ticker(void) { - /* livesync_issue_peersync sets g_next_packet_time */ - pthread_mutex_lock(&g_outbuf_mutex); - if (time_now_seconds() > g_v6_buf.next_packet_time && g_v6_buf.fill > sizeof(g_tracker_id) + sizeof(uint32_t)) - livesync_issue_peersync(&g_v6_buf); - else - pthread_mutex_unlock(&g_outbuf_mutex); - - pthread_mutex_lock(&g_outbuf_mutex); - if (time_now_seconds() > g_v4_buf.next_packet_time && g_v4_buf.fill > sizeof(g_tracker_id) + sizeof(uint32_t)) - livesync_issue_peersync(&g_v4_buf); - else - pthread_mutex_unlock(&g_outbuf_mutex); -} - -/* Inform live sync about whats going on. */ -void livesync_tell(struct ot_workstruct *ws) { - size_t peer_size; /* initialized in next line */ - ot_peer *peer_src = peer_from_peer6(&ws->peer, &peer_size); - sync_buffer *dest_buf = peer_size == OT_PEER_SIZE6 ? &g_v6_buf : &g_v4_buf; - - pthread_mutex_lock(&g_outbuf_mutex); - - memcpy(dest_buf->data + dest_buf->fill, ws->hash, sizeof(ot_hash)); - dest_buf->fill += sizeof(ot_hash); - - memcpy(dest_buf->data + dest_buf->fill, peer_src, peer_size); - dest_buf->fill += peer_size; - - if (dest_buf->fill >= LIVESYNC_OUTGOING_BUFFSIZE_PEERS - LIVESYNC_OUTGOING_WATERMARK_PEERS) - livesync_issue_peersync(dest_buf); - else - pthread_mutex_unlock(&g_outbuf_mutex); -} - -static void *livesync_worker(void *args) { - struct ot_workstruct ws; - ot_ip6 in_ip; - uint16_t in_port; - - (void)args; - - /* Initialize our "thread local storage" */ - ws.inbuf = ws.request = malloc(LIVESYNC_INCOMING_BUFFSIZE); - ws.outbuf = ws.reply = 0; - - while (g_opentracker_running) { - uint32_t packet_type; - - ws.request_size = socket_recv(g_socket_in, ws.inbuf, LIVESYNC_INCOMING_BUFFSIZE, in_ip, &in_port); - - /* Expect at least tracker id and packet type */ - if (ws.request_size <= (ssize_t)(sizeof(g_tracker_id) + sizeof(uint32_t))) - continue; - if (!accesslist_is_blessed(in_ip, OT_PERMISSION_MAY_LIVESYNC)) - continue; - if (!memcmp(ws.inbuf, &g_tracker_id, sizeof(g_tracker_id))) { - /* TODO: log packet coming from ourselves */ - continue; - } - - memcpy(&packet_type, ws.inbuf + sizeof(g_tracker_id), sizeof(packet_type)); - - switch (ntohl(packet_type)) { - case OT_SYNC_PEER6: - livesync_handle_peersync(&ws, OT_PEER_SIZE6); - break; - case OT_SYNC_PEER4: - livesync_handle_peersync(&ws, OT_PEER_SIZE4); - break; - default: - break; - } - } - - free(ws.inbuf); - atomic_store(&thread_left, 1); - return NULL; -} - -#endif diff --git a/ot_livesync.h b/ot_livesync.h deleted file mode 100644 index 4bd3af0..0000000 --- a/ot_livesync.h +++ /dev/null @@ -1,75 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_LIVESYNC_H__ -#define OT_LIVESYNC_H__ - -#include "trackerlogic.h" - -/* - Syncing is done as udp packets in the multicast domain 224.0.42.5 port 9696 - - Each tracker should join the multicast group and send its live sync packets - to that group, using a ttl of 1 - - Format of all sync packets is straight forward, packet type determines - which kind of packet this is: - - 0x0000 0x04 id of tracker instance - 0x0004 0x04 packet type - - ######## - ######## PEER SYNC PROTOCOL ######## - ######## - - Each tracker instance accumulates announce requests until its buffer is - full or a timeout is reached. Then it broadcasts its live sync packer: - - packet type SYNC_LIVE4 - [ 0x0008 0x14 info_hash - 0x001c 0x04 peer's ipv4 address - 0x0020 0x02 peer's port - 0x0024 0x02 peer flags v1 ( SEEDING = 0x80, COMPLETE = 0x40, STOPPED = 0x20 ) - ]* - - packet type SYNC_LIVE6 - [ 0x0008 0x14 info_hash - 0x001c 0x10 peer's ipv6 address - 0x002c 0x02 peer's port - 0x002e 0x02 peer flags v1 ( SEEDING = 0x80, COMPLETE = 0x40, STOPPED = 0x20 ) - ]* -*/ - -#ifdef WANT_SYNC_LIVE - -#define LIVESYNC_PORT 9696 - -void livesync_init(void); -void livesync_deinit(void); - -/* Join multicast group for listening and create sending socket */ -void livesync_bind_mcast(ot_ip6 ip, uint16_t port); - -/* Inform live sync about whats going on. */ -void livesync_tell(struct ot_workstruct *ws); - -/* Tickle the live sync module from time to time, so no events get - stuck when there's not enough traffic to fill udp packets fast - enough */ -void livesync_ticker(void); - -/* Handle an incoming live sync packet */ -void handle_livesync(const int64_t sock); - -#else - -/* If no syncing is required, save calling code from #ifdef - constructions */ -#define livesync_deinit() -#define livesync_init() -#define livesync_ticker() -#define handle_livesync(a) - -#endif - -#endif diff --git a/ot_mutex.c b/ot_mutex.c deleted file mode 100644 index 884b74c..0000000 --- a/ot_mutex.c +++ /dev/null @@ -1,70 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include -#include -#include -#include -#include -#include -#include - - -/* Opentracker */ -#include "ot_iovec.h" -#include "ot_mutex.h" -#include "ot_stats.h" -#include "trackerlogic.h" - -/* #define MTX_DBG( STRING ) fprintf( stderr, STRING ) */ -#define MTX_DBG(STRING) - -/* Our global all torrents list */ -static ot_vector all_torrents[OT_BUCKET_COUNT]; -static pthread_mutex_t bucket_mutex[OT_BUCKET_COUNT]; -/* Every bucket's unlock adjusts this, and a bucket lock only keeps other - threads out of that one bucket, so the counter needs its own protection */ -static _Atomic size_t g_torrent_count; - -ot_vector *mutex_bucket_lock(int bucket) { - pthread_mutex_lock(bucket_mutex + bucket); - return all_torrents + bucket; -} - -/* Torrents are spread over buckets by the first four bytes of their info_hash, - which may sit at any alignment, hence the memcpy */ -static int bucket_from_hash(ot_hash const hash) { - uint32_t bucket; - memcpy(&bucket, hash, sizeof(bucket)); - return ntohl(bucket) >> OT_BUCKET_COUNT_SHIFT; -} - -ot_vector *mutex_bucket_lock_by_hash(ot_hash const hash) { return mutex_bucket_lock(bucket_from_hash(hash)); } - -void mutex_bucket_unlock(int bucket, int delta_torrentcount) { - pthread_mutex_unlock(bucket_mutex + bucket); - if (delta_torrentcount) - atomic_fetch_add_explicit(&g_torrent_count, (size_t)delta_torrentcount, memory_order_relaxed); -} - -void mutex_bucket_unlock_by_hash(ot_hash const hash, int delta_torrentcount) { - mutex_bucket_unlock(bucket_from_hash(hash), delta_torrentcount); -} - -size_t mutex_get_torrent_count(void) { return atomic_load_explicit(&g_torrent_count, memory_order_relaxed); } - -void mutex_init(void) { - size_t i; - for (i = 0; i < OT_BUCKET_COUNT; ++i) - pthread_mutex_init(bucket_mutex + i, NULL); - memset(all_torrents, 0, sizeof(all_torrents)); -} - -void mutex_deinit(void) { - size_t i; - for (i = 0; i < OT_BUCKET_COUNT; ++i) - pthread_mutex_destroy(bucket_mutex + i); - memset(all_torrents, 0, sizeof(all_torrents)); -} diff --git a/ot_mutex.h b/ot_mutex.h deleted file mode 100644 index d7ded92..0000000 --- a/ot_mutex.h +++ /dev/null @@ -1,21 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_MUTEX_H__ -#define OT_MUTEX_H__ - -#include -#include "trackerlogic.h" - -void mutex_init(void); -void mutex_deinit(void); - -ot_vector *mutex_bucket_lock(int bucket); -ot_vector *mutex_bucket_lock_by_hash(ot_hash const hash); - -void mutex_bucket_unlock(int bucket, int delta_torrentcount); -void mutex_bucket_unlock_by_hash(ot_hash const hash, int delta_torrentcount); - -size_t mutex_get_torrent_count(void); - -#endif diff --git a/ot_random.c b/ot_random.c deleted file mode 100644 index 43f91bb..0000000 --- a/ot_random.c +++ /dev/null @@ -1,51 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include - -/* Opentracker */ -#include "ot_random.h" - -#ifdef WANT_INSECURE_RANDOM_FALLBACK - -#warning WANT_INSECURE_RANDOM_FALLBACK builds without arc4random. UDP connection ids become predictable and announces can be forged. Do not ship this. - -#include - -void random_init(void) { -#ifdef WANT_DEV_RANDOM - srandomdev(); -#else - srandom(time(NULL)); -#endif -} - -/* random() yields 31 bits, so one draw can never set the top bit. Combine - two of them rather than hand out a value that is quietly half as wide as - its type claims. */ -uint32_t random_uint32(void) { return ((uint32_t)random() << 16) ^ (uint32_t)random(); } - -void random_bytes(void *buf, size_t len) { - uint8_t *out = buf; - /* random() leaves the top bit clear and its lowest bits are the weakest, - so take a byte from the middle of each draw */ - while (len--) - *out++ = (uint8_t)(random() >> 8); -} - -#else - -#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__) && !defined(__DragonFly__) && !defined(__APPLE__) && \ - !(defined(__GLIBC__) && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 36))) -#include -#endif - -void random_init(void) {} - -uint32_t random_uint32(void) { return arc4random(); } - -void random_bytes(void *buf, size_t len) { arc4random_buf(buf, len); } - -#endif diff --git a/ot_random.h b/ot_random.h deleted file mode 100644 index c2d0387..0000000 --- a/ot_random.h +++ /dev/null @@ -1,17 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_RANDOM_H__ -#define OT_RANDOM_H__ - -#include -#include - -/* Seed whatever generator the build settled on. A no-op when backed by - arc4random, which seeds itself from the kernel. */ -void random_init(void); - -uint32_t random_uint32(void); -void random_bytes(void *buf, size_t len); - -#endif diff --git a/ot_rijndael.c b/ot_rijndael.c deleted file mode 100644 index 3f36bde..0000000 --- a/ot_rijndael.c +++ /dev/null @@ -1,488 +0,0 @@ -/** - * rijndael-alg-fst.c - * - * @version 3.0 (December 2000) - * - * Optimised ANSI C code for the Rijndael cipher (now AES) - * - * @author Vincent Rijmen - * @author Antoon Bosselaers - * @author Paulo Barreto - * - * This code is hereby placed in the public domain. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS - * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE - * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, - * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -#include "ot_rijndael.h" - -static const uint32_t Te0[256] = { - 0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU, - 0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U, - 0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU, - 0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU, - 0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U, - 0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU, - 0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU, - 0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU, - 0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU, - 0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU, - 0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U, - 0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU, - 0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU, - 0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U, - 0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU, - 0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU, - 0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU, - 0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU, - 0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU, - 0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U, - 0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU, - 0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU, - 0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU, - 0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU, - 0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U, - 0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U, - 0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U, - 0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U, - 0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU, - 0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U, - 0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U, - 0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU, - 0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU, - 0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U, - 0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U, - 0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U, - 0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU, - 0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U, - 0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU, - 0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U, - 0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU, - 0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U, - 0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U, - 0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU, - 0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U, - 0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U, - 0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U, - 0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U, - 0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U, - 0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U, - 0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U, - 0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U, - 0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU, - 0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U, - 0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U, - 0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U, - 0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U, - 0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U, - 0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U, - 0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU, - 0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U, - 0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U, - 0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U, - 0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU, -}; -static const uint32_t Te1[256] = { - 0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU, - 0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U, - 0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU, - 0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U, - 0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU, - 0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U, - 0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU, - 0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U, - 0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U, - 0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU, - 0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U, - 0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U, - 0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U, - 0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU, - 0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U, - 0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U, - 0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU, - 0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U, - 0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U, - 0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U, - 0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU, - 0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU, - 0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U, - 0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU, - 0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU, - 0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U, - 0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU, - 0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U, - 0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU, - 0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U, - 0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U, - 0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U, - 0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU, - 0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U, - 0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU, - 0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U, - 0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU, - 0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U, - 0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U, - 0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU, - 0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU, - 0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU, - 0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U, - 0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U, - 0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU, - 0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U, - 0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU, - 0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U, - 0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU, - 0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U, - 0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU, - 0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU, - 0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U, - 0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU, - 0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U, - 0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU, - 0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U, - 0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U, - 0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U, - 0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU, - 0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU, - 0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U, - 0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU, - 0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U, -}; -static const uint32_t Te2[256] = { - 0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU, - 0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U, - 0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU, - 0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U, - 0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU, - 0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U, - 0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU, - 0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U, - 0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U, - 0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU, - 0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U, - 0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U, - 0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U, - 0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU, - 0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U, - 0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U, - 0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU, - 0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U, - 0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U, - 0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U, - 0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU, - 0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU, - 0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U, - 0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU, - 0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU, - 0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U, - 0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU, - 0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U, - 0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU, - 0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U, - 0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U, - 0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U, - 0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU, - 0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U, - 0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU, - 0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U, - 0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU, - 0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U, - 0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U, - 0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU, - 0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU, - 0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU, - 0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U, - 0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U, - 0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU, - 0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U, - 0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU, - 0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U, - 0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU, - 0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U, - 0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU, - 0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU, - 0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U, - 0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU, - 0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U, - 0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU, - 0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U, - 0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U, - 0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U, - 0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU, - 0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU, - 0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U, - 0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU, - 0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U, -}; -static const uint32_t Te3[256] = { - - 0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U, - 0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U, - 0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U, - 0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU, - 0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU, - 0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU, - 0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U, - 0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU, - 0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU, - 0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U, - 0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U, - 0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU, - 0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU, - 0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU, - 0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU, - 0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU, - 0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U, - 0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU, - 0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU, - 0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U, - 0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U, - 0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U, - 0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U, - 0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U, - 0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU, - 0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U, - 0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU, - 0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU, - 0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U, - 0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U, - 0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U, - 0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU, - 0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U, - 0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU, - 0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU, - 0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U, - 0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U, - 0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU, - 0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U, - 0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU, - 0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U, - 0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U, - 0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U, - 0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U, - 0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU, - 0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U, - 0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU, - 0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U, - 0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU, - 0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U, - 0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU, - 0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU, - 0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU, - 0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU, - 0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U, - 0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U, - 0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U, - 0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U, - 0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U, - 0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U, - 0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU, - 0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U, - 0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU, - 0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU, -}; -static const uint32_t Te4[256] = { - 0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU, - 0xf2f2f2f2U, 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U, - 0x30303030U, 0x01010101U, 0x67676767U, 0x2b2b2b2bU, - 0xfefefefeU, 0xd7d7d7d7U, 0xababababU, 0x76767676U, - 0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU, - 0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U, - 0xadadadadU, 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU, - 0x9c9c9c9cU, 0xa4a4a4a4U, 0x72727272U, 0xc0c0c0c0U, - 0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, 0x26262626U, - 0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU, - 0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U, - 0x71717171U, 0xd8d8d8d8U, 0x31313131U, 0x15151515U, - 0x04040404U, 0xc7c7c7c7U, 0x23232323U, 0xc3c3c3c3U, - 0x18181818U, 0x96969696U, 0x05050505U, 0x9a9a9a9aU, - 0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U, - 0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U, - 0x09090909U, 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU, - 0x1b1b1b1bU, 0x6e6e6e6eU, 0x5a5a5a5aU, 0xa0a0a0a0U, - 0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, 0xb3b3b3b3U, - 0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U, - 0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU, - 0x20202020U, 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU, - 0x6a6a6a6aU, 0xcbcbcbcbU, 0xbebebebeU, 0x39393939U, - 0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, 0xcfcfcfcfU, - 0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU, - 0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U, - 0x45454545U, 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU, - 0x50505050U, 0x3c3c3c3cU, 0x9f9f9f9fU, 0xa8a8a8a8U, - 0x51515151U, 0xa3a3a3a3U, 0x40404040U, 0x8f8f8f8fU, - 0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U, - 0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U, - 0x10101010U, 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U, - 0xcdcdcdcdU, 0x0c0c0c0cU, 0x13131313U, 0xececececU, - 0x5f5f5f5fU, 0x97979797U, 0x44444444U, 0x17171717U, - 0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU, - 0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U, - 0x60606060U, 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU, - 0x22222222U, 0x2a2a2a2aU, 0x90909090U, 0x88888888U, - 0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, 0x14141414U, - 0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU, - 0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU, - 0x49494949U, 0x06060606U, 0x24242424U, 0x5c5c5c5cU, - 0xc2c2c2c2U, 0xd3d3d3d3U, 0xacacacacU, 0x62626262U, - 0x91919191U, 0x95959595U, 0xe4e4e4e4U, 0x79797979U, - 0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU, - 0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U, - 0x6c6c6c6cU, 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU, - 0x65656565U, 0x7a7a7a7aU, 0xaeaeaeaeU, 0x08080808U, - 0xbabababaU, 0x78787878U, 0x25252525U, 0x2e2e2e2eU, - 0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U, - 0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU, - 0x4b4b4b4bU, 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU, - 0x70707070U, 0x3e3e3e3eU, 0xb5b5b5b5U, 0x66666666U, - 0x48484848U, 0x03030303U, 0xf6f6f6f6U, 0x0e0e0e0eU, - 0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U, - 0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU, - 0xe1e1e1e1U, 0xf8f8f8f8U, 0x98989898U, 0x11111111U, - 0x69696969U, 0xd9d9d9d9U, 0x8e8e8e8eU, 0x94949494U, - 0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, 0xe9e9e9e9U, - 0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU, - 0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU, - 0xbfbfbfbfU, 0xe6e6e6e6U, 0x42424242U, 0x68686868U, - 0x41414141U, 0x99999999U, 0x2d2d2d2dU, 0x0f0f0f0fU, - 0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, 0x16161616U, -}; -static const uint32_t rcon[] = { - 0x01000000, 0x02000000, 0x04000000, 0x08000000, - 0x10000000, 0x20000000, 0x40000000, 0x80000000, - 0x1B000000, 0x36000000, /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */ -}; - -#define GETU32(pt) (((uint32_t)(pt)[0] << 24) ^ ((uint32_t)(pt)[1] << 16) ^ ((uint32_t)(pt)[2] << 8) ^ ((uint32_t)(pt)[3])) -#define PUTU32(ct, st) { (ct)[0] = (uint8_t)((st) >> 24); (ct)[1] = (uint8_t)((st) >> 16); (ct)[2] = (uint8_t)((st) >> 8); (ct)[3] = (uint8_t)(st); } - -/** - * Expand the cipher key into the encryption key schedule. - * - * @return the number of rounds for the given cipher key size. - */ -int rijndaelKeySetupEnc128(uint32_t rk[44], const uint8_t cipherKey[] ) { - int i = 0; - uint32_t temp; - - rk[0] = GETU32(cipherKey ); - rk[1] = GETU32(cipherKey + 4); - rk[2] = GETU32(cipherKey + 8); - rk[3] = GETU32(cipherKey + 12); - for (;;) { - temp = rk[3]; - rk[4] = rk[0] ^ - (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ - (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ - (Te4[(temp ) & 0xff] & 0x0000ff00) ^ - (Te4[(temp >> 24) ] & 0x000000ff) ^ - rcon[i]; - rk[5] = rk[1] ^ rk[4]; - rk[6] = rk[2] ^ rk[5]; - rk[7] = rk[3] ^ rk[6]; - if (++i == 10) { - return 10; - } - rk += 4; - } -} - -void rijndaelEncrypt128(const uint32_t rk[44], const uint8_t pt[16], uint8_t ct[16]) { - uint32_t s0, s1, s2, s3, t0, t1, t2, t3; - - /* - * map byte array block to cipher state - * and add initial round key: - */ - s0 = GETU32(pt ) ^ rk[0]; - s1 = GETU32(pt + 4) ^ rk[1]; - s2 = GETU32(pt + 8) ^ rk[2]; - s3 = GETU32(pt + 12) ^ rk[3]; - /* round 1: */ - t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[ 4]; - t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[ 5]; - t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[ 6]; - t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[ 7]; - /* round 2: */ - s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[ 8]; - s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[ 9]; - s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[10]; - s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[11]; - /* round 3: */ - t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[12]; - t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[13]; - t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[14]; - t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[15]; - /* round 4: */ - s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[16]; - s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[17]; - s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[18]; - s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[19]; - /* round 5: */ - t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[20]; - t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[21]; - t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[22]; - t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[23]; - /* round 6: */ - s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[24]; - s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[25]; - s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[26]; - s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[27]; - /* round 7: */ - t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[28]; - t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[29]; - t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[30]; - t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[31]; - /* round 8: */ - s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[32]; - s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[33]; - s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[34]; - s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[35]; - /* round 9: */ - t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[36]; - t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[37]; - t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[38]; - t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[39]; - - /* - * apply last round and - * map cipher state to byte array block: - */ - s0 = - (Te4[(t0 >> 24) ] & 0xff000000) ^ - (Te4[(t1 >> 16) & 0xff] & 0x00ff0000) ^ - (Te4[(t2 >> 8) & 0xff] & 0x0000ff00) ^ - (Te4[(t3 ) & 0xff] & 0x000000ff) ^ - rk[40]; - PUTU32(ct , s0); - s1 = - (Te4[(t1 >> 24) ] & 0xff000000) ^ - (Te4[(t2 >> 16) & 0xff] & 0x00ff0000) ^ - (Te4[(t3 >> 8) & 0xff] & 0x0000ff00) ^ - (Te4[(t0 ) & 0xff] & 0x000000ff) ^ - rk[41]; - PUTU32(ct + 4, s1); - s2 = - (Te4[(t2 >> 24) ] & 0xff000000) ^ - (Te4[(t3 >> 16) & 0xff] & 0x00ff0000) ^ - (Te4[(t0 >> 8) & 0xff] & 0x0000ff00) ^ - (Te4[(t1 ) & 0xff] & 0x000000ff) ^ - rk[42]; - PUTU32(ct + 8, s2); - s3 = - (Te4[(t3 >> 24) ] & 0xff000000) ^ - (Te4[(t0 >> 16) & 0xff] & 0x00ff0000) ^ - (Te4[(t1 >> 8) & 0xff] & 0x0000ff00) ^ - (Te4[(t2 ) & 0xff] & 0x000000ff) ^ - rk[43]; - PUTU32(ct + 12, s3); -} diff --git a/ot_rijndael.h b/ot_rijndael.h deleted file mode 100644 index 1235215..0000000 --- a/ot_rijndael.h +++ /dev/null @@ -1,20 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. - - The rijndael implementation was taken from - - http://www.cs.ucdavis.edu/~rogaway/ocb/ocb-ref/rijndael-alg-fst.c - - and modified to work with 128 bits (this is 10 rounds) only. */ - -#ifndef OT_RIJNDAEL_H__ -#define OT_RIJNDAEL_H__ - -#include - -int rijndaelKeySetupEnc128(uint32_t rk[44], const uint8_t cipherKey[] ); -void rijndaelEncrypt128(const uint32_t rk[44], const uint8_t pt[16], uint8_t ct[16]); - -extern const char *g_version_rijndael_c; - -#endif diff --git a/ot_scan.c b/ot_scan.c deleted file mode 100644 index 3b8e87c..0000000 --- a/ot_scan.c +++ /dev/null @@ -1,65 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include -#include - -/* Opentracker */ -#include "ot_scan.h" - -size_t scan_uint(const char *src, uint64_t *dest, uint64_t max) { - unsigned long long value; - char *end; - - if (*src < '0' || *src > '9') - return 0; - - errno = 0; - value = strtoull(src, &end, 10); - - if (errno == ERANGE || value > (unsigned long long)max) - return 0; - - *dest = (uint64_t)value; - return (size_t)(end - src); -} - -size_t scan_uint_bencode(const char *src, uint64_t *dest, uint64_t max) { - uint64_t value; - size_t consumed = scan_uint(src, &value, max); - - /* bencode writes no leading zeros, so anything longer than a single digit - must not begin with one. */ - if (consumed > 1 && *src == '0') - return 0; - - if (consumed) - *dest = value; - - return consumed; -} - -int scan_int(const char *src, size_t len, int64_t *dest) { - int64_t value = 0; - int negate = 0; - - if (len && *src == '-') - --len, ++src, negate = 1; - if (!len) - return -1; - - while (len--) { - if (*src < '0' || *src > '9') - return -1; - /* A twenty character field of nines does not fit, and silently wrapping - would magically heal a nonsense announce */ - if (value > (INT64_MAX - (*src - '0')) / 10) - return -1; - value = 10 * value + (*src++ - '0'); - } - - *dest = negate ? -value : value; - return 0; -} diff --git a/ot_scan.h b/ot_scan.h deleted file mode 100644 index a79ec13..0000000 --- a/ot_scan.h +++ /dev/null @@ -1,25 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_SCAN_H__ -#define OT_SCAN_H__ - -#include -#include -#include - -/* Read an unsigned decimal number from src into dest and return how many - characters were consumed. Returns zero and leaves dest untouched when src - does not begin with a digit, or when the value would exceed max. */ -size_t scan_uint(const char *src, uint64_t *dest, uint64_t max); - -/* As scan_uint, but following the bencode integer grammar, which permits a - leading zero on zero alone. */ -size_t scan_uint_bencode(const char *src, uint64_t *dest, uint64_t max); - -/* Read a signed decimal number that has to take up exactly len characters, - which is how a value lifted out of a query string arrives. Returns 0, or - -1 leaving dest untouched. */ -int scan_int(const char *src, size_t len, int64_t *dest); - -#endif diff --git a/ot_socket.c b/ot_socket.c deleted file mode 100644 index 79bb01a..0000000 --- a/ot_socket.c +++ /dev/null @@ -1,310 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#define _GNU_SOURCE /* accept4 on glibc */ -#include -#include -#include -#include -#include -#include -#include - -/* Opentracker */ -#include "ot_ip.h" -#include "ot_socket.h" - -static void sockaddr_from_ip(struct sockaddr_in6 *sa, const ot_ip6 ip, uint16_t port) { - memset(sa, 0, sizeof(*sa)); - sa->sin6_family = AF_INET6; - sa->sin6_port = htons(port); - memcpy(&sa->sin6_addr, ip, sizeof(ot_ip6)); -} - -/* Linux reports an IPv4 peer on a dual stack socket as a v4 mapped - sockaddr_in6, the BSDs and macOS hand back a plain sockaddr_in instead, so - both forms have to be read. */ -static void ip_from_sockaddr(ot_ip6 ip, uint16_t *port, const struct sockaddr_storage *ss) { - if (ss->ss_family == AF_INET) { - const struct sockaddr_in *sa = (const struct sockaddr_in *)ss; - - memcpy(ip, ip_v4mapped_prefix, sizeof(ip_v4mapped_prefix)); - memcpy(ip + sizeof(ip_v4mapped_prefix), &sa->sin_addr, 4); - *port = ntohs(sa->sin_port); - } else { - const struct sockaddr_in6 *sa = (const struct sockaddr_in6 *)ss; - - memcpy(ip, &sa->sin6_addr, sizeof(ot_ip6)); - *port = ntohs(sa->sin6_port); - } -} - -int socket_set_blocking(int s, int blocking) { - int flags = fcntl(s, F_GETFL, 0); - - if (flags == -1) - return -1; - return fcntl(s, F_SETFL, blocking ? (flags & ~O_NONBLOCK) : (flags | O_NONBLOCK)); -} - -static int set_nonblocking(int s) { return socket_set_blocking(s, 0); } - -/* Close without letting the failure we are about to report be overwritten */ -static int close_and_fail(int s) { - int saved = errno; - close(s); - errno = saved; - return -1; -} - -static int socket_v6(int type) { - int s = socket(PF_INET6, type, 0); - int zero = 0; - - if (s == -1) - return -1; - - /* A tracker that answered only one family would be half a tracker */ - if (setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &zero, sizeof(zero)) == -1) - return close_and_fail(s); - - if (set_nonblocking(s) == -1) - return close_and_fail(s); - - return s; -} - -/* SO_REUSEPORT means three different things. Linux spreads datagrams over - the group. FreeBSD permits the bind and spreads nothing, which is what - SO_REUSEPORT_LB was added for. Darwin hands every datagram to whichever - socket bound last, so a group there starves all but one reader. */ -#if defined(SO_REUSEPORT_LB) -#define OT_BALANCE_OPTION SO_REUSEPORT_LB -#elif defined(SO_REUSEPORT) && !defined(__APPLE__) -#define OT_BALANCE_OPTION SO_REUSEPORT -#endif - -int socket_can_balance(void) { -#ifdef OT_BALANCE_OPTION - return 1; -#else - return 0; -#endif -} - -/* The only place in the tree that needs to know what a kernel calls this */ -static void socket_set_sharing(int s, socket_sharing sharing) { - int one = 1; - - setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); -#ifdef OT_BALANCE_OPTION - if (sharing == SOCKET_BALANCED) { - setsockopt(s, SOL_SOCKET, OT_BALANCE_OPTION, &one, sizeof(one)); - return; - } -#endif -#ifdef SO_REUSEPORT - if (sharing != SOCKET_EXCLUSIVE) - setsockopt(s, SOL_SOCKET, SO_REUSEPORT, &one, sizeof(one)); -#endif -} - -static int socket_bind_v6(int s, const ot_ip6 ip, uint16_t port, socket_sharing sharing) { - struct sockaddr_in6 sa; - - socket_set_sharing(s, sharing); - - sockaddr_from_ip(&sa, ip, port); - return bind(s, (struct sockaddr *)&sa, sizeof(sa)); -} - -/* Defer the listening socket reporting it to be ready to accept until enough - sensible data is available for an accept+read pair. Where available. */ -int socket_defer_accept(int s) { -#if defined(TCP_DEFER_ACCEPT) - int patience = OT_CLIENT_TIMEOUT; - - return setsockopt(s, IPPROTO_TCP, TCP_DEFER_ACCEPT, &patience, sizeof(patience)); -#elif defined(SO_ACCEPTFILTER) - struct accept_filter_arg filter; - const char name[sizeof(filter.af_name)] = "dataready"; - - memset(&filter, 0, sizeof(filter)); - memcpy(filter.af_name, name, sizeof(name)); - return setsockopt(s, SOL_SOCKET, SO_ACCEPTFILTER, &filter, sizeof(filter)); -#else - (void)s; - return -1; -#endif -} - -int socket_tcp_listen(const ot_ip6 ip, uint16_t port, socket_sharing sharing) { - int s = socket_v6(SOCK_STREAM); - - if (s == -1) - return -1; - if (socket_bind_v6(s, ip, port, sharing) == -1 || listen(s, SOMAXCONN) == -1) - return close_and_fail(s); - - return s; -} - -int socket_udp_bind(const ot_ip6 ip, uint16_t port, socket_sharing sharing) { - int s = socket_v6(SOCK_DGRAM); - - if (s == -1) - return -1; - if (socket_bind_v6(s, ip, port, sharing) == -1) - return close_and_fail(s); - - return s; -} - -int socket_tcp_connect(const ot_ip6 srcip, uint16_t srcport, const ot_ip6 dstip, uint16_t dstport) { - struct sockaddr_in6 sa; - int s = socket_v6(SOCK_STREAM); - - if (s == -1) - return -1; - if (socket_bind_v6(s, srcip, srcport, SOCKET_EXCLUSIVE) == -1) - return close_and_fail(s); - - sockaddr_from_ip(&sa, dstip, dstport); - if (connect(s, (struct sockaddr *)&sa, sizeof(sa)) == -1 && errno != EINPROGRESS && errno != EWOULDBLOCK) - return close_and_fail(s); - - return s; -} - -int socket_is_connected(int s) { - int error = 0; - socklen_t len = sizeof(error); - - if (getsockopt(s, SOL_SOCKET, SO_ERROR, &error, &len) == -1) - return 0; - if (error) { - errno = error; - return 0; - } - return 1; -} - -/* Linux hands over a blocking socket whatever the listening socket was, the - BSDs and macOS inherit the listener's flags, so the accepted socket has to - be told either way. accept4 carries the flag itself, which saves the two - fcntl calls it otherwise takes. Darwin has neither accept4 nor the flag. */ -int socket_accept(int s, ot_ip6 ip, uint16_t *port) { - struct sockaddr_storage ss; - socklen_t len = sizeof(ss); - int fd; - - memset(&ss, 0, sizeof(ss)); -#ifdef SOCK_NONBLOCK - if ((fd = accept4(s, (struct sockaddr *)&ss, &len, SOCK_NONBLOCK)) == -1) - return -1; -#else - if ((fd = accept(s, (struct sockaddr *)&ss, &len)) == -1) - return -1; - if (set_nonblocking(fd) == -1) - return close_and_fail(fd); -#endif - - ip_from_sockaddr(ip, port, &ss); - return fd; -} - -ssize_t socket_recv(int s, void *buf, size_t len, ot_ip6 ip, uint16_t *port) { - struct sockaddr_storage ss; - socklen_t sslen = sizeof(ss); - ssize_t received; - - memset(&ss, 0, sizeof(ss)); - received = recvfrom(s, buf, len, 0, (struct sockaddr *)&ss, &sslen); - if (received == -1) - return -1; - - ip_from_sockaddr(ip, port, &ss); - return received; -} - -ssize_t socket_send(int s, const void *buf, size_t len, const ot_ip6 ip, uint16_t port) { - struct sockaddr_in6 sa; - - sockaddr_from_ip(&sa, ip, port); - return sendto(s, buf, len, 0, (struct sockaddr *)&sa, sizeof(sa)); -} - -#ifdef WANT_SYNC_LIVE -static void sockaddr4_from_ip(struct sockaddr_in *sa, const ot_ip6 ip, uint16_t port) { - memset(sa, 0, sizeof(*sa)); - sa->sin_family = AF_INET; - sa->sin_port = htons(port); - memcpy(&sa->sin_addr, ip + sizeof(ip_v4mapped_prefix), 4); -} - -/* A multicast group is meant to have several listeners, so these are always - shared */ -static int socket_v4_bind(const ot_ip6 ip, uint16_t port) { - struct sockaddr_in sa; - int s; - - if ((s = socket(PF_INET, SOCK_DGRAM, 0)) == -1) - return -1; - - socket_set_sharing(s, SOCKET_SHARED); - - sockaddr4_from_ip(&sa, ip, port); - if (bind(s, (struct sockaddr *)&sa, sizeof(sa)) == -1) - return close_and_fail(s); - - return s; -} - -int socket_mcast_join(const ot_ip6 group, const ot_ip6 interface, uint16_t port) { - struct ip_mreq mreq; - ot_ip6 any; - int s; - - /* The group is what is listened to, so the socket takes the wildcard and - the join names the interface to receive it on */ - memcpy(any, ip_v4mapped_prefix, sizeof(ip_v4mapped_prefix)); - memset(any + sizeof(ip_v4mapped_prefix), 0, 4); - - if ((s = socket_v4_bind(any, port)) == -1) - return -1; - - memset(&mreq, 0, sizeof(mreq)); - memcpy(&mreq.imr_multiaddr, group + sizeof(ip_v4mapped_prefix), 4); - memcpy(&mreq.imr_interface, interface + sizeof(ip_v4mapped_prefix), 4); - if (setsockopt(s, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) == -1) - return close_and_fail(s); - - return s; -} - -int socket_mcast_bind(const ot_ip6 interface, uint16_t port, int loopback) { - /* The BSDs and macOS want a byte for the IPv4 multicast options, Linux - accepts either that or an int. */ - unsigned char ttl = 1, loop = (unsigned char)(loopback != 0); - int s = socket_v4_bind(interface, port); - - if (s == -1) - return -1; - if (set_nonblocking(s) == -1) - return close_and_fail(s); - - if (setsockopt(s, IPPROTO_IP, IP_MULTICAST_TTL, &ttl, sizeof(ttl)) == -1 || - setsockopt(s, IPPROTO_IP, IP_MULTICAST_LOOP, &loop, sizeof(loop)) == -1) - return close_and_fail(s); - - return s; -} - -ssize_t socket_mcast_send(int s, const void *buf, size_t len, const ot_ip6 group, uint16_t port) { - struct sockaddr_in sa; - - sockaddr4_from_ip(&sa, group, port); - return sendto(s, buf, len, 0, (struct sockaddr *)&sa, sizeof(sa)); -} -#endif diff --git a/ot_socket.h b/ot_socket.h deleted file mode 100644 index 5c1d840..0000000 --- a/ot_socket.h +++ /dev/null @@ -1,56 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_SOCKET_H__ -#define OT_SOCKET_H__ - -#include -#include - -#include "trackerlogic.h" - -/* Every socket opentracker opens is an AF_INET6 socket with IPV6_V6ONLY - cleared, so one descriptor serves both families and an IPv4 peer arrives - as the v4 mapped address */ - -/* How an address may be shared with other sockets. Which socket option says - so differs between kernels, which is socket_set_sharing's business. - - SHARED lets another socket bind the same address. BALANCED asks the kernel - to spread arriving datagrams over the sockets that did, which is a - different thing and not everywhere available: ask socket_can_balance - before binding a group that depends on it. */ -typedef enum { SOCKET_EXCLUSIVE, SOCKET_SHARED, SOCKET_BALANCED } socket_sharing; - -/* Whether this kernel spreads datagrams over a group of sockets bound to one - address. Where it does not, binding a group is worse than one socket with - several readers, so the caller wants to know first. */ -int socket_can_balance(void); - -/* Create, bind and, for TCP, listen. The returned socket is non blocking - Returns -1 on failure with errno set */ -int socket_tcp_listen(const ot_ip6 ip, uint16_t port, socket_sharing sharing); -int socket_udp_bind(const ot_ip6 ip, uint16_t port, socket_sharing sharing); - -int socket_tcp_connect(const ot_ip6 srcip, uint16_t srcport, const ot_ip6 dstip, uint16_t dstport); -int socket_is_connected(int s); -int socket_accept(int s, ot_ip6 ip, uint16_t *port); -int socket_set_blocking(int s, int blocking); -int socket_defer_accept(int s); - -ssize_t socket_recv(int s, void *buf, size_t len, ot_ip6 ip, uint16_t *port); -ssize_t socket_send(int s, const void *buf, size_t len, const ot_ip6 ip, uint16_t port); - -#ifdef WANT_SYNC_LIVE -/* - socket_mcast_join takes the wildcard address and names the interface in the - join, and hands back a blocking socket, since the thread that reads it does - nothing else. socket_mcast_bind hands back a non blocking one, because the - thread that sends from it is also serving clients. Incoming datagrams are - read with socket_recv */ -int socket_mcast_join(const ot_ip6 group, const ot_ip6 interface, uint16_t port); -int socket_mcast_bind(const ot_ip6 interface, uint16_t port, int loopback); -ssize_t socket_mcast_send(int s, const void *buf, size_t len, const ot_ip6 group, uint16_t port); -#endif - -#endif diff --git a/ot_stats.c b/ot_stats.c deleted file mode 100644 index 2d6335b..0000000 --- a/ot_stats.c +++ /dev/null @@ -1,866 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef WANT_SYSLOGS -#include -#endif - -/* Opentracker */ -#include "ot_accesslist.h" -#include "ot_event.h" -#include "ot_hex.h" -#include "ot_iovec.h" -#include "ot_ip.h" -#include "ot_mutex.h" -#include "ot_workqueue.h" -#include "ot_fmt.h" -#include "ot_stats.h" -#include "trackerlogic.h" - -#ifndef NO_FULLSCRAPE_LOGGING -#define LOG_TO_STDERR(...) fprintf(stderr, __VA_ARGS__) -#else -#define LOG_TO_STDERR(...) -#endif - -/* Forward declaration */ -static void stats_make(size_t *iovec_entries, struct iovec **iovector, ot_tasktype mode); -#define OT_STATS_TMPSIZE 8192 - -/* Each serving thread keeps its own statistics tally. The list is linked - through intrusive next pointers within the workstruct. - - Registration happens once per thread and the adding up once per statistics - request, so this lock is never on anybody's hot path. */ -static struct ot_workstruct *g_counter_contexts; -static ot_counter g_retired[OT_COUNTER_COUNT]; -static pthread_mutex_t g_counter_mutex = PTHREAD_MUTEX_INITIALIZER; - -void stats_counters_register(struct ot_workstruct *ws) { - pthread_mutex_lock(&g_counter_mutex); - ws->counters_next = g_counter_contexts; - g_counter_contexts = ws; - pthread_mutex_unlock(&g_counter_mutex); -} - -void stats_counters_retire(struct ot_workstruct *ws) { - struct ot_workstruct **link; - size_t i; - - pthread_mutex_lock(&g_counter_mutex); - for (link = &g_counter_contexts; *link; link = &(*link)->counters_next) - if (*link == ws) { - *link = ws->counters_next; - break; - } - - /* When retiring a worker thread, collect the counter tally in the global */ - for (i = 0; i < OT_COUNTER_COUNT; ++i) - atomic_fetch_add_explicit(g_retired + i, atomic_load_explicit(ws->counters + i, memory_order_relaxed), memory_order_relaxed); - pthread_mutex_unlock(&g_counter_mutex); -} - -/* What every thread has counted so far, added together */ -static void counters_total(unsigned long long *total) { - struct ot_workstruct *ws; - size_t i; - - pthread_mutex_lock(&g_counter_mutex); - for (i = 0; i < OT_COUNTER_COUNT; ++i) - total[i] = atomic_load_explicit(g_retired + i, memory_order_relaxed); - for (ws = g_counter_contexts; ws; ws = ws->counters_next) - for (i = 0; i < OT_COUNTER_COUNT; ++i) - total[i] += atomic_load_explicit(ws->counters + i, memory_order_relaxed); - pthread_mutex_unlock(&g_counter_mutex); -} - -#define COUNT(index, by) atomic_fetch_add_explicit(ws->counters + (index), (unsigned long long)(by), memory_order_relaxed) - -static const char *ot_failed_request_names[] = { - "302 Redirect", "400 Parse Error", "400 Invalid Parameter", "400 Invalid Parameter (compact=0)", "400 Not Modest", - "402 Payment Required", "403 Access Denied", "404 Not found", "500 Internal Server Error"}; - -static time_t ot_start_time; - -#define STATS_NETWORK_NODE_BITWIDTH 4 -#define STATS_NETWORK_NODE_COUNT (1 << STATS_NETWORK_NODE_BITWIDTH) - -#define __BYTE(P, D) (((uint8_t *)P)[D / 8]) -#define __MSK (STATS_NETWORK_NODE_COUNT - 1) -#define __SHFT(D) ((D ^ STATS_NETWORK_NODE_BITWIDTH) & STATS_NETWORK_NODE_BITWIDTH) - -#define __LDR(P, D) ((__BYTE((P), (D)) >> __SHFT((D))) & __MSK) -#define __STR(P, D, V) __BYTE((P), (D)) = (__BYTE((P), (D)) & ~(__MSK << __SHFT((D)))) | ((V) << __SHFT((D))) - -#if 0 -// XXX -#define STATS_NETWORK_NODE_MAXDEPTH (68 - STATS_NETWORK_NODE_BITWIDTH) -#define STATS_NETWORK_NODE_LIMIT (48 - STATS_NETWORK_NODE_BITWIDTH) -#endif -#define STATS_NETWORK_NODE_MAXDEPTH (28 - STATS_NETWORK_NODE_BITWIDTH) -#define STATS_NETWORK_NODE_LIMIT (24 - STATS_NETWORK_NODE_BITWIDTH) - -typedef union stats_network_node stats_network_node; -union stats_network_node { - size_t counters[STATS_NETWORK_NODE_COUNT]; - stats_network_node *children[STATS_NETWORK_NODE_COUNT]; -}; - -static int stat_increase_network_count(stats_network_node **pnode, int depth, uintptr_t ip) { - int foo = __LDR(ip, depth); - stats_network_node *node; - - if (!*pnode) { - *pnode = malloc(sizeof(stats_network_node)); - if (!*pnode) - return -1; - memset(*pnode, 0, sizeof(stats_network_node)); - } - node = *pnode; - - if (depth < STATS_NETWORK_NODE_MAXDEPTH) - return stat_increase_network_count(node->children + foo, depth + STATS_NETWORK_NODE_BITWIDTH, ip); - - node->counters[foo]++; - return 0; -} - -static int stats_shift_down_network_count(stats_network_node **node, int depth, int shift) { - size_t i, rest = 0; - - if (!*node) - return 0; - - for (i = 0; i < STATS_NETWORK_NODE_COUNT; ++i) - if (depth < STATS_NETWORK_NODE_MAXDEPTH) - rest += stats_shift_down_network_count((*node)->children + i, depth + STATS_NETWORK_NODE_BITWIDTH, shift); - else - rest += (*node)->counters[i] >>= shift; - - if (!rest) { - free(*node); - *node = NULL; - } - - return rest; -} - -static size_t stats_get_highscore_networks(stats_network_node *node, int depth, ot_ip6 node_value, size_t *scores, ot_ip6 *networks, int network_count, - int limit) { - size_t i, score = 0; - - if (!node) - return 0; - - if (depth < limit) { - for (i = 0; i < STATS_NETWORK_NODE_COUNT; ++i) - if (node->children[i]) { - __STR(node_value, depth, i); - score += stats_get_highscore_networks(node->children[i], depth + STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit); - } - return score; - } - - if (depth > limit && depth < STATS_NETWORK_NODE_MAXDEPTH) { - for (i = 0; i < STATS_NETWORK_NODE_COUNT; ++i) - if (node->children[i]) - score += stats_get_highscore_networks(node->children[i], depth + STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit); - return score; - } - - if (depth > limit && depth == STATS_NETWORK_NODE_MAXDEPTH) { - for (i = 0; i < STATS_NETWORK_NODE_COUNT; ++i) - score += node->counters[i]; - return score; - } - - /* if( depth == limit ) */ - for (i = 0; i < STATS_NETWORK_NODE_COUNT; ++i) { - int j = 1; - size_t node_score; - - if (depth == STATS_NETWORK_NODE_MAXDEPTH) - node_score = node->counters[i]; - else - node_score = stats_get_highscore_networks(node->children[i], depth + STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit); - - score += node_score; - - if (node_score <= scores[0]) - continue; - - __STR(node_value, depth, i); - while (j < network_count && node_score > scores[j]) - ++j; - --j; - - memmove(scores, scores + 1, j * sizeof(*scores)); - memmove(networks, networks + 1, j * sizeof(*networks)); - scores[j] = node_score; - memcpy(networks + j, node_value, sizeof(*networks)); - } - - return score; -} - -static void stats_return_busy_networks(char **reply, const char *reply_end, stats_network_node *tree, size_t amount, int limit) { - ot_ip6 networks[amount]; - ot_ip6 node_value; - size_t scores[amount]; - size_t i; - - memset(scores, 0, sizeof(scores)); - memset(networks, 0, sizeof(networks)); - memset(node_value, 0, sizeof(node_value)); - - stats_get_highscore_networks(tree, 0, node_value, scores, networks, amount, limit); - - fmt_append(reply, reply_end, "Networks, limit /%d:\n", limit + STATS_NETWORK_NODE_BITWIDTH); - for (i = amount; i > 0; --i) { - if (scores[i - 1]) { - char ip_readable[64]; - - ip_readable[ip_format(ip_readable, networks[i - 1])] = 0; - fmt_append(reply, reply_end, "%08zd: %s\n", scores[i - 1], ip_readable); - } - } - fmt_append(reply, reply_end, "\n"); -} - -static void stats_slash24s_txt(char **reply, const char *reply_end, size_t amount) { - stats_network_node *slash24s_network_counters_root = NULL; - size_t i, bucket, peer_size = OT_PEER_SIZE4; - - for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { - ot_vector *torrents_list = mutex_bucket_lock(bucket); - for (i = 0; i < torrents_list->size; ++i) { - ot_peerlist *peer_list = (((ot_torrent *)(torrents_list->data))[i]).peer_list4; - ot_vector *bucket_list = &peer_list->peers; - int num_buckets = 1; - - if (OT_PEERLIST_HASBUCKETS(peer_list)) { - num_buckets = bucket_list->size; - bucket_list = (ot_vector *)bucket_list->data; - } - - while (num_buckets--) { - ot_peer *peers = (ot_peer *)bucket_list->data; - size_t numpeers = bucket_list->size; - while (numpeers--) { - if (stat_increase_network_count(&slash24s_network_counters_root, 0, (uintptr_t)(peers))) - goto bailout_unlock; - peers += peer_size; - } - ++bucket_list; - } - } - mutex_bucket_unlock(bucket, 0); - if (!g_opentracker_running) - goto done; - } - - /* The tree is built. Now analyze */ - stats_return_busy_networks(reply, reply_end, slash24s_network_counters_root, amount, STATS_NETWORK_NODE_MAXDEPTH); - stats_return_busy_networks(reply, reply_end, slash24s_network_counters_root, amount, STATS_NETWORK_NODE_LIMIT); - goto done; - -bailout_unlock: - mutex_bucket_unlock(bucket, 0); -done: - stats_shift_down_network_count(&slash24s_network_counters_root, 0, sizeof(int) * 8 - 1); -} - -#ifdef WANT_SPOT_WOODPECKER -static stats_network_node *stats_woodpeckers_tree; -static pthread_mutex_t g_woodpeckers_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void stats_return_woodpeckers(char **reply, const char *reply_end, int amount) { - pthread_mutex_lock(&g_woodpeckers_mutex); - stats_return_busy_networks(reply, reply_end, stats_woodpeckers_tree, amount, STATS_NETWORK_NODE_MAXDEPTH); - pthread_mutex_unlock(&g_woodpeckers_mutex); -} -#endif - -typedef struct { - unsigned long long torrent_count; - unsigned long long peer_count; - unsigned long long seed_count; -} torrent_stats; - -static int torrent_statter(ot_torrent *torrent, uintptr_t data) { - torrent_stats *stats = (torrent_stats *)data; - stats->torrent_count++; - stats->peer_count += torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; - stats->seed_count += torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; - return 0; -} - -/* Converter function from memory to human readable hex strings */ -typedef struct { - size_t val; - ot_hash hash; -} ot_record; - -static inline void stats_insert_into_highscore(size_t val, ot_record* list, size_t amount, ot_hash* hash) { - size_t i; - for (i = 0; (i < amount) && (list[i].val > val); ++i) - {} - /* if we score better than a record in the list, we need to make room in the high - score array by moving lower scoring entries up */ - if (i < amount) { - memmove(list + i + 1, list + i, (amount - i - 1) * sizeof(ot_record)); - - memcpy(&list[i].hash, hash, sizeof(ot_hash)); - list[i].val = val; - } -} - - -/* Fetches stats from tracker */ -void stats_top_txt(char **reply, const char *reply_end, size_t amount) { - size_t j, idx, bucket; - ot_record top100s[100], top100c[100], top100l[100]; - char hex_out[42]; - - if (amount > 100) - amount = 100; - - memset(top100s, 0, sizeof(top100s)); - memset(top100c, 0, sizeof(top100c)); - memset(top100l, 0, sizeof(top100l)); - - /* Iterate over complete torrent list */ - for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { - ot_vector *torrents_list = mutex_bucket_lock(bucket); - for (j = 0; j < torrents_list->size; ++j) { - ot_torrent *torrent = (ot_torrent *)(torrents_list->data) + j; - size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; - size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; - size_t leech_count = peer_count - seed_count; - - stats_insert_into_highscore(peer_count, top100c, amount, &torrent->hash); - stats_insert_into_highscore(seed_count, top100s, amount, &torrent->hash); - stats_insert_into_highscore(leech_count, top100l, amount, &torrent->hash); - } - mutex_bucket_unlock(bucket, 0); - if (!g_opentracker_running) - return; - } - - fmt_append(reply, reply_end, "Top %zd torrents by peers:\n", amount); - for (idx = 0; idx < amount; ++idx) - if (top100c[idx].val) - fmt_append(reply, reply_end, "\t%zd\t%s\n", top100c[idx].val, hex_from_hash(hex_out, top100c[idx].hash)); - fmt_append(reply, reply_end, "Top %zd torrents by seeds:\n", amount); - for (idx = 0; idx < amount; ++idx) - if (top100s[idx].val) - fmt_append(reply, reply_end, "\t%zd\t%s\n", top100s[idx].val, hex_from_hash(hex_out, top100s[idx].hash)); - fmt_append(reply, reply_end, "Top %zd torrents by leechers:\n", amount); - for (idx = 0; idx < amount; ++idx) - if (top100l[idx].val) - fmt_append(reply, reply_end, "\t%zd\t%s\n", top100l[idx].val, hex_from_hash(hex_out, top100l[idx].hash)); -} - -static unsigned long events_per_time(unsigned long long events, time_t t) { return events / ((unsigned int)t ? (unsigned int)t : 1); } - -static void stats_connections_mrtg(char **reply, const char *reply_end) { - ot_time t = time(NULL) - ot_start_time; - unsigned long long total[OT_COUNTER_COUNT]; - - counters_total(total); - - fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker connections, %lu conns/s :: %lu success/s.", - total[OT_COUNTER_TCP_CONNECTIONS] + total[OT_COUNTER_UDP_CONNECTIONS], - total[OT_COUNTER_TCP_ANNOUNCES] + total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], (int)t, (int)(t / 3600), - events_per_time(total[OT_COUNTER_TCP_CONNECTIONS] + total[OT_COUNTER_UDP_CONNECTIONS], t), - events_per_time(total[OT_COUNTER_TCP_ANNOUNCES] + total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], t)); -} - -static void stats_udpconnections_mrtg(char **reply, const char *reply_end) { - ot_time t = time(NULL) - ot_start_time; - unsigned long long total[OT_COUNTER_COUNT]; - - counters_total(total); - - fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker udp4 stats, %lu conns/s :: %lu success/s.", total[OT_COUNTER_UDP_CONNECTIONS], - total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], (int)t, (int)(t / 3600), - events_per_time(total[OT_COUNTER_UDP_CONNECTIONS], t), - events_per_time(total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], t)); -} - -static void stats_tcpconnections_mrtg(char **reply, const char *reply_end) { - time_t t = time(NULL) - ot_start_time; - unsigned long long total[OT_COUNTER_COUNT]; - - counters_total(total); - - fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker tcp4 stats, %lu conns/s :: %lu success/s.", total[OT_COUNTER_TCP_CONNECTIONS], - total[OT_COUNTER_TCP_ANNOUNCES], (int)t, (int)(t / 3600), events_per_time(total[OT_COUNTER_TCP_CONNECTIONS], t), - events_per_time(total[OT_COUNTER_TCP_ANNOUNCES], t)); -} - -static void stats_scrape_mrtg(char **reply, const char *reply_end) { - time_t t = time(NULL) - ot_start_time; - unsigned long long total[OT_COUNTER_COUNT]; - - counters_total(total); - - fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker scrape stats, %lu scrape/s (tcp and udp)", - total[OT_COUNTER_TCP_SCRAPES], total[OT_COUNTER_UDP_SCRAPES], (int)t, (int)(t / 3600), - events_per_time((total[OT_COUNTER_TCP_SCRAPES] + total[OT_COUNTER_UDP_SCRAPES]), t)); -} - -static void stats_fullscrapes_mrtg(char **reply, const char *reply_end) { - ot_time t = time(NULL) - ot_start_time; - unsigned long long total[OT_COUNTER_COUNT]; - - counters_total(total); - - fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker full scrape stats, %lu conns/s :: %lu bytes/s.", - total[OT_COUNTER_FULLSCRAPES] * 1000, total[OT_COUNTER_FULLSCRAPE_BYTES], (int)t, (int)(t / 3600), - events_per_time(total[OT_COUNTER_FULLSCRAPES], t), - events_per_time(total[OT_COUNTER_FULLSCRAPE_BYTES], t)); -} - -static void stats_peers_mrtg(char **reply, const char *reply_end) { - torrent_stats stats = {0, 0, 0}; - - iterate_all_torrents(torrent_statter, (uintptr_t)&stats); - - fmt_append(reply, reply_end, "%llu\n%llu\nopentracker serving %llu torrents\nopentracker", stats.peer_count, stats.seed_count, - stats.torrent_count); -} - -static void stats_torrents_mrtg(char **reply, const char *reply_end) { - size_t torrent_count = mutex_get_torrent_count(); - - fmt_append(reply, reply_end, "%zd\n%zd\nopentracker serving %zd torrents\nopentracker", torrent_count, (size_t)0, torrent_count); -} - -static void stats_httperrors_txt(char **reply, const char *reply_end) { - unsigned long long total[OT_COUNTER_COUNT]; - - counters_total(total); - - fmt_append(reply, reply_end, "302 RED %llu\n400 ... %llu\n400 PAR %llu\n400 COM %llu\n403 IP %llu\n404 INV %llu\n500 SRV %llu\n", - total[OT_COUNTER_FAILED + 0], total[OT_COUNTER_FAILED + 1], total[OT_COUNTER_FAILED + 2], total[OT_COUNTER_FAILED + 3], - total[OT_COUNTER_FAILED + 4], total[OT_COUNTER_FAILED + 5], total[OT_COUNTER_FAILED + 6]); -} - -static void stats_return_renew_bucket(char **reply, const char *reply_end) { - unsigned long long total[OT_COUNTER_COUNT]; - - counters_total(total); - - int i; - - for (i = 0; i < OT_PEER_TIMEOUT; ++i) - fmt_append(reply, reply_end, "%02i %llu\n", i, total[OT_COUNTER_RENEWED + i]); -} - -static void stats_return_sync_mrtg(char **reply, const char *reply_end) { - ot_time t = time(NULL) - ot_start_time; - unsigned long long total[OT_COUNTER_COUNT]; - - counters_total(total); - - fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker connections, %lu conns/s :: %lu success/s.", total[OT_COUNTER_SYNCS], 0LL, - (int)t, (int)(t / 3600), events_per_time(total[OT_COUNTER_TCP_CONNECTIONS] + total[OT_COUNTER_UDP_CONNECTIONS], t), - events_per_time(total[OT_COUNTER_TCP_ANNOUNCES] + total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], t)); -} - -static void stats_return_completed_mrtg(char **reply, const char *reply_end) { - ot_time t = time(NULL) - ot_start_time; - unsigned long long total[OT_COUNTER_COUNT]; - - counters_total(total); - - - fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker, %lu completed/h.", total[OT_COUNTER_COMPLETED], 0LL, (int)t, (int)(t / 3600), - events_per_time(total[OT_COUNTER_COMPLETED], t / 3600)); -} - -#ifdef WANT_LOG_NUMWANT -extern unsigned long long numwants[201]; -static void stats_return_numwants(char **reply, const char *reply_end) { - int i; - for (i = 0; i <= 200; ++i) - fmt_append(reply, reply_end, "%03d => %lld\n", i, numwants[i]); -} -#endif - -static void stats_return_everything(char **reply, const char *reply_end) { - unsigned long long total[OT_COUNTER_COUNT]; - - counters_total(total); - - torrent_stats stats = {0, 0, 0}; - int i; - - iterate_all_torrents(torrent_statter, (uintptr_t)&stats); - - fmt_append(reply, reply_end, "\n"); - fmt_append(reply, reply_end, "\n"); - fmt_append(reply, reply_end, " %llu\n", (unsigned long long)(time(NULL) - ot_start_time)); - fmt_append(reply, reply_end, " \n"); - fmt_append(reply, reply_end, " %zd\n", mutex_get_torrent_count()); - fmt_append(reply, reply_end, " %llu\n", stats.torrent_count); - fmt_append(reply, reply_end, " \n"); - fmt_append(reply, reply_end, " \n %llu\n \n", stats.peer_count); - fmt_append(reply, reply_end, " \n %llu\n \n", stats.seed_count); - fmt_append(reply, reply_end, " \n %llu\n \n", total[OT_COUNTER_COMPLETED]); - fmt_append(reply, reply_end, " \n"); - fmt_append(reply, reply_end, " \n %llu\n %llu\n %llu\n \n", - total[OT_COUNTER_TCP_CONNECTIONS], total[OT_COUNTER_TCP_ANNOUNCES], total[OT_COUNTER_TCP_SCRAPES]); - fmt_append(reply, reply_end, " \n %llu\n %llu\n %llu\n %llu\n %llu\n \n", - total[OT_COUNTER_UDP_CONNECTIONS], total[OT_COUNTER_UDP_CONNECTS], total[OT_COUNTER_UDP_ANNOUNCES], total[OT_COUNTER_UDP_SCRAPES], - total[OT_COUNTER_UDP_CONNID_MISSMATCH]); - fmt_append(reply, reply_end, " \n %llu\n \n", total[OT_COUNTER_SYNCS]); - fmt_append(reply, reply_end, " \n"); - fmt_append(reply, reply_end, " \n"); - fmt_append(reply, reply_end, " \n"); - for (i = 0; i < OT_PEER_TIMEOUT; ++i) - fmt_append(reply, reply_end, " %llu\n", i, total[OT_COUNTER_RENEWED + i]); - fmt_append(reply, reply_end, " \n"); - fmt_append(reply, reply_end, " \n"); - for (i = 0; i < CODE_HTTPERROR_COUNT; ++i) - fmt_append(reply, reply_end, " %llu\n", ot_failed_request_names[i], total[OT_COUNTER_FAILED + i]); - fmt_append(reply, reply_end, " \n"); - fmt_append(reply, reply_end, " \n %llu\n \n", total[OT_COUNTER_STALLS]); - fmt_append(reply, reply_end, " \n"); - fmt_append(reply, reply_end, ""); -} - -/* The names as the Makefile spells them, so that what a binary reports maps - straight back onto the FEATURES lines it was built from */ -static const char *const build_features[] = { -#ifdef WANT_V4_ONLY - "WANT_V4_ONLY", -#endif -#ifdef WANT_ACCESSLIST_BLACK - "WANT_ACCESSLIST_BLACK", -#endif -#ifdef WANT_ACCESSLIST_WHITE - "WANT_ACCESSLIST_WHITE", -#endif -#ifdef WANT_DYNAMIC_ACCESSLIST - "WANT_DYNAMIC_ACCESSLIST", -#endif -#ifdef WANT_SYNC_LIVE - "WANT_SYNC_LIVE", -#endif -#ifdef WANT_IP_FROM_QUERY_STRING - "WANT_IP_FROM_QUERY_STRING", -#endif -#ifdef WANT_IP_FROM_PROXY - "WANT_IP_FROM_PROXY", -#endif -#ifdef WANT_COMPRESSION_GZIP - "WANT_COMPRESSION_GZIP", -#endif -#ifdef WANT_COMPRESSION_GZIP_ALWAYS - "WANT_COMPRESSION_GZIP_ALWAYS", -#endif -#ifdef WANT_COMPRESSION_ZSTD - "WANT_COMPRESSION_ZSTD", -#endif -#ifdef WANT_COMPRESSION_ZSTD_ALWAYS - "WANT_COMPRESSION_ZSTD_ALWAYS", -#endif -#ifdef WANT_FULLSCRAPE - "WANT_FULLSCRAPE", -#endif -#ifdef WANT_MODEST_FULLSCRAPES - "WANT_MODEST_FULLSCRAPES", -#endif -#ifdef WANT_KEEPALIVE - "WANT_KEEPALIVE", -#endif -#ifdef WANT_RESTRICT_STATS - "WANT_RESTRICT_STATS", -#endif -#ifdef WANT_LOG_NUMWANT - "WANT_LOG_NUMWANT", -#endif -#ifdef WANT_SPOT_WOODPECKER - "WANT_SPOT_WOODPECKER", -#endif -#ifdef WANT_SYSLOGS - "WANT_SYSLOGS", -#endif -#ifdef WANT_DEV_RANDOM - "WANT_DEV_RANDOM", -#endif -#ifdef WANT_INSECURE_RANDOM_FALLBACK - "WANT_INSECURE_RANDOM_FALLBACK", -#endif -#ifdef _DEBUG - "_DEBUG", -#endif -#ifdef _DEBUG_HTTPERROR - "_DEBUG_HTTPERROR", -#endif -#ifdef _DEBUG_RANDOMTORRENTS - "_DEBUG_RANDOMTORRENTS", -#endif - NULL}; - -void stats_return_tracker_features(char **reply, const char *reply_end) { - size_t i; - - fmt_append(reply, reply_end, "events: %s\n", event_mechanism()); - fmt_append(reply, reply_end, "features:"); - for (i = 0; build_features[i]; ++i) - fmt_append(reply, reply_end, " %s", build_features[i]); - if (!build_features[0]) - fmt_append(reply, reply_end, " none"); - fmt_append(reply, reply_end, "\n"); -} - -void stats_return_tracker_version(char **reply, const char *reply_end) { - fmt_append(reply, reply_end, "opentracker %s", OT_VERSION); - if (*GIT_VERSION && strcmp(GIT_VERSION, OT_TAG)) - fmt_append(reply, reply_end, " (%s)", GIT_VERSION); - fmt_append(reply, reply_end, "\n"); - if (*OT_TAG) - fmt_append(reply, reply_end, "https://erdgeist.org/gitweb/opentracker/tag/?h=%s\n", OT_TAG); -} - -void return_stats_for_tracker(char **reply, const char *reply_end, int mode, int format) { - (void)format; - switch (mode & TASK_TASK_MASK) { - case TASK_STATS_CONNS: - stats_connections_mrtg(reply, reply_end); - break; - case TASK_STATS_SCRAPE: - stats_scrape_mrtg(reply, reply_end); - break; - case TASK_STATS_UDP: - stats_udpconnections_mrtg(reply, reply_end); - break; - case TASK_STATS_TCP: - stats_tcpconnections_mrtg(reply, reply_end); - break; - case TASK_STATS_FULLSCRAPE: - stats_fullscrapes_mrtg(reply, reply_end); - break; - case TASK_STATS_COMPLETED: - stats_return_completed_mrtg(reply, reply_end); - break; - case TASK_STATS_HTTPERRORS: - stats_httperrors_txt(reply, reply_end); - break; - case TASK_STATS_RENEW: - stats_return_renew_bucket(reply, reply_end); - break; - case TASK_STATS_SYNCS: - stats_return_sync_mrtg(reply, reply_end); - break; -#ifdef WANT_LOG_NUMWANT - case TASK_STATS_NUMWANTS: - stats_return_numwants(reply, reply_end); - break; -#endif - default: - break; - } -} - -static void stats_make(size_t *iovec_entries, struct iovec **iovector, ot_tasktype mode) { - char *r; - const char *r_end; - - *iovec_entries = 0; - *iovector = NULL; - if (!(r = iovec_new_chunk(iovec_entries, iovector, OT_STATS_TMPSIZE))) - return; - r_end = r + OT_STATS_TMPSIZE; - - switch (mode & TASK_TASK_MASK) { - case TASK_STATS_TORRENTS: - stats_torrents_mrtg(&r, r_end); - break; - case TASK_STATS_PEERS: - stats_peers_mrtg(&r, r_end); - break; - case TASK_STATS_SLASH24S: - stats_slash24s_txt(&r, r_end, 128); - break; - case TASK_STATS_TOP10: - stats_top_txt(&r, r_end, 10); - break; - case TASK_STATS_TOP100: - r = iovec_next_chunk(iovec_entries, iovector, r, 4 * OT_STATS_TMPSIZE); - if (!r) - return; - r_end = r + 4 * OT_STATS_TMPSIZE; - stats_top_txt(&r, r_end, 100); - break; - case TASK_STATS_EVERYTHING: - r = iovec_next_chunk(iovec_entries, iovector, r, OT_STATS_TMPSIZE + 64 * OT_PEER_TIMEOUT); - if (!r) - return; - r_end = r + OT_STATS_TMPSIZE + 64 * OT_PEER_TIMEOUT; - stats_return_everything(&r, r_end); - break; -#ifdef WANT_SPOT_WOODPECKER - case TASK_STATS_WOODPECKERS: - stats_return_woodpeckers(&r, r_end, 128); - break; -#endif - default: - iovec_free(iovec_entries, iovector); - return; - } - iovec_trim_last(iovec_entries, iovector, r); -} - -void stats_issue_event(struct ot_workstruct *ws, ot_status_event event, PROTO_FLAG proto, uintptr_t event_data) { - if (!ws) - return; - - switch (event) { - case EVENT_ACCEPT: - if (proto == FLAG_TCP) - COUNT(OT_COUNTER_TCP_CONNECTIONS, 1); - else - COUNT(OT_COUNTER_UDP_CONNECTIONS, 1); - break; - case EVENT_ANNOUNCE: - if (proto == FLAG_TCP) - COUNT(OT_COUNTER_TCP_ANNOUNCES, 1); - else - COUNT(OT_COUNTER_UDP_ANNOUNCES, 1); - break; - case EVENT_CONNECT: - if (proto == FLAG_TCP) - COUNT(OT_COUNTER_TCP_CONNECTS, 1); - else - COUNT(OT_COUNTER_UDP_CONNECTS, 1); - break; - case EVENT_COMPLETED: -#ifdef WANT_SYSLOGS - { - char timestring[64]; - char hash_hex[42], peerid_hex[42], ip_readable[64]; - struct tm time_now; - time_t ttt; - - time(&ttt); - localtime_r(&ttt, &time_now); - strftime(timestring, sizeof(timestring), "%FT%T%z", &time_now); - - hex_from_hash(hash_hex, *ws->hash); - if (ws->peer_id) - hex_from_hash(peerid_hex, (uint8_t *)ws->peer_id); - else { - *peerid_hex = 0; - } - - ip_readable[ip_format(ip_readable, (char *)&ws->peer)] = 0; - syslog(LOG_INFO, "time=%s event=completed info_hash=%s peer_id=%s ip=%s", timestring, hash_hex, peerid_hex, ip_readable); - } -#endif - COUNT(OT_COUNTER_COMPLETED, 1); - break; - case EVENT_SCRAPE: - if (proto == FLAG_TCP) - COUNT(OT_COUNTER_TCP_SCRAPES, 1); - else - COUNT(OT_COUNTER_UDP_SCRAPES, 1); - break; - case EVENT_FULLSCRAPE: - COUNT(OT_COUNTER_FULLSCRAPES, 1); - COUNT(OT_COUNTER_FULLSCRAPE_BYTES, event_data); - break; - case EVENT_FULLSCRAPE_REQUEST: { - ot_ip6 *ip = (ot_ip6 *)event_data; /* ugly hack to transfer ip to stats */ - char _debug[512]; - int off = snprintf(_debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(time_now_seconds() - ot_start_time) / 60); - off += ip_format(_debug + off, *ip); - off += snprintf(_debug + off, sizeof(_debug) - off, " - FULL SCRAPE\n"); - (void)write(2, _debug, off); - COUNT(OT_COUNTER_FULLSCRAPE_REQUESTS, 1); - } break; - case EVENT_FULLSCRAPE_REQUEST_GZIP: { - ot_ip6 *ip = (ot_ip6 *)event_data; /* ugly hack to transfer ip to stats */ - char _debug[512]; - int off = snprintf(_debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(time_now_seconds() - ot_start_time) / 60); - off += ip_format(_debug + off, *ip); - off += snprintf(_debug + off, sizeof(_debug) - off, " - FULL SCRAPE\n"); - (void)write(2, _debug, off); - COUNT(OT_COUNTER_FULLSCRAPE_REQUESTS, 1); - } break; - case EVENT_FAILED: - COUNT(OT_COUNTER_FAILED + event_data, 1); - break; - case EVENT_RENEW: - COUNT(OT_COUNTER_RENEWED + event_data, 1); - break; - case EVENT_SYNC: - COUNT(OT_COUNTER_SYNCS, event_data); - break; - case EVENT_BUCKET_LOCKED: - COUNT(OT_COUNTER_STALLS, 1); - break; -#ifdef WANT_SPOT_WOODPECKER - case EVENT_WOODPECKER: - pthread_mutex_lock(&g_woodpeckers_mutex); - stat_increase_network_count(&stats_woodpeckers_tree, 0, event_data); - pthread_mutex_unlock(&g_woodpeckers_mutex); - break; -#endif - case EVENT_CONNID_MISSMATCH: - COUNT(OT_COUNTER_UDP_CONNID_MISSMATCH, 1); - default: - break; - } -} - -void stats_cleanup(void) { -#ifdef WANT_SPOT_WOODPECKER - pthread_mutex_lock(&g_woodpeckers_mutex); - stats_shift_down_network_count(&stats_woodpeckers_tree, 0, 1); - pthread_mutex_unlock(&g_woodpeckers_mutex); -#endif -} - -static void *stats_worker(void *args) { - size_t iovec_entries; - struct iovec *iovector; - - (void)args; - - while (g_opentracker_running) { - ot_tasktype tasktype = TASK_STATS; - ot_taskid taskid = workqueue_poptask(&tasktype); - if (!taskid) - break; - stats_make(&iovec_entries, &iovector, tasktype); - if (workqueue_pushresult(taskid, iovec_entries, iovector)) - iovec_free(&iovec_entries, &iovector); - } - return NULL; -} - -void stats_deliver(int64_t sock, int tasktype) { workqueue_pushtask(sock, tasktype); } - -static pthread_t thread_id; -void stats_init(void) { - ot_start_time = time_now_seconds(); - pthread_create(&thread_id, NULL, stats_worker, NULL); -} - -void stats_deinit(void) { - workqueue_wakeall(); - pthread_join(thread_id, NULL); -} diff --git a/ot_stats.h b/ot_stats.h deleted file mode 100644 index 45a547e..0000000 --- a/ot_stats.h +++ /dev/null @@ -1,40 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_STATS_H__ -#define OT_STATS_H__ - -#include "trackerlogic.h" - -typedef enum { - EVENT_ACCEPT, - EVENT_READ, - EVENT_CONNECT, /* UDP only */ - EVENT_ANNOUNCE, - EVENT_COMPLETED, - EVENT_RENEW, - EVENT_SYNC, - EVENT_SCRAPE, - EVENT_FULLSCRAPE_REQUEST, - EVENT_FULLSCRAPE_REQUEST_GZIP, - EVENT_FULLSCRAPE_REQUEST_ZSTD, - EVENT_FULLSCRAPE, /* TCP only */ - EVENT_FAILED, - EVENT_BUCKET_LOCKED, - EVENT_WOODPECKER, - EVENT_CONNID_MISSMATCH -} ot_status_event; - -void stats_counters_register(struct ot_workstruct *ws); -void stats_counters_retire(struct ot_workstruct *ws); - -void stats_issue_event(struct ot_workstruct *ws, ot_status_event event, PROTO_FLAG proto, uintptr_t event_data); -void stats_deliver(int64_t sock, int tasktype); -void stats_cleanup(void); -void return_stats_for_tracker(char **reply, const char *reply_end, int mode, int format); -void stats_return_tracker_version(char **reply, const char *reply_end); -void stats_return_tracker_features(char **reply, const char *reply_end); -void stats_init(void); -void stats_deinit(void); - -#endif diff --git a/ot_tcp.c b/ot_tcp.c deleted file mode 100644 index 1351b92..0000000 --- a/ot_tcp.c +++ /dev/null @@ -1,180 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include -#include -#include -#include -#include -#include - -/* Opentracker */ -#include "ot_batch.h" -#include "ot_buf.h" -#include "ot_event.h" -#include "ot_fdstate.h" -#include "ot_http.h" -#include "ot_mutex.h" -#include "ot_socket.h" -#include "ot_worker.h" -#include "ot_stats.h" -#include "ot_tcp.h" -#include "trackerlogic.h" - -static ssize_t header_complete(char *request, ssize_t byte_count) { - ssize_t i = 0, state = 0; - - for (i = 1; i < byte_count; i += 2) - if (request[i] <= 13) { - i--; - for (state = 0; i < byte_count; ++i) { - char c = request[i]; - if (c == '\r' || c == '\n') - state = (state >> 2) | ((c << 6) & 0xc0); - else - break; - if (state >= 0xa0 || state == 0x99) - return i + 1; - } - } - return 0; -} - -void tcp_read(const int64_t sock, struct ot_workstruct *ws) { - struct http_data *cookie = fdstate_cookie(sock); - ssize_t byte_count = read((int)sock, ws->inbuf, G_INBUF_SIZE); - - if (byte_count == 0 || (byte_count == -1 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR)) { - http_close(sock); - return; - } - - if (byte_count == -1) - return; - - ws->request = NULL; - ws->request_size = 0; - ws->header_size = 0; - ws->keep_alive = 0; - - /* If we get the whole request in one packet, handle it without copying */ - if (!cookie->request.data) { - if ((ws->header_size = header_complete(ws->inbuf, byte_count))) { - ws->request = ws->inbuf; - ws->request_size = byte_count; - http_handle_request(sock, ws); - } else if (buf_cat(&cookie->request, ws->inbuf, (size_t)byte_count)) - http_close(sock); - return; - } - - if (buf_cat(&cookie->request, ws->inbuf, (size_t)byte_count) || cookie->request.size > G_INBUF_SIZE) { - http_issue_error(sock, ws, CODE_HTTPERROR_500); - return; - } - - while ((ws->header_size = header_complete(cookie->request.data, (ssize_t)cookie->request.size))) { - ws->request = cookie->request.data; - ws->request_size = (ssize_t)cookie->request.size; - http_handle_request(sock, ws); -#ifdef WANT_KEEPALIVE - if (!ws->keep_alive) -#endif - return; - } -} - -void tcp_write(const int64_t sock) { - struct http_data *cookie = fdstate_cookie(sock); - int chunked = 0; - int keep_alive = 0; - - if (cookie) { - chunked = (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) != 0; - keep_alive = (cookie->flag & STRUCT_HTTP_FLAG_KEEPALIVE) != 0; - - if (cookie->batch) { - if (batch_send(sock, cookie->batch) == BATCH_FAILED) { - http_close(sock); - return; - } - if (batch_pending(cookie->batch)) - return; - } - } - - /* In a chunked transfer, wait for the next piece once everything queued has gone out */ - if (chunked) - http_interest_set(sock, cookie, 0); - else if (keep_alive) { - /* The reply is out and the client asked to keep the connection, so go - back to waiting for the next request */ - http_interest_set(sock, cookie, EVENT_IN); - fdstate_set_expiry(sock, time_future_seconds(OT_CLIENT_TIMEOUT)); - } else - http_close(sock); -} - -/* The fast path, assuming that the complete request has usually arrived by the time a - connection is accepted, so it is answered here outside the event loop. The more load - a tracker has, the more likely it is that by the time it accepts, the roundtrip to - the peer has completed. - If the request could not be completely handled, the stack cookie is promoted to heap - and the connection handed over to the event loop to finish */ -void tcp_accept(const int64_t serversocket, struct ot_workstruct *ws) { - int64_t sock; - ot_ip6 ip; - uint16_t port; - - while ((sock = socket_accept(serversocket, ip, &port)) != -1) { - struct http_data cookie; - uint32_t generation; - - memset(&cookie, 0, sizeof(cookie)); - memcpy(cookie.ip, ip, sizeof(ot_ip6)); - cookie.interest = EVENT_IN; - cookie.flag = STRUCT_HTTP_FLAG_ONSTACK; - - generation = fdstate_open(sock, &cookie, time_future_seconds(OT_CLIENT_TIMEOUT)); - if (!fdstate_claim((int)sock, generation)) { - fdstate_close(sock); - event_close((int)sock); - continue; - } - - stats_issue_event(ws, EVENT_ACCEPT, FLAG_TCP, (uintptr_t)ip); - - tcp_read(sock, ws); - - /* Gone already if it was answered and closed inline */ - if (fdstate_generation((int)sock) != generation) - continue; - - /* Handling it may already have given it a heap home and handed it to - another thread, and then it is no longer this frame's to touch */ - if (fdstate_cookie((int)sock) != &cookie) - fdstate_release((int)sock); - else if (!http_adopt(sock)) - http_close(sock); - else - fdstate_release((int)sock); - } -} - -/* A worker blocks in accept and nowhere else: the connection it is handed is - non blocking, so a client that stops talking is given to the event loop - rather than keeping this thread. */ -static void tcp_run(struct ot_worker *me) { - while (g_opentracker_running) - tcp_accept(me->sock, &me->ws); -} - -void tcp_worker_install(int64_t sock) { - socket_set_blocking((int)sock, 1); - socket_defer_accept((int)sock); - - if (worker_start(tcp_run, sock) == -1) - fprintf(stderr, "Warning: could not install a tcp worker, this socket stays with the workers already running.\n"); -} diff --git a/ot_tcp.h b/ot_tcp.h deleted file mode 100644 index afed142..0000000 --- a/ot_tcp.h +++ /dev/null @@ -1,24 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_TCP_H__ -#define OT_TCP_H__ - -#include - -#include "trackerlogic.h" - -/* Take whatever connections are waiting on a listening socket and answer - them, each on this thread's stack, handing a connection that cannot be - finished to the event loop */ -void tcp_accept(const int64_t serversocket, struct ot_workstruct *ws); - -/* Handler connections in the event loop has become readable or writable */ -void tcp_read(const int64_t sock, struct ot_workstruct *ws); -void tcp_write(const int64_t sock); - -/* One thread accepting on one socket. Called once per worker. */ -void tcp_worker_install(int64_t sock); - - -#endif diff --git a/ot_udp.c b/ot_udp.c deleted file mode 100644 index 644b71d..0000000 --- a/ot_udp.c +++ /dev/null @@ -1,226 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include -#include -#include -#include -#include - -/* Opentracker */ -#include "ot_ip.h" -#include "ot_random.h" -#include "ot_rijndael.h" -#include "ot_socket.h" -#include "ot_worker.h" -#include "ot_stats.h" -#include "ot_udp.h" -#include "trackerlogic.h" - -static uint32_t g_rijndael_round_key[44] = {0}; -static volatile uint32_t g_key_of_the_hour[2] = {0}; -static volatile ot_time g_hour_of_the_key; - - -static void udp_generate_rijndael_round_key(void) { - uint32_t key[16]; - random_bytes(key, sizeof(key)); - rijndaelKeySetupEnc128(g_rijndael_round_key, (uint8_t *)key); - - g_key_of_the_hour[0] = random_uint32(); - g_hour_of_the_key = time_now_minutes(); -} - -/* Generate current and previous connection id for ip */ -static void udp_make_connectionid(uint32_t connid[2], const ot_ip6 remoteip, int age) { - uint32_t plain[4], crypt[4]; - int i; - uint32_t current_key_of_the_hour; - - if (time_now_minutes() - g_hour_of_the_key >= 60) { - uint32_t old_key_of_the_hour = g_key_of_the_hour[0]; - uint32_t new_key_of_the_hour = random_uint32(); - /* If in the meantime another thread has performed - key rotation, do not overwrite their results */ - if (time_now_minutes() - g_hour_of_the_key >= 60) { - /* Upgrade en bloc */ - g_hour_of_the_key = time_now_minutes(); - g_key_of_the_hour[0] = new_key_of_the_hour; - g_key_of_the_hour[1] = old_key_of_the_hour; - } - } - - memcpy(plain, remoteip, sizeof(plain)); - current_key_of_the_hour = g_key_of_the_hour[age]; - for (i = 0; i < 4; ++i) - plain[i] ^= current_key_of_the_hour; - rijndaelEncrypt128(g_rijndael_round_key, (uint8_t *)plain, (uint8_t *)crypt); - connid[0] = crypt[0] ^ crypt[1]; - connid[1] = crypt[2] ^ crypt[3]; -} - -/* UDP implementation according to http://xbtt.sourceforge.net/udp_tracker_protocol.html */ -int handle_udp6(int64_t serversocket, struct ot_workstruct *ws) { - ot_ip6 remoteip; - uint32_t *inpacket = (uint32_t *)ws->inbuf; - uint32_t *outpacket = (uint32_t *)ws->outbuf; - uint32_t left, event; - uint32_t connid[2]; - uint32_t action; - uint16_t port, remoteport; - size_t byte_count, scrape_count; - - ssize_t result = socket_recv(serversocket, ws->inbuf, G_INBUF_SIZE, remoteip, &remoteport); - if (result <= 0) - return 0; - byte_count = (size_t)result; - - stats_issue_event(ws, EVENT_ACCEPT, FLAG_UDP, (uintptr_t)remoteip); - stats_issue_event(ws, EVENT_READ, FLAG_UDP, byte_count); - - /* Minimum udp tracker packet size, also catches error */ - if (byte_count < 16) - return 1; - - /* Get action to take. Ignore error messages and broken packets */ - action = ntohl(inpacket[2]); - if (action > 2) - return 1; - - /* Generate the connection id we give out and expect to and from - the requesting ip address, this prevents udp spoofing */ - udp_make_connectionid(connid, remoteip, 0); - - /* Initialise hash pointer */ - ws->hash = NULL; - ws->peer_id = NULL; - - /* If action is not 0 (connect), then we expect the derived - connection id in first 64 bit */ - if ((action > 0) && (inpacket[0] != connid[0] || inpacket[1] != connid[1])) { - /* If connection id does not match, try the one that was - valid in the previous hour. Only if this also does not - match, return an error packet */ - udp_make_connectionid(connid, remoteip, 1); - if (inpacket[0] != connid[0] || inpacket[1] != connid[1]) { - const char err_idmiss[] = "Connection ID missmatch."; - size_t s = sizeof(err_idmiss); - outpacket[0] = htonl(3); - outpacket[1] = inpacket[3]; - memcpy(&outpacket[2], &err_idmiss[0], s); - socket_send(serversocket, ws->outbuf, 8 + s, remoteip, remoteport); - stats_issue_event(ws, EVENT_CONNID_MISSMATCH, FLAG_UDP, 8 + s); - return 1; - } - } - - switch (action) { - case 0: /* This is a connect action */ - /* look for udp bittorrent magic id */ - if ((ntohl(inpacket[0]) != 0x00000417) || (ntohl(inpacket[1]) != 0x27101980)) - return 1; - - outpacket[0] = 0; - outpacket[1] = inpacket[3]; - outpacket[2] = connid[0]; - outpacket[3] = connid[1]; - - socket_send(serversocket, ws->outbuf, 16, remoteip, remoteport); - stats_issue_event(ws, EVENT_CONNECT, FLAG_UDP, 16); - break; - case 1: /* This is an announce action */ - /* Minimum udp announce packet size */ - if (byte_count < 98) - return 1; - - /* We do only want to know, if it is zero */ - left = inpacket[64 / 4] | inpacket[68 / 4]; - - event = ntohl(inpacket[80 / 4]); - port = *(uint16_t *)(((char *)inpacket) + 96); - ws->hash = (ot_hash *)(((char *)inpacket) + 16); - - OT_SETIP(ws->peer, remoteip); - OT_SETPORT(ws->peer, &port); - OT_PEERFLAG(ws->peer) = 0; - - switch (event) { - case 1: - OT_PEERFLAG(ws->peer) |= PEER_FLAG_COMPLETED; - break; - case 3: - OT_PEERFLAG(ws->peer) |= PEER_FLAG_STOPPED; - break; - default: - break; - } - - if (!left) - OT_PEERFLAG(ws->peer) |= PEER_FLAG_SEEDING; - - outpacket[0] = htonl(1); /* announce action */ - outpacket[1] = inpacket[12 / 4]; - - if (OT_PEERFLAG(ws->peer) & PEER_FLAG_STOPPED) { /* Peer is gone. */ - ws->reply = ws->outbuf; - ws->reply_size = remove_peer_from_torrent(FLAG_UDP, ws); - } else { - /* Limit amount of peers to OT_MAX_PEERS_UDP */ - uint32_t numwant = ntohl(inpacket[92 / 4]); - size_t max_peers = ip_is_v4mapped(remoteip) ? OT_MAX_PEERS_UDP4 : OT_MAX_PEERS_UDP6; - if (numwant > max_peers) - numwant = max_peers; - - ws->reply = ws->outbuf + 8; - ws->reply_size = add_peer_to_torrent_and_return_peers(FLAG_UDP, ws, numwant); - - if (!ws->reply_size) { - const char err_unauth[] = "Requested download is not authorized for use with this tracker."; - size_t s = sizeof(err_unauth); - outpacket[0] = htonl(3); - memcpy(&outpacket[2], &err_unauth[0], s); - socket_send(serversocket, ws->outbuf, 8 + s, remoteip, remoteport); - return 1; - } - ws->reply_size += 8; - } - - socket_send(serversocket, ws->outbuf, ws->reply_size, remoteip, remoteport); - stats_issue_event(ws, EVENT_ANNOUNCE, FLAG_UDP, ws->reply_size); - break; - - case 2: /* This is a scrape action */ - outpacket[0] = htonl(2); /* scrape action */ - outpacket[1] = inpacket[12 / 4]; - - for (scrape_count = 0; (scrape_count * 20 < byte_count - 16) && (scrape_count <= 74); scrape_count++) - return_udp_scrape_for_torrent(*(ot_hash *)(((char *)inpacket) + 16 + 20 * scrape_count), ((char *)outpacket) + 8 + 12 * scrape_count); - - socket_send(serversocket, ws->outbuf, 8 + 12 * scrape_count, remoteip, remoteport); - stats_issue_event(ws, EVENT_SCRAPE, FLAG_UDP, scrape_count); - break; - } - return 1; -} - -static void udp_run(struct ot_worker *me) { - while (g_opentracker_running) - handle_udp6(me->sock, &me->ws); -} - -void udp_init(void) { - if (!g_rijndael_round_key[0]) - udp_generate_rijndael_round_key(); -} - -void udp_worker_install(int64_t sock) { -#ifdef _DEBUG - fprintf(stderr, "Installing a worker on udp socket %ld\n", (unsigned long)sock); -#endif - socket_set_blocking((int)sock, 1); - - if (worker_start(udp_run, sock) == -1) - fprintf(stderr, "Warning: could not install a udp worker, this socket stays with the workers already running.\n"); -} diff --git a/ot_udp.h b/ot_udp.h deleted file mode 100644 index 2710744..0000000 --- a/ot_udp.h +++ /dev/null @@ -1,17 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_UDP_H__ -#define OT_UDP_H__ - -#include - -#include "trackerlogic.h" - -void udp_init(void); -/* One thread reading one socket. Called once per worker. */ -void udp_worker_install(int64_t sock); - -int handle_udp6(int64_t serversocket, struct ot_workstruct *ws); - -#endif diff --git a/ot_urlencoded.c b/ot_urlencoded.c deleted file mode 100644 index 4109bb5..0000000 --- a/ot_urlencoded.c +++ /dev/null @@ -1,168 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* Opentracker */ -#include "ot_urlencoded.h" - -/* System */ -#include - -/* Idea is to do a in place replacement or guarantee at least - strlen( string ) bytes in deststring - watch http://www.ietf.org/rfc/rfc2396.txt - unreserved = alphanum | mark - mark = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")" - we add '%' to the matrix to not stop at encoded chars. - After losing too many requests to being too strict, add the following characters to reserved matrix - relax = "+" | "," | "/" | ";" | "<" | ">" | ":" -*/ - -/* This matrix holds for each ascii character the information, - whether it is a non-terminating character for one of the three - scan states we are in, that is 'path', 'param' and 'value' from - /path?param=value¶m=value, it is encoded in bit 0, 1 and 2 - respectively - - The top bit of lower nibble indicates, whether this character is - a hard terminator, ie. \0, \n or \s, where the whole scanning - process should terminate - */ -static const unsigned char is_unreserved[256] = { - 8,0,0,0,0,0,0,0,0,0,8,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 8,7,8,8,8,7,0,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,4,7,6, - 4,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,7, - 8,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,7,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -}; - -/* Do a fast nibble to hex representation conversion */ -static unsigned char fromhex(unsigned char x) { - x -= '0'; - if (x <= 9) - return x; - x &= ~0x20; - x -= 'A' - '0'; - if (x < 6) - return x + 10; - return 0xff; -} - -/* Skip the value of a param=value pair */ -void urlencoded_skip_value(char **string) { - const unsigned char *s = *(const unsigned char **)string; - unsigned char f; - - /* Since we are asked to skip the 'value', we assume to stop at - terminators for a 'value' string position */ - while ((f = is_unreserved[*s++]) & SCAN_SEARCHPATH_VALUE) - ; - - /* If we stopped at a hard terminator like \0 or \n, make the - next urlencoded_parse encounter it again */ - if (f & SCAN_SEARCHPATH_TERMINATOR) - --s; - - *string = (char *)s; -} - -int urlencoded_match_keyword(const ot_keywords *keywords, char **string, SCAN_SEARCHPATH_FLAG flags) { - char *deststring = *string; - ssize_t match_length = urlencoded_parse(string, deststring, flags); - - if (match_length < 0) - return match_length; - if (match_length == 0) - return -3; - - while (keywords->key) { - if (!strncmp(keywords->key, deststring, match_length) && !keywords->key[match_length]) - return keywords->value; - keywords++; - } - - return -3; -} - -ssize_t urlencoded_parse(char **string, char *deststring, SCAN_SEARCHPATH_FLAG flags) { - const unsigned char *s = *(const unsigned char **)string; - unsigned char *d = (unsigned char *)deststring; - unsigned char b, c; - - /* This is the main decoding loop. - 'flag' determines, which characters are non-terminating in current context - (ie. stop at '=' and '&' if scanning for a 'param'; stop at '?' if scanning for the path ) - */ - while (is_unreserved[c = *s++] & flags) { - - /* When encountering an url escaped character, try to decode */ - if (c == '%') { - if ((b = fromhex(*s++)) == 0xff) - return -1; - if ((c = fromhex(*s++)) == 0xff) - return -1; - c |= (b << 4); - } - - /* Write (possibly decoded) character to output */ - *d++ = c; - } - - switch (c) { - case 0: - case '\r': - case '\n': - case ' ': - /* If we started scanning on a hard terminator, indicate we've finished */ - if (d == (unsigned char *)deststring) - return -2; - - /* Else make the next call to urlencoded_parse encounter it again */ - --s; - break; - case '?': - if (flags != SCAN_PATH) - return -1; - break; - case '=': - if (flags != SCAN_SEARCHPATH_PARAM) - return -1; - break; - case '&': - if (flags == SCAN_PATH) - return -1; - if (flags == SCAN_SEARCHPATH_PARAM) - --s; - break; - default: - return -1; - } - - *string = (char *)s; - return d - (unsigned char *)deststring; -} - - -size_t urlencoded_format(char *dest, const uint8_t *src, size_t len) { - static const char safe[] = "$/.=~-_"; - static const char hexdigit[] = "0123456789abcdef"; - size_t written = 0, i; - - for (i = 0; i < len; ++i) { - uint8_t c = src[i]; - - /* strchr would find the terminator, so a zero byte never counts as safe */ - if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || (c && strchr(safe, c))) { - dest[written++] = (char)c; - } else { - dest[written] = '%'; - dest[written + 1] = hexdigit[c >> 4]; - dest[written + 2] = hexdigit[c & 15]; - written += 3; - } - } - - return written; -} diff --git a/ot_urlencoded.h b/ot_urlencoded.h deleted file mode 100644 index 9882468..0000000 --- a/ot_urlencoded.h +++ /dev/null @@ -1,53 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef SCAN_URLENCODED_QUERY_H__ -#define SCAN_URLENCODED_QUERY_H__ - -#include -#include -#include - -typedef struct { - const char *key; - int value; -} ot_keywords; - -typedef enum { - SCAN_PATH = 1, - SCAN_SEARCHPATH_PARAM = 2, - SCAN_SEARCHPATH_VALUE = 4, - SCAN_SEARCHPATH_TERMINATOR = 8 -} SCAN_SEARCHPATH_FLAG; - -/* string in: pointer to source - out: pointer to next scan position - deststring pointer to destination - flags determines, what to parse - returns number of valid converted characters in deststring - or -1 for parse error - or -2 for terminator found -*/ -ssize_t urlencoded_parse(char **string, char *deststring, SCAN_SEARCHPATH_FLAG flags); - -/* string in: pointer to source - out: pointer to next scan position - flags determines, what to parse - returns value for matched keyword - or -1 for parse error - or -2 for terminator found - or -3 for no keyword matched - */ -int urlencoded_match_keyword(const ot_keywords *keywords, char **string, SCAN_SEARCHPATH_FLAG flags); - -/* string in: pointer to value of a param=value pair to skip - out: pointer to next scan position on return -*/ -void urlencoded_skip_value(char **string); - - -/* The other direction: percent encode len bytes from src into dest and return - how many characters were written. dest needs room for three times len. */ -size_t urlencoded_format(char *dest, const uint8_t *src, size_t len); - -#endif diff --git a/ot_vector.c b/ot_vector.c deleted file mode 100644 index ab0566c..0000000 --- a/ot_vector.c +++ /dev/null @@ -1,287 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include -#include -#include - -/* Opentracker */ -#include "trackerlogic.h" - -static int vector_compare_peer6(const void *peer1, const void *peer2) { return memcmp(peer1, peer2, OT_PEER_COMPARE_SIZE6); } -static int vector_compare_peer4(const void *peer1, const void *peer2) { return memcmp(peer1, peer2, OT_PEER_COMPARE_SIZE4); } - -/* This function gives us a binary search that returns a pointer, even if - no exact match is found. In that case it sets exactmatch 0 and gives - calling functions the chance to insert data -*/ -void *vector_binary_search(const void *const key, const void *base, const size_t member_count, const size_t member_size, size_t compare_size, int *exactmatch) { - size_t interval = member_count; - - while (interval) { - uint8_t *lookat = ((uint8_t *)base) + member_size * (interval / 2); - int cmp = memcmp(lookat, key, compare_size); - if (cmp == 0) { - base = lookat; - break; - } - if (cmp < 0) { - base = lookat + member_size; - interval--; - } - interval /= 2; - } - - *exactmatch = interval; - return (void *)base; -} - -static uint8_t vector_hash_peer(ot_peer const *peer, size_t compare_size, size_t bucket_count) { - unsigned int hash = 5381; - uint8_t *p = (uint8_t *)peer; - while (compare_size--) - hash += (hash << 5) + *(p++); - return hash % bucket_count; -} - -/* This is the generic insert operation for our vector type. - It tries to locate the object at "key" with size "member_size" by comparing its first "compare_size" bytes with - those of objects in vector. Our special "vector_binary_search" function does that and either returns the match or a - pointer to where the object is to be inserted. vector_find_or_insert makes space for the object and copies it, - if it wasn't found in vector. Caller needs to check the passed "exactmatch" variable to see, whether an insert - took place. If resizing the vector failed, NULL is returned, else the pointer to the object in vector. -*/ -void *vector_find_or_insert(ot_vector *vector, void *key, size_t member_size, size_t compare_size, int *exactmatch) { - uint8_t *match = vector_binary_search(key, vector->data, vector->size, member_size, compare_size, exactmatch); - size_t match_offset, bytes_after; - - if (*exactmatch) - return match; - - /* Hold the insertion point as an offset rather than a pointer: it survives - the realloc below, and an empty vector has no data pointer to offset from - in the first place */ - match_offset = vector->size ? (size_t)(match - (uint8_t *)vector->data) : 0; - bytes_after = member_size * vector->size - match_offset; - - if (vector->size + 1 > vector->space) { - size_t new_space = vector->space ? OT_VECTOR_GROW_RATIO * vector->space : OT_VECTOR_MIN_MEMBERS; - uint8_t *new_data = realloc(vector->data, new_space * member_size); - - if (!new_data) - return NULL; - - vector->data = new_data; - vector->space = new_space; - } - - match = (uint8_t *)vector->data + match_offset; - memmove(match + member_size, match, bytes_after); - - vector->size++; - return match; -} - -ot_peer *vector_find_or_insert_peer(ot_vector *vector, ot_peer const *peer, size_t peer_size, int *exactmatch) { - ot_peer *match; - const size_t compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size); - size_t match_offset, bytes_after; - - /* If space is zero but size is set, we're dealing with a list of vector->size buckets */ - if (vector->space < vector->size) - vector = ((ot_vector *)vector->data) + vector_hash_peer(peer, compare_size, vector->size); - match = vector_binary_search(peer, vector->data, vector->size, peer_size, compare_size, exactmatch); - - if (*exactmatch) - return match; - - /* Hold the insertion point as an offset rather than a pointer: it survives - the realloc below, and an empty vector has no data pointer to offset from - in the first place. bytes_after is what has to move up by peer_size to - make room for the new peer */ - match_offset = vector->size ? (size_t)(match - (ot_peer *)vector->data) : 0; - bytes_after = vector->size * peer_size - match_offset; - - if (vector->size + 1 > vector->space) { - size_t new_space = vector->space ? OT_VECTOR_GROW_RATIO * vector->space : OT_VECTOR_MIN_MEMBERS; - ot_peer *new_data = realloc(vector->data, new_space * peer_size); - - if (!new_data) - return NULL; - - vector->data = new_data; - vector->space = new_space; - } - - match = (ot_peer *)vector->data + match_offset; - memmove(match + peer_size, match, bytes_after); - - vector->size++; - return match; -} - -/* This is the non-generic delete from vector-operation specialized for peers in pools. - It returns 0 if no peer was found (and thus not removed) - 1 if a non-seeding peer was removed - 2 if a seeding peer was removed -*/ -int vector_remove_peer(ot_vector *vector, ot_peer const *peer, size_t peer_size) { - int exactmatch, was_seeder; - ot_peer *match, *end; - size_t compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size); - - if (!vector->size) - return 0; - - /* If space is zero but size is set, we're dealing with a list of vector->size buckets */ - if (vector->space < vector->size) - vector = ((ot_vector *)vector->data) + vector_hash_peer(peer, compare_size, vector->size); - - end = ((ot_peer *)vector->data) + peer_size * vector->size; - match = (ot_peer *)vector_binary_search(peer, vector->data, vector->size, peer_size, compare_size, &exactmatch); - if (!exactmatch) - return 0; - - was_seeder = (OT_PEERFLAG_D(match, peer_size) & PEER_FLAG_SEEDING) ? 2 : 1; - memmove(match, match + peer_size, end - match - peer_size); - - vector->size--; - vector_fixup_peers(vector, peer_size); - return was_seeder; -} - -void vector_remove_torrent(ot_vector *vector, ot_torrent *match) { - ot_torrent *end; - - if (!vector->size) - return; - - end = ((ot_torrent *)vector->data) + vector->size; - - free_peerlist(match->peer_list6); - free_peerlist(match->peer_list4); - - memmove(match, match + 1, sizeof(ot_torrent) * (end - match - 1)); - if ((--vector->size * OT_VECTOR_SHRINK_THRESH < vector->space) && (vector->space >= OT_VECTOR_SHRINK_RATIO * OT_VECTOR_MIN_MEMBERS)) { - vector->space /= OT_VECTOR_SHRINK_RATIO; - vector->data = realloc(vector->data, vector->space * sizeof(ot_torrent)); - } -} - -void vector_clean_list(ot_vector *vector, size_t num_buckets) { - while (num_buckets--) - free(vector[num_buckets].data); - free(vector); - return; -} - -void vector_redistribute_buckets(ot_peerlist *peer_list, size_t peer_size) { - size_t bucket, bucket_size_new, num_buckets_new, num_buckets_old = 1; - ot_vector *bucket_list_new, *bucket_list_old = &peer_list->peers; - int (*sort_func)(const void *, const void *) = peer_size == OT_PEER_SIZE6 ? &vector_compare_peer6 : &vector_compare_peer4; - - if (OT_PEERLIST_HASBUCKETS(peer_list)) { - num_buckets_old = peer_list->peers.size; - bucket_list_old = peer_list->peers.data; - } - - if (peer_list->peer_count < 255) - num_buckets_new = 1; - else if (peer_list->peer_count > 8192) - num_buckets_new = 64; - else if (peer_list->peer_count >= 512 && peer_list->peer_count < 4096) - num_buckets_new = 16; - else if (peer_list->peer_count < 512 && num_buckets_old <= 16) - num_buckets_new = num_buckets_old; - else if (peer_list->peer_count < 512) - num_buckets_new = 1; - else if (peer_list->peer_count < 8192 && num_buckets_old > 1) - num_buckets_new = num_buckets_old; - else - num_buckets_new = 16; - - if (num_buckets_new == num_buckets_old) - return; - - /* Assume near perfect distribution */ - bucket_list_new = calloc(1, num_buckets_new * sizeof(ot_vector)); - if (!bucket_list_new) - return; - - bucket_size_new = OT_VECTOR_MIN_MEMBERS; - while (bucket_size_new * num_buckets_new < peer_list->peer_count) - bucket_size_new *= OT_VECTOR_GROW_RATIO; - - /* preallocate vectors to hold all peers */ - for (bucket = 0; bucket < num_buckets_new; ++bucket) { - bucket_list_new[bucket].space = bucket_size_new; - bucket_list_new[bucket].data = malloc(bucket_size_new * peer_size); - if (!bucket_list_new[bucket].data) { - vector_clean_list(bucket_list_new, num_buckets_new); - return; - } - } - - /* Now sort them into the correct bucket */ - for (bucket = 0; bucket < num_buckets_old; ++bucket) { - ot_peer *peers_old = bucket_list_old[bucket].data; - size_t peer_count_old = bucket_list_old[bucket].size; - while (peer_count_old--) { - ot_vector *bucket_dest = bucket_list_new; - if (num_buckets_new > 1) - bucket_dest += vector_hash_peer(peers_old, OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size), num_buckets_new); - if (bucket_dest->size + 1 > bucket_dest->space) { - void *tmp = realloc(bucket_dest->data, peer_size * OT_VECTOR_GROW_RATIO * bucket_dest->space); - if (!tmp) { - vector_clean_list(bucket_list_new, num_buckets_new); - return; - } - bucket_dest->data = tmp; - bucket_dest->space *= OT_VECTOR_GROW_RATIO; - } - memcpy((ot_peer *)bucket_dest->data + peer_size * bucket_dest->size++, peers_old, peer_size); - peers_old += peer_size; - } - } - - /* Now sort each bucket to later allow bsearch */ - for (bucket = 0; bucket < num_buckets_new; ++bucket) - qsort(bucket_list_new[bucket].data, bucket_list_new[bucket].size, peer_size, sort_func); - - /* Everything worked fine. Now link new bucket_list to peer_list */ - if (OT_PEERLIST_HASBUCKETS(peer_list)) - vector_clean_list((ot_vector *)peer_list->peers.data, peer_list->peers.size); - else - free(peer_list->peers.data); - - if (num_buckets_new > 1) { - peer_list->peers.data = bucket_list_new; - peer_list->peers.size = num_buckets_new; - peer_list->peers.space = 0; /* Magic marker for "is list of buckets" */ - } else { - peer_list->peers.data = bucket_list_new->data; - peer_list->peers.size = bucket_list_new->size; - peer_list->peers.space = bucket_list_new->space; - free(bucket_list_new); - } -} - -void vector_fixup_peers(ot_vector *vector, size_t peer_size) { - int need_fix = 0; - - if (!vector->size) { - free(vector->data); - vector->data = NULL; - vector->space = 0; - return; - } - - while ((vector->size * OT_VECTOR_SHRINK_THRESH < vector->space) && (vector->space >= OT_VECTOR_SHRINK_RATIO * OT_VECTOR_MIN_MEMBERS)) { - vector->space /= OT_VECTOR_SHRINK_RATIO; - need_fix++; - } - if (need_fix) - vector->data = realloc(vector->data, vector->space * peer_size); -} diff --git a/ot_vector.h b/ot_vector.h deleted file mode 100644 index 8a5ba86..0000000 --- a/ot_vector.h +++ /dev/null @@ -1,39 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_VECTOR_H__ -#define OT_VECTOR_H__ - -#include - -/* These defines control vectors behaviour */ -#define OT_VECTOR_MIN_MEMBERS 2 -#define OT_VECTOR_GROW_RATIO 2 -#define OT_VECTOR_SHRINK_THRESH 4 -#define OT_VECTOR_SHRINK_RATIO 2 - -#define OT_PEER_BUCKET_MINCOUNT 512 -#define OT_PEER_BUCKET_MAXCOUNT 256 - -typedef struct { - void *data; - size_t size; - size_t space; -} ot_vector; - -/* trackerlogic.h needs ot_vector, break circular dependency here */ -#include "trackerlogic.h" - -void *vector_binary_search(const void *const key, const void *base, const size_t member_count, const size_t member_size, size_t compare_size, int *exactmatch); -void *vector_find_or_insert(ot_vector *vector, void *key, size_t member_size, size_t compare_size, int *exactmatch); -ot_peer *vector_find_or_insert_peer(ot_vector *vector, ot_peer const *peer, size_t peer_size, int *exactmatch); - -int vector_remove_peer(ot_vector *vector, ot_peer const *peer, size_t peer_size); -void vector_remove_torrent(ot_vector *vector, ot_torrent *match); - -/* For ot_clean.c */ -void vector_redistribute_buckets(ot_peerlist *peer_list, size_t peer_size); -void vector_fixup_peers(ot_vector *vector, size_t peer_size); -void vector_clean_list(ot_vector *vector, size_t num_buckets); - -#endif diff --git a/ot_worker.c b/ot_worker.c deleted file mode 100644 index 432d272..0000000 --- a/ot_worker.c +++ /dev/null @@ -1,73 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include -#include -#include -#include - -/* Opentracker */ -#include "ot_worker.h" -#include "trackerlogic.h" - -static struct ot_worker **g_workers; -static size_t g_worker_count; - -/* Everything a worker needs doing around its own loop, in one place, so that - a new kind of worker cannot miss a piece of it. */ -static void *worker_trampoline(void *args) { - struct ot_worker *me = args; - - if (workstruct_init(&me->ws) == -1) - fprintf(stderr, "Warning: a worker could not get its buffers and will not run.\n"); - else { - me->run(me); - workstruct_deinit(&me->ws); - } - - atomic_store(&me->left, 1); - return NULL; -} - -int worker_start(ot_worker_run run, int64_t sock) { - struct ot_worker **grown = realloc(g_workers, (g_worker_count + 1) * sizeof(*g_workers)); - struct ot_worker *worker; - - if (!grown) - return -1; - g_workers = grown; - - if (!(worker = calloc(1, sizeof(*worker)))) - return -1; - worker->sock = sock; - worker->run = run; - - if (pthread_create(&worker->thread, NULL, worker_trampoline, worker)) { - free(worker); - return -1; - } - g_workers[g_worker_count++] = worker; - return 0; -} - -void worker_stop_all(void) { - size_t i; - - for (i = 0; i < g_worker_count; ++i) { - struct ot_worker *worker = g_workers[i]; - - /* Repeated because a signal only interrupts a call already under way */ - while (!atomic_load(&worker->left)) { - pthread_kill(worker->thread, OT_SIGNAL_WAKE); - usleep(1000); - } - pthread_join(worker->thread, NULL); - free(worker); - } - - free(g_workers); - g_workers = NULL; - g_worker_count = 0; -} diff --git a/ot_worker.h b/ot_worker.h deleted file mode 100644 index 4954773..0000000 --- a/ot_worker.h +++ /dev/null @@ -1,37 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_WORKER_H__ -#define OT_WORKER_H__ - -#include -#include -#include - -#include "trackerlogic.h" - -struct ot_worker; - -/* What a worker does between being started and being told to stop. It reads - worker->sock and works out of worker->ws; everything around it, from the - buffers to the counters to saying goodbye, is handled here. */ -typedef void (*ot_worker_run)(struct ot_worker *worker); - -/* A thread that serves requests on a socket of its own, with the buffers and - counters it needs. Each is allocated on its own, so taking on another one - never moves the record a running worker holds. */ -struct ot_worker { - pthread_t thread; - _Atomic int left; - int64_t sock; - ot_worker_run run; - struct ot_workstruct ws; -}; - -/* Start one. Returns -1 if it could not be started. */ -int worker_start(ot_worker_run run, int64_t sock); - -/* Wake every worker until it has left, then join it */ -void worker_stop_all(void); - -#endif diff --git a/ot_workqueue.c b/ot_workqueue.c deleted file mode 100644 index 56a7cd2..0000000 --- a/ot_workqueue.c +++ /dev/null @@ -1,241 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include -#include -#include -#include - -/* Opentracker */ -#include "ot_iovec.h" -#include "ot_workqueue.h" -#include "trackerlogic.h" - -/* Pushing a result wakes the thread that will send it */ -extern int g_self_pipe[2]; - -struct ot_task { - ot_taskid taskid; - ot_tasktype tasktype; - int64_t sock; - size_t iovec_entries; - struct iovec *iovec; - struct ot_task *next; -}; - -static ot_taskid next_free_taskid = 1; -static struct ot_task *tasklist; -static pthread_mutex_t tasklist_mutex; -static pthread_cond_t tasklist_being_filled; - -int workqueue_pushtask(int64_t sock, ot_tasktype tasktype) { - struct ot_task **tmptask, *task; - - task = malloc(sizeof(struct ot_task)); - if (!task) - return -1; - - task->taskid = 0; - task->tasktype = tasktype; - task->sock = sock; - task->iovec_entries = 0; - task->iovec = NULL; - task->next = 0; - - /* Want exclusive access to tasklist */ - pthread_mutex_lock(&tasklist_mutex); - - /* Skip to end of list */ - tmptask = &tasklist; - while (*tmptask) - tmptask = &(*tmptask)->next; - *tmptask = task; - - /* Inform waiting workers and release lock */ - pthread_cond_broadcast(&tasklist_being_filled); - pthread_mutex_unlock(&tasklist_mutex); - return 0; -} - -void workqueue_canceltask(int64_t sock) { - struct ot_task **task; - - /* Want exclusive access to tasklist */ - pthread_mutex_lock(&tasklist_mutex); - - for (task = &tasklist; *task; task = &((*task)->next)) - if ((*task)->sock == sock) { - struct iovec *iovec = (*task)->iovec; - struct ot_task *ptask = *task; - size_t i; - - /* Free task's iovec, the buffers it points at and the vector itself */ - for (i = 0; i < (*task)->iovec_entries; ++i) - free(iovec[i].iov_base); - free(iovec); - - *task = (*task)->next; - free(ptask); - break; - } - - /* Release lock */ - pthread_mutex_unlock(&tasklist_mutex); -} - -ot_taskid workqueue_poptask(ot_tasktype *tasktype) { - struct ot_task *task; - ot_taskid taskid = 0; - - /* Want exclusive access to tasklist */ - pthread_mutex_lock(&tasklist_mutex); - - while (!taskid) { - /* Skip to the first unassigned task this worker wants to do */ - for (task = tasklist; task; task = task->next) - if (!task->taskid && (TASK_CLASS_MASK & task->tasktype) == *tasktype) { - /* If we found an outstanding task, assign a taskid to it - and leave the loop */ - task->taskid = taskid = ++next_free_taskid; - *tasktype = task->tasktype; - break; - } - - /* Wait until the next task is being fed, unless the tracker is on its - way out, in which case the worker is meant to leave rather than wait */ - if (!taskid) { - if (!g_opentracker_running) - break; - pthread_cond_wait(&tasklist_being_filled, &tasklist_mutex); - } - } - - /* Release lock */ - pthread_mutex_unlock(&tasklist_mutex); - - return taskid; -} - -void workqueue_pushsuccess(ot_taskid taskid) { - struct ot_task **task; - - /* Want exclusive access to tasklist */ - pthread_mutex_lock(&tasklist_mutex); - - for (task = &tasklist; *task; task = &((*task)->next)) - if ((*task)->taskid == taskid) { - struct ot_task *ptask = *task; - *task = (*task)->next; - free(ptask); - break; - } - - /* Release lock */ - pthread_mutex_unlock(&tasklist_mutex); -} - -int workqueue_pushresult(ot_taskid taskid, size_t iovec_entries, struct iovec *iovec) { - struct ot_task *task; - const char byte = 'o'; - - /* Want exclusive access to tasklist */ - pthread_mutex_lock(&tasklist_mutex); - - for (task = tasklist; task; task = task->next) - if (task->taskid == taskid) { - task->iovec_entries = iovec_entries; - task->iovec = iovec; - task->tasktype = TASK_DONE; - break; - } - - /* Release lock */ - pthread_mutex_unlock(&tasklist_mutex); - - /* A full pipe has already done this nudge's job, so a failure is fine */ - (void)!write(g_self_pipe[1], &byte, 1); - - /* Indicate whether the worker has to throw away results */ - return task ? 0 : -1; -} - -int workqueue_pushchunked(ot_taskid taskid, struct iovec *iovec) { - struct ot_task *task; - const char byte = 'o'; - - /* Want exclusive access to tasklist */ - pthread_mutex_lock(&tasklist_mutex); - - for (task = tasklist; task; task = task->next) - if (task->taskid == taskid) { - if (iovec) { - if (iovec_append(&task->iovec_entries, &task->iovec, iovec)) - task->tasktype = TASK_DONE_PARTIAL; - else - task = NULL; - } else - task->tasktype = TASK_DONE; - break; - } - - /* Release lock */ - pthread_mutex_unlock(&tasklist_mutex); - - /* A full pipe has already done this nudge's job, so a failure is fine */ - (void)!write(g_self_pipe[1], &byte, 1); - - /* Indicate whether the worker has to throw away results */ - return task ? 0 : -1; -} - -int64_t workqueue_popresult(size_t *iovec_entries, struct iovec **iovec, int *is_partial) { - struct ot_task **task; - int64_t sock = -1; - - *is_partial = 0; - - /* Want exclusive access to tasklist */ - pthread_mutex_lock(&tasklist_mutex); - - for (task = &tasklist; *task; task = &((*task)->next)) - if (((*task)->tasktype & TASK_CLASS_MASK) == TASK_DONE) { - struct ot_task *ptask = *task; - *iovec_entries = ptask->iovec_entries; - *iovec = ptask->iovec; - sock = ptask->sock; - - if ((*task)->tasktype == TASK_DONE) { - *task = ptask->next; - free(ptask); - } else { - ptask->iovec_entries = 0; - ptask->iovec = NULL; - *is_partial = 1; - /* Prevent task from showing up immediately again unless new data was added */ - (*task)->tasktype = TASK_FULLSCRAPE; - } - break; - } - - /* Release lock */ - pthread_mutex_unlock(&tasklist_mutex); - return sock; -} - -void workqueue_wakeall(void) { - pthread_mutex_lock(&tasklist_mutex); - pthread_cond_broadcast(&tasklist_being_filled); - pthread_mutex_unlock(&tasklist_mutex); -} - -void workqueue_init(void) { - pthread_mutex_init(&tasklist_mutex, NULL); - pthread_cond_init(&tasklist_being_filled, NULL); -} - -void workqueue_deinit(void) { - pthread_mutex_destroy(&tasklist_mutex); - pthread_cond_destroy(&tasklist_being_filled); -} diff --git a/ot_workqueue.h b/ot_workqueue.h deleted file mode 100644 index 82ae29b..0000000 --- a/ot_workqueue.h +++ /dev/null @@ -1,74 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_WORKQUEUE_H__ -#define OT_WORKQUEUE_H__ - -#include -#include - -#include "trackerlogic.h" - -/* Work the tracker hands to a thread of its own: a full scrape or an - expensive stats page, which take too long to build while a client waits. - The task goes in here, the reply comes back out. */ - -typedef enum { - TASK_STATS_CONNS = 0x0001, - TASK_STATS_TCP = 0x0002, - TASK_STATS_UDP = 0x0003, - TASK_STATS_SCRAPE = 0x0004, - TASK_STATS_FULLSCRAPE = 0x0005, - TASK_STATS_TPB = 0x0006, - TASK_STATS_HTTPERRORS = 0x0007, - TASK_STATS_RENEW = 0x000a, - TASK_STATS_SYNCS = 0x000b, - TASK_STATS_COMPLETED = 0x000c, - TASK_STATS_NUMWANTS = 0x000d, - - TASK_STATS = 0x0100, /* Mask */ - TASK_STATS_TORRENTS = 0x0101, - TASK_STATS_PEERS = 0x0102, - TASK_STATS_SLASH24S = 0x0103, - TASK_STATS_TOP10 = 0x0104, - TASK_STATS_TOP100 = 0x0105, - TASK_STATS_EVERYTHING = 0x0106, - TASK_STATS_WOODPECKERS = 0x0108, - - TASK_FULLSCRAPE = 0x0200, /* Default mode */ - TASK_FULLSCRAPE_TPB_BINARY = 0x0201, - TASK_FULLSCRAPE_TPB_ASCII = 0x0202, - TASK_FULLSCRAPE_TPB_ASCII_PLUS = 0x0203, - TASK_FULLSCRAPE_TPB_URLENCODED = 0x0204, - TASK_FULLSCRAPE_TRACKERSTATE = 0x0205, - - TASK_DMEM = 0x0300, - - TASK_DONE = 0x0f00, - TASK_DONE_PARTIAL = 0x0f01, - - TASK_FLAG_GZIP = 0x1000, - TASK_FLAG_BZIP2 = 0x2000, - TASK_FLAG_ZSTD = 0x4000, - TASK_FLAG_CHUNKED = 0x8000, - - TASK_TASK_MASK = 0x0fff, - TASK_CLASS_MASK = 0x0f00, - TASK_FLAGS_MASK = 0xf000 -} ot_tasktype; - -typedef unsigned long ot_taskid; - -int workqueue_pushtask(int64_t sock, ot_tasktype tasktype); -void workqueue_canceltask(int64_t sock); -void workqueue_pushsuccess(ot_taskid taskid); -ot_taskid workqueue_poptask(ot_tasktype *tasktype); -int workqueue_pushresult(ot_taskid taskid, size_t iovec_entries, struct iovec *iovector); -int workqueue_pushchunked(ot_taskid taskid, struct iovec *iovec); -int64_t workqueue_popresult(size_t *iovec_entries, struct iovec **iovector, int *is_partial); - -void workqueue_init(void); -void workqueue_deinit(void); -void workqueue_wakeall(void); - -#endif diff --git a/proxy.c b/proxy.c deleted file mode 100644 index 8589db5..0000000 --- a/proxy.c +++ /dev/null @@ -1,871 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* This relays sync traffic between trackers, and between multicast domains - that have no route to each other, so that no tracker has to reach out to - the outside world by itself. Defunc. */ - -/* System */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* Libowfat */ -#include "io.h" - -/* Opentracker */ -#include "ot_batch.h" -#include "ot_fdstate.h" -#include "ot_ip.h" -#include "ot_mutex.h" -#include "ot_random.h" -#include "ot_scan.h" -#include "ot_socket.h" -#include "ot_stats.h" -#include "ot_vector.h" -#include "trackerlogic.h" - -#include "ot_livesync.h" - -ot_ip6 g_serverip; -uint16_t g_serverport = 9009; -uint32_t g_tracker_id; -ot_ip6 groupip_1 = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 224, 0, 23, 5}; -int g_self_pipe[2]; - -/* If you have more than 10 peers, don't use this proxy - Use 20 slots for 10 peers to have room for 10 incoming connection slots - */ -#define MAX_PEERS 20 - -#define LIVESYNC_INCOMING_BUFFSIZE (256 * 256) -#define STREAMSYNC_OUTGOING_BUFFSIZE (256 * 256) - -#define LIVESYNC_OUTGOING_BUFFSIZE_PEERS 1480 -#define LIVESYNC_OUTGOING_WATERMARK_PEERS (sizeof(ot_peer) + sizeof(ot_hash)) -#define LIVESYNC_MAXDELAY 15 /* seconds */ - -/* The amount of time a complete sync cycle should take */ -#define OT_SYNC_INTERVAL_MINUTES 2 - -/* So after each bucket wait 1 / OT_BUCKET_COUNT intervals */ -#define OT_SYNC_SLEEP (((OT_SYNC_INTERVAL_MINUTES) * 60 * 1000000) / (OT_BUCKET_COUNT)) - -enum { OT_SYNC_PEER4, OT_SYNC_PEER6 }; -enum { FLAG_SERVERSOCKET = 1 }; - -/* For incoming packets */ -static int g_socket_in = -1; -static uint8_t g_inbuffer[LIVESYNC_INCOMING_BUFFSIZE]; - -/* For outgoing packets */ -static int g_socket_out = -1; -static uint8_t g_peerbuffer_start[LIVESYNC_OUTGOING_BUFFSIZE_PEERS]; -static uint8_t *g_peerbuffer_pos; -static uint8_t *g_peerbuffer_highwater = g_peerbuffer_start + LIVESYNC_OUTGOING_BUFFSIZE_PEERS - LIVESYNC_OUTGOING_WATERMARK_PEERS; -static ot_time g_next_packet_time; - -static void *livesync_worker(void *args); -static void *streamsync_worker(void *args); -static void livesync_proxytell(uint8_t prefix, uint8_t *info_hash, uint8_t *peer); - -void exerr(const char *message) { - fprintf(stderr, "%s\n", message); - exit(111); -} - -void stats_issue_event(struct ot_workstruct *ws, ot_status_event event, PROTO_FLAG proto, uintptr_t event_data) { - (void)ws; - (void)event; - (void)proto; - (void)event_data; -} - -void livesync_bind_mcast(ot_ip6 ip, uint16_t port) { - if (!ip_is_v4mapped(ip)) - exerr("v6 mcast support not yet available."); - - if (g_socket_in != -1) - exerr("Error: Livesync listen ip specified twice."); - - if ((g_socket_in = socket_mcast_join(groupip_1, ip, port)) == -1) - exerr("Error: Cant bind live sync incoming socket."); - - if ((g_socket_out = socket_mcast_bind(ip, port, 1)) == -1) - exerr("Error: Cant bind live sync outgoing socket."); -} - -size_t add_peer_to_torrent_proxy(ot_hash hash, ot_peer *peer, size_t peer_size) { - int exactmatch; - ot_torrent *torrent; - ot_peerlist *peer_list; - ot_peer *peer_dest; - ot_vector *torrents_list = mutex_bucket_lock_by_hash(hash); - size_t compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size); - - torrent = vector_find_or_insert(torrents_list, (void *)hash, sizeof(ot_torrent), compare_size, &exactmatch); - if (!torrent) - return -1; - - if (!exactmatch) { - /* Create a new torrent entry, then */ - memcpy(torrent->hash, hash, sizeof(ot_hash)); - - if (!(torrent->peer_list6 = malloc(sizeof(ot_peerlist))) || !(torrent->peer_list4 = malloc(sizeof(ot_peerlist)))) { - vector_remove_torrent(torrents_list, torrent); - mutex_bucket_unlock_by_hash(hash, 0); - return -1; - } - - memset(torrent->peer_list6, 0, sizeof(ot_peerlist)); - memset(torrent->peer_list4, 0, sizeof(ot_peerlist)); - } - - peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; - - /* Check for peer in torrent */ - peer_dest = vector_find_or_insert_peer(&(peer_list->peers), peer, peer_size, &exactmatch); - if (!peer_dest) { - mutex_bucket_unlock_by_hash(hash, 0); - return -1; - } - /* Tell peer that it's fresh */ - OT_PEERTIME(peer, peer_size) = 0; - - /* If we hadn't had a match create peer there */ - if (!exactmatch) { - peer_list->peer_count++; - if (OT_PEERFLAG_D(peer, peer_size) & PEER_FLAG_SEEDING) - peer_list->seed_count++; - } - memcpy(peer_dest, peer, peer_size); - mutex_bucket_unlock_by_hash(hash, 0); - return 0; -} - -size_t remove_peer_from_torrent_proxy(ot_hash hash, ot_peer *peer, size_t peer_size) { - int exactmatch; - ot_vector *torrents_list = mutex_bucket_lock_by_hash(hash); - ot_torrent *torrent = vector_binary_search(hash, torrents_list->data, torrents_list->size, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); - - if (exactmatch) { - ot_peerlist *peer_list = peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; - switch (vector_remove_peer(&peer_list->peers, peer, peer_size)) { - case 2: - peer_list->seed_count--; /* Intentional fallthrough */ - case 1: - peer_list->peer_count--; /* Intentional fallthrough */ - default: - break; - } - } - - mutex_bucket_unlock_by_hash(hash, 0); - return 0; -} - -void free_peerlist(ot_peerlist *peer_list) { - if (peer_list->peers.data) { - if (OT_PEERLIST_HASBUCKETS(peer_list)) { - ot_vector *bucket_list = (ot_vector *)(peer_list->peers.data); - - while (peer_list->peers.size--) - free(bucket_list++->data); - } - free(peer_list->peers.data); - } - free(peer_list); -} - -static void livesync_handle_peersync(ssize_t datalen, size_t peer_size) { - int off = sizeof(g_tracker_id) + sizeof(uint32_t); - - fprintf(stderr, "."); - - while ((ssize_t)(off + sizeof(ot_hash) + peer_size) <= datalen) { - ot_peer *peer = (ot_peer *)(g_inbuffer + off + sizeof(ot_hash)); - ot_hash *hash = (ot_hash *)(g_inbuffer + off); - - if (OT_PEERFLAG_D(peer, peer_size) & PEER_FLAG_STOPPED) - remove_peer_from_torrent_proxy(*hash, peer, peer_size); - else - add_peer_to_torrent_proxy(*hash, peer, peer_size); - - off += sizeof(ot_hash) + peer_size; - } -} - -int usage(char *self) { - fprintf(stderr, "Usage: %s -L -l : -c :\n", self); - return 0; -} - -enum { - FLAG_OUTGOING = 0x80, - - FLAG_DISCONNECTED = 0x00, - FLAG_CONNECTING = 0x01, - FLAG_WAITTRACKERID = 0x02, - FLAG_CONNECTED = 0x03, - - FLAG_MASK = 0x07 -}; - -#define PROXYPEER_NEEDSCONNECT(flag) ((flag) == FLAG_OUTGOING) -#define PROXYPEER_ISCONNECTED(flag) (((flag) & FLAG_MASK) == FLAG_CONNECTED) -#define PROXYPEER_SETDISCONNECTED(flag) (flag) = (((flag) & FLAG_OUTGOING) | FLAG_DISCONNECTED) -#define PROXYPEER_SETCONNECTING(flag) (flag) = (((flag) & FLAG_OUTGOING) | FLAG_CONNECTING) -#define PROXYPEER_SETWAITTRACKERID(flag) (flag) = (((flag) & FLAG_OUTGOING) | FLAG_WAITTRACKERID) -#define PROXYPEER_SETCONNECTED(flag) (flag) = (((flag) & FLAG_OUTGOING) | FLAG_CONNECTED) - -typedef struct { - int state; /* Whether we want to connect, how far our handshake is, etc. */ - ot_ip6 ip; /* The peer to connect to */ - uint16_t port; /* The peers port */ - uint8_t indata[8192 * 16]; /* Any data not processed yet */ - size_t indata_length; /* Length of unprocessed data */ - uint32_t tracker_id; /* How the other end greeted */ - int64_t fd; /* A file handle, if connected, <= 0 is disconnected (0 initially, -1 else) */ - ot_batch outdata; /* The batch containing our sync data */ - - size_t packet_tcount; /* Number of unprocessed torrents in packet we currently receive */ - uint8_t packet_tprefix; /* Prefix byte for all torrents in current packet */ - uint8_t packet_type; /* Type of current packet */ - uint32_t packet_tid; /* Tracker id for current packet */ - -} proxy_peer; -static void process_indata(proxy_peer *peer); - -void reset_info_block(proxy_peer *peer) { - peer->indata_length = 0; - peer->tracker_id = 0; - peer->fd = -1; - peer->packet_tcount = 0; - batch_reset(&peer->outdata); - PROXYPEER_SETDISCONNECTED(peer->state); -} - -/* Number of connections to peers - * If a peer's IP is set, we try to reconnect, when the connection drops - * If we already have a connected tracker_id in our records for an _incoming_ connection, drop it - * Multiple connections to/from the same ip are okay, if tracker_id doesn't match - * Reconnect attempts occur only twice a minute - */ -static size_t g_connection_count; -static ot_time g_connection_reconn; -static proxy_peer g_connections[MAX_PEERS]; - -static void handle_reconnects(void) { - size_t i; - for (i = 0; i < g_connection_count; ++i) - if (PROXYPEER_NEEDSCONNECT(g_connections[i].state)) { - int64_t newfd = socket_tcp_connect(g_serverip, g_serverport, g_connections[i].ip, g_connections[i].port); - fprintf(stderr, "(Re)connecting to peer..."); - if (newfd == -1) - continue; /* No socket for you */ - io_fd(newfd); - io_wantwrite(newfd); /* So we will be informed when it is connected */ - fdstate_open(newfd, g_connections + i, 0); - - /* Prepare connection info block */ - reset_info_block(g_connections + i); - g_connections[i].fd = newfd; - PROXYPEER_SETCONNECTING(g_connections[i].state); - } - g_connection_reconn = time(NULL) + 30; -} - -/* Handle incoming connection requests, check against whitelist */ -static void handle_accept(int64_t serversocket) { - int64_t newfd; - ot_ip6 ip; - uint16_t port; - - while ((newfd = socket_accept(serversocket, ip, &port)) != -1) { - - /* XXX some access control */ - - /* Put fd into a non-blocking mode */ - io_nonblock(newfd); - - if (!io_fd(newfd)) - io_close(newfd); - else { - /* Find a new home for our incoming connection */ - int i; - for (i = 0; i < MAX_PEERS; ++i) - if (g_connections[i].state == FLAG_DISCONNECTED) - break; - if (i == MAX_PEERS) { - fprintf(stderr, "No room for incoming connection."); - close(newfd); - continue; - } - - /* Prepare connection info block */ - reset_info_block(g_connections + i); - PROXYPEER_SETCONNECTING(g_connections[i].state); - g_connections[i].port = port; - g_connections[i].fd = newfd; - - fdstate_open(newfd, g_connections + i, 0); - - /* We expect the connecting side to begin with its tracker_id */ - io_wantread(newfd); - } - } - - return; -} - -/* New sync data on the stream */ -static void handle_read(int64_t peersocket) { - int i; - int64_t datalen; - uint32_t tracker_id; - proxy_peer *peer = fdstate_cookie(peersocket); - - if (!peer) { - /* Can't happen ;) */ - io_close(peersocket); - return; - } - switch (peer->state & FLAG_MASK) { - case FLAG_DISCONNECTED: - io_close(peersocket); - break; /* Shouldnt happen */ - case FLAG_CONNECTING: - case FLAG_WAITTRACKERID: - /* We want at least the first four bytes to come at once, to avoid keeping extra states (for now) - This also catches 0 bytes reads == EOF and negative values, denoting connection errors */ - if (io_tryread(peersocket, (void *)&tracker_id, sizeof(tracker_id)) != sizeof(tracker_id)) - goto close_socket; - - /* See, if we already have a connection to that peer */ - for (i = 0; i < MAX_PEERS; ++i) - if ((g_connections[i].state & FLAG_MASK) == FLAG_CONNECTED && g_connections[i].tracker_id == tracker_id) { - fprintf(stderr, "Peer already connected. Closing connection.\n"); - goto close_socket; - } - - /* Also no need for soliloquy */ - if (tracker_id == g_tracker_id) - goto close_socket; - - /* The new connection is good, send our tracker_id on incoming connections */ - if (peer->state == FLAG_CONNECTING) - if (io_trywrite(peersocket, (void *)&g_tracker_id, sizeof(g_tracker_id)) != sizeof(g_tracker_id)) - goto close_socket; - - peer->tracker_id = tracker_id; - PROXYPEER_SETCONNECTED(peer->state); - - if (peer->state & FLAG_OUTGOING) - fprintf(stderr, "succeeded.\n"); - else - fprintf(stderr, "Incoming connection successful.\n"); - - break; - close_socket: - fprintf(stderr, "Handshake incomplete, closing socket\n"); - io_close(peersocket); - reset_info_block(peer); - break; - case FLAG_CONNECTED: - /* Here we acutally expect data from peer - indata_length should be less than 20+256*7 bytes, for incomplete torrent entries */ - datalen = io_tryread(peersocket, (void *)(peer->indata + peer->indata_length), sizeof(peer->indata) - peer->indata_length); - if (!datalen || datalen < -1) { - fprintf(stderr, "Connection closed by remote peer.\n"); - io_close(peersocket); - reset_info_block(peer); - } else if (datalen > 0) { - peer->indata_length += datalen; - process_indata(peer); - } - break; - } -} - -/* Can write new sync data to the stream */ -static void handle_write(int64_t peersocket) { - proxy_peer *peer = fdstate_cookie(peersocket); - - if (!peer) { - /* Can't happen ;) */ - io_close(peersocket); - return; - } - - switch (peer->state & FLAG_MASK) { - case FLAG_DISCONNECTED: - default: /* Should not happen */ - io_close(peersocket); - break; - case FLAG_CONNECTING: - /* Ensure that the connection is established and handle connection error */ - if (peer->state & FLAG_OUTGOING && !socket_is_connected(peersocket)) { - fprintf(stderr, "failed\n"); - reset_info_block(peer); - io_close(peersocket); - break; - } - - if (io_trywrite(peersocket, (void *)&g_tracker_id, sizeof(g_tracker_id)) == sizeof(g_tracker_id)) { - PROXYPEER_SETWAITTRACKERID(peer->state); - io_dontwantwrite(peersocket); - io_wantread(peersocket); - } else { - fprintf(stderr, "Handshake incomplete, closing socket\n"); - io_close(peersocket); - reset_info_block(peer); - } - break; - case FLAG_CONNECTED: - switch (batch_send(peersocket, &peer->outdata)) { - case BATCH_DONE: - io_dontwantwrite(peersocket); - break; - case BATCH_FAILED: - io_close(peersocket); - reset_info_block(peer); - break; - default: /* Normal operation or eagain */ - break; - } - break; - } - - return; -} - -static void server_mainloop() { - int64_t sock; - uint32_t packet_type; - - /* inlined livesync_init() */ - memset(g_peerbuffer_start, 0, sizeof(g_peerbuffer_start)); - g_peerbuffer_pos = g_peerbuffer_start; - memcpy(g_peerbuffer_pos, &g_tracker_id, sizeof(g_tracker_id)); - packet_type = htonl(OT_SYNC_PEER); - memcpy(g_peerbuffer_pos + sizeof(g_tracker_id), &packet_type, sizeof(packet_type)); - g_peerbuffer_pos += sizeof(g_tracker_id) + sizeof(uint32_t); - g_next_packet_time = time(NULL) + LIVESYNC_MAXDELAY; - - while (1) { - /* See if we need to connect to anyone */ - if (time(NULL) > g_connection_reconn) - handle_reconnects(); - - /* Wait for io events until next approx reconn check time */ - io_waituntil2(30 * 1000); - - /* Loop over readable sockets */ - while ((sock = io_canread()) != -1) { - const void *cookie = fdstate_cookie(sock); - if ((uintptr_t)cookie == FLAG_SERVERSOCKET) - handle_accept(sock); - else - handle_read(sock); - } - - /* Loop over writable sockets */ - while ((sock = io_canwrite()) != -1) - handle_write(sock); - - livesync_ticker(); - } -} - -static void panic(const char *routine) { - fprintf(stderr, "%s: %s\n", routine, strerror(errno)); - exit(111); -} - -static int64_t ot_try_bind(ot_ip6 ip, uint16_t port) { - int64_t sock = socket_tcp_listen(ip, port, SOCKET_EXCLUSIVE); - - if (sock == -1) - panic("socket_tcp_listen"); - - if (!io_fd(sock)) - panic("io_fd"); - - fdstate_open(sock, (void *)FLAG_SERVERSOCKET, 0); - io_wantread(sock); - return sock; -} - -static int scan_ip6_port(const char *src, ot_ip6 ip, uint16_t *port) { - const char *s = src; - uint64_t value; - int off, bracket = 0; - while (isspace(*s)) - ++s; - if (*s == '[') - ++s, ++bracket; /* for v6 style notation */ - if (!(off = ip_parse(s, ip))) - return 0; - s += off; - if (*s == 0 || isspace(*s)) - return s - src; - if (*s == ']' && bracket) - ++s; - if (!ip_is_v4mapped(ip)) { - if ((bracket && *(s) != ':') || (*(s) != '.')) - return 0; - s++; - } else { - if (*(s++) != ':') - return 0; - } - if (!(off = scan_uint(s, &value, 65535))) - return 0; - *port = (uint16_t)value; - return off + s - src; -} - -int main(int argc, char **argv) { - static pthread_t sync_in_thread_id; - static pthread_t sync_out_thread_id; - ot_ip6 serverip; - uint16_t tmpport; - int scanon = 1, lbound = 0, sbound = 0; - - random_init(); - if (fdstate_init() == -1) - panic("fdstate_init"); - g_tracker_id = random_uint32(); - - while (scanon) { - switch (getopt(argc, argv, ":l:c:L:h")) { - case -1: - scanon = 0; - break; - case 'l': - tmpport = 0; - if (!scan_ip6_port(optarg, serverip, &tmpport) || !tmpport) { - usage(argv[0]); - exit(1); - } - ot_try_bind(serverip, tmpport); - ++sbound; - break; - case 'c': - if (g_connection_count > MAX_PEERS / 2) - exerr("Connection limit exceeded.\n"); - tmpport = 0; - if (!scan_ip6_port(optarg, g_connections[g_connection_count].ip, &g_connections[g_connection_count].port) || !g_connections[g_connection_count].port) { - usage(argv[0]); - exit(1); - } - g_connections[g_connection_count++].state = FLAG_OUTGOING; - break; - case 'L': - tmpport = 9696; - if (!scan_ip6_port(optarg, serverip, &tmpport) || !tmpport) { - usage(argv[0]); - exit(1); - } - livesync_bind_mcast(serverip, tmpport); - ++lbound; - break; - default: - case '?': - usage(argv[0]); - exit(1); - } - } - - if (!lbound) - exerr("No livesync port bound."); - if (!g_connection_count && !sbound) - exerr("No streamsync port bound."); - pthread_create(&sync_in_thread_id, NULL, livesync_worker, NULL); - pthread_create(&sync_out_thread_id, NULL, streamsync_worker, NULL); - - server_mainloop(); - return 0; -} - -static void *streamsync_worker(void *args) { - (void)args; - while (1) { - size_t bucket; - /* For each bucket... */ - for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { - /* Get exclusive access to that bucket */ - ot_vector *torrents_list = mutex_bucket_lock(bucket); - size_t tor_offset, count_def = 0, count_one = 0, count_two = 0, count_peers = 0; - size_t mem, mem_a = 0, mem_b = 0; - uint8_t *ptr = 0, *ptr_a, *ptr_b, *ptr_c; - - if (!torrents_list->size) - goto unlock_continue; - - /* For each torrent in this bucket.. */ - for (tor_offset = 0; tor_offset < torrents_list->size; ++tor_offset) { - /* Address torrents members */ - ot_peerlist *peer_list = (((ot_torrent *)(torrents_list->data))[tor_offset]).peer_list; - switch (peer_list->peer_count) { - case 2: - count_two++; - break; - case 1: - count_one++; - break; - case 0: - break; - default: - count_def++; - count_peers += peer_list->peer_count; - } - } - - /* Maximal memory requirement: max 3 blocks, max torrents * 20 + max peers * 7 */ - mem = 3 * (1 + 1 + 2) + (count_one + count_two) * (19 + 1) + count_def * (19 + 8) + (count_one + 2 * count_two + count_peers) * 7; - - fprintf(stderr, "Mem: %zd\n", mem); - - ptr = ptr_a = ptr_b = ptr_c = malloc(mem); - if (!ptr) - goto unlock_continue; - - if (count_one > 4 || !count_def) { - mem_a = 1 + 1 + 2 + count_one * (19 + 7); - ptr_b += mem_a; - ptr_c += mem_a; - ptr_a[0] = 1; /* Offset 0: packet type 1 */ - ptr_a[1] = (bucket << 8) >> OT_BUCKET_COUNT_BITS; /* Offset 1: the shared prefix */ - ptr_a[2] = count_one >> 8; - ptr_a[3] = count_one & 255; - ptr_a += 4; - } else - count_def += count_one; - - if (count_two > 4 || !count_def) { - mem_b = 1 + 1 + 2 + count_two * (19 + 14); - ptr_c += mem_b; - ptr_b[0] = 2; /* Offset 0: packet type 2 */ - ptr_b[1] = (bucket << 8) >> OT_BUCKET_COUNT_BITS; /* Offset 1: the shared prefix */ - ptr_b[2] = count_two >> 8; - ptr_b[3] = count_two & 255; - ptr_b += 4; - } else - count_def += count_two; - - if (count_def) { - ptr_c[0] = 0; /* Offset 0: packet type 0 */ - ptr_c[1] = (bucket << 8) >> OT_BUCKET_COUNT_BITS; /* Offset 1: the shared prefix */ - ptr_c[2] = count_def >> 8; - ptr_c[3] = count_def & 255; - ptr_c += 4; - } - - /* For each torrent in this bucket.. */ - for (tor_offset = 0; tor_offset < torrents_list->size; ++tor_offset) { - /* Address torrents members */ - ot_torrent *torrent = ((ot_torrent *)(torrents_list->data)) + tor_offset; - ot_peerlist *peer_list = torrent->peer_list; - ot_peer *peers = (ot_peer *)(peer_list->peers.data); - uint8_t **dst; - - /* Determine destination slot */ - count_peers = peer_list->peer_count; - switch (count_peers) { - case 0: - continue; - case 1: - dst = mem_a ? &ptr_a : &ptr_c; - break; - case 2: - dst = mem_b ? &ptr_b : &ptr_c; - break; - default: - dst = &ptr_c; - break; - } - - /* Copy tail of info_hash, advance pointer */ - memcpy(*dst, ((uint8_t *)torrent->hash) + 1, sizeof(ot_hash) - 1); - *dst += sizeof(ot_hash) - 1; - - /* Encode peer count */ - if (dst == &ptr_c) - while (count_peers) { - if (count_peers <= 0x7f) - *(*dst)++ = count_peers; - else - *(*dst)++ = 0x80 | (count_peers & 0x7f); - count_peers >>= 7; - } - - /* Copy peers */ - count_peers = peer_list->peer_count; - while (count_peers--) { - memcpy(*dst, peers++, OT_IP_SIZE + 3); - *dst += OT_IP_SIZE + 3; - } - free_peerlist(peer_list); - } - - free(torrents_list->data); - memset(torrents_list, 0, sizeof(*torrents_list)); - unlock_continue: - mutex_bucket_unlock(bucket, 0); - - if (ptr) { - int i; - - if (ptr_b > ptr_c) - ptr_c = ptr_b; - if (ptr_a > ptr_c) - ptr_c = ptr_a; - mem = ptr_c - ptr; - - for (i = 0; i < MAX_PEERS; ++i) { - if (PROXYPEER_ISCONNECTED(g_connections[i].state)) { - void *tmp = malloc(mem); - if (tmp) { - memcpy(tmp, ptr, mem); - batch_add(&g_connections[i].outdata, tmp, mem); - io_wantwrite(g_connections[i].fd); - } - } - } - - free(ptr); - } - usleep(OT_SYNC_SLEEP); - } - } - return 0; -} - -static void livesync_issue_peersync() { - socket_mcast_send(g_socket_out, g_peerbuffer_start, (size_t)(g_peerbuffer_pos - g_peerbuffer_start), groupip_1, LIVESYNC_PORT); - g_peerbuffer_pos = g_peerbuffer_start + sizeof(g_tracker_id) + sizeof(uint32_t); - g_next_packet_time = time(NULL) + LIVESYNC_MAXDELAY; -} - -void livesync_ticker() { - /* livesync_issue_peersync sets g_next_packet_time */ - if (time(NULL) > g_next_packet_time && g_peerbuffer_pos > g_peerbuffer_start + sizeof(g_tracker_id)) - livesync_issue_peersync(); -} - -static void livesync_proxytell(uint8_t prefix, uint8_t *info_hash, uint8_t *peer) { - // unsigned int i; - - *g_peerbuffer_pos = prefix; - memcpy(g_peerbuffer_pos + 1, info_hash, sizeof(ot_hash) - 1); - memcpy(g_peerbuffer_pos + sizeof(ot_hash), peer, sizeof(ot_peer) - 1); - -#if 0 - /* Dump info_hash */ - for( i=0; i= g_peerbuffer_highwater) - livesync_issue_peersync(); -} - -static void process_indata(proxy_peer *peer) { - size_t consumed, peers; - uint8_t *data = peer->indata, *hash; - uint8_t *dataend = data + peer->indata_length; - - while (1) { - /* If we're not inside of a packet, make a new one */ - if (!peer->packet_tcount) { - /* Ensure the header is complete or postpone processing */ - if (data + 4 > dataend) - break; - peer->packet_type = data[0]; - peer->packet_tprefix = data[1]; - peer->packet_tcount = data[2] * 256 + data[3]; - data += 4; - printf("type: %hhu, prefix: %02X, torrentcount: %zd\n", peer->packet_type, peer->packet_tprefix, peer->packet_tcount); - } - - /* Ensure size for a minimal torrent block */ - if (data + sizeof(ot_hash) + OT_IP_SIZE + 3 > dataend) - break; - - /* Advance pointer to peer count or peers */ - hash = data; - data += sizeof(ot_hash) - 1; - - /* Type 0 has peer count encoded before each peers */ - peers = peer->packet_type; - if (!peers) { - int shift = 0; - do - peers |= (0x7f & *data) << (7 * shift); - while (*(data++) & 0x80 && shift++ < 6); - } -#if 0 -printf( "peers: %zd\n", peers ); -#endif - /* Ensure enough data being read to hold all peers */ - if (data + (OT_IP_SIZE + 3) * peers > dataend) { - data = hash; - break; - } - while (peers--) { - livesync_proxytell(peer->packet_tprefix, hash, data); - data += OT_IP_SIZE + 3; - } - --peer->packet_tcount; - } - - consumed = data - peer->indata; - memmove(peer->indata, data, peer->indata_length - consumed); - peer->indata_length -= consumed; -} - -static void *livesync_worker(void *args) { - (void)args; - while (1) { - ot_ip6 in_ip; - uint16_t in_port; - uint32_t packet_type; - ssize_t datalen = socket_recv(g_socket_in, g_inbuffer, LIVESYNC_INCOMING_BUFFSIZE, in_ip, &in_port); - - /* Expect at least tracker id and packet type */ - if (datalen <= (ssize_t)(sizeof(g_tracker_id) + sizeof(uint32_t))) - continue; - if (!memcmp(g_inbuffer, &g_tracker_id, sizeof(g_tracker_id))) { - /* drop packet coming from ourselves */ - continue; - } - memcpy(&packet_type, g_inbuffer + sizeof(g_tracker_id), sizeof(packet_type)); - - switch (ntohl(packet_type)) { - case OT_SYNC_PEER4: - livesync_handle_peersync(datalen, OT_PEER_SIZE4); - break; - case OT_SYNC_PEER6: - livesync_handle_peersync(datalen, OT_PEER_SIZE6); - break; - default: - // fprintf( stderr, "Received an unknown live sync packet type %u.\n", ntohl( packet_type ) ); - break; - } - } - return 0; -} diff --git a/src/opentracker.c b/src/opentracker.c new file mode 100644 index 0000000..14b817e --- /dev/null +++ b/src/opentracker.c @@ -0,0 +1,443 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. + Some of the stuff below is stolen from Fefes example libowfat httpd. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef WANT_SYSLOGS +#include +#endif + +/* Opentracker */ +#include "ot_accesslist.h" +#include "ot_config.h" +#include "ot_hex.h" +#include "ot_http.h" +#include "ot_ip.h" +#include "ot_livesync.h" +#include "ot_mutex.h" +#include "ot_workqueue.h" +#include "ot_random.h" +#include "ot_scan.h" +#include "ot_event.h" +#include "ot_fdstate.h" +#include "ot_socket.h" +#include "ot_stats.h" +#include "ot_tcp.h" +#include "ot_udp.h" +#include "ot_worker.h" +#include "trackerlogic.h" + +/* Globals */ +uint32_t g_tracker_id; +_Atomic int g_opentracker_running = 1; +int g_self_pipe[2]; + +static void panic(const char *routine) __attribute__((noreturn)); +static void panic(const char *routine) { + fprintf(stderr, "%s: %s\n", routine, strerror(errno)); + exit(111); +} + +/* Say we are stopping, wake the loop, and let main do the work on its own + stack. Everything other than SIGINT falls through and does nothing, which + is what OT_SIGNAL_WAKE wants: returning from the read is all it is for. */ +static void signal_handler(int s) { + if (s == SIGINT) { + const char byte = 0; + int saved = errno; + + /* Any new interrupt signal quits the application */ + signal(SIGINT, SIG_DFL); + + /* Tell all other threads to not acquire any new lock on a bucket + but cancel their operations and return */ + g_opentracker_running = 0; + + (void)!write(g_self_pipe[1], &byte, 1); + errno = saved; + } +} + +static void defaul_signal_handlers(void) { + sigset_t signal_mask; + sigemptyset(&signal_mask); + sigaddset(&signal_mask, SIGPIPE); + sigaddset(&signal_mask, SIGHUP); + sigaddset(&signal_mask, SIGINT); + sigaddset(&signal_mask, SIGALRM); + pthread_sigmask(SIG_BLOCK, &signal_mask, NULL); +} + +static void install_signal_handlers(void) { + struct sigaction sa; + sigset_t signal_mask; + sigemptyset(&signal_mask); + + sa.sa_handler = signal_handler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + if ((sigaction(SIGINT, &sa, NULL) == -1) || (sigaction(SIGALRM, &sa, NULL) == -1) || + (sigaction(OT_SIGNAL_WAKE, &sa, NULL) == -1)) + panic("install_signal_handlers"); + + sigaddset(&signal_mask, SIGINT); + pthread_sigmask(SIG_UNBLOCK, &signal_mask, NULL); +} + + +/* How many ready descriptors one wait hands back at a time */ +#define OT_EVENT_BATCH 512 + +static void *server_mainloop(void *args) { + struct ot_workstruct ws; + time_t next_timeout_check = time_future_seconds(OT_CLIENT_TIMEOUT_CHECKINTERVAL); + struct iovec *iovector; + size_t iovec_entries; + int is_partial; + + (void)args; + + if (workstruct_init(&ws) == -1) + panic("Initializing worker failed"); + + while (g_opentracker_running) { + ot_event ready[OT_EVENT_BATCH]; + int64_t sock; + int count, i; + + count = event_wait(ready, OT_EVENT_BATCH, OT_CLIENT_TIMEOUT_CHECKINTERVAL * 1000); + + for (i = 0; i < count; ++i) { + const void *cookie; + + sock = ready[i].fd; + /* A descriptor closed earlier in this same batch may already have been + handed out again by accept, so an event is only ours if the + generation it was registered under is still the one in place */ + if (fdstate_generation((int)sock) != ready[i].generation) + continue; + cookie = fdstate_cookie((int)sock); + + if ((intptr_t)cookie == FLAG_TCP) + tcp_accept(sock, &ws); + else if ((intptr_t)cookie == FLAG_UDP) + handle_udp6(sock, &ws); + else if ((intptr_t)cookie == FLAG_SELFPIPE) + while (read((int)sock, ws.inbuf, G_INBUF_SIZE) > 0) + ; + else if (fdstate_claim((int)sock, ready[i].generation)) { + if (ready[i].what & EVENT_IN) + tcp_read(sock, &ws); + /* The read may have closed it, so look again before writing */ + if ((ready[i].what & EVENT_OUT) && fdstate_generation((int)sock) == ready[i].generation) + tcp_write(sock); + if (fdstate_generation((int)sock) == ready[i].generation) + fdstate_release((int)sock); + } + } + + while ((sock = workqueue_popresult(&iovec_entries, &iovector, &is_partial)) != -1) + http_sendiovecdata(sock, &ws, iovec_entries, iovector, is_partial); + + if (time_now_seconds() > next_timeout_check) { + int cursor = 0; + + while ((sock = fdstate_next_expired(time_now_seconds(), &cursor)) != -1) + http_close(sock); + next_timeout_check = time_future_seconds(OT_CLIENT_TIMEOUT_CHECKINTERVAL); + } + + livesync_ticker(); + } + + /* This work struct is about to go out of scope, so hand its tally over + before anything can read it there */ + workstruct_deinit(&ws); + return 0; +} + + +/* A group of sockets sharing one address only spreads datagrams if every + member asked for it before binding, the first one included, so whether + this entry will have workers has to be known here. */ +static socket_sharing listen_sharing_for(const ot_listen *entry) { + if (entry->proto != FLAG_MCA && entry->workers > 1 && socket_can_balance()) + return SOCKET_BALANCED; + return entry->shared ? SOCKET_SHARED : SOCKET_EXCLUSIVE; +} + +static void listen_bind_one(ot_listen *entry) { + socket_sharing sharing = listen_sharing_for(entry); + +#ifdef _DEBUG + { + const char *protos[] = {"TCP", "UDP", "UDP mcast"}; + char _debug[512]; + int off = snprintf(_debug, sizeof(_debug), "Binding socket type %s to address [", protos[entry->proto]); + off += ip_format(_debug + off, entry->ip); + snprintf(_debug + off, sizeof(_debug) - off, "]:%d...", entry->port); + fputs(_debug, stderr); + } +#endif + +#ifdef WANT_SYNC_LIVE + if (entry->proto == FLAG_MCA) { + livesync_bind_mcast(entry->ip, entry->port); +#ifdef _DEBUG + fputs(" success.\n", stderr); +#endif + return; + } +#endif + + if (entry->proto == FLAG_TCP) { + if ((entry->sock = socket_tcp_listen(entry->ip, entry->port, sharing)) == -1) + panic("socket_tcp_listen"); + socket_defer_accept((int)entry->sock); + } else { + if ((entry->sock = socket_udp_bind(entry->ip, entry->port, sharing)) == -1) + panic("socket_udp_bind"); + } + + if (event_add((int)entry->sock, EVENT_IN, fdstate_open(entry->sock, (void *)entry->proto, 0)) == -1) + panic("event_add"); + +#ifdef _DEBUG + fputs(" success.\n", stderr); +#endif +} + +/* Bound in the order the entries were given. + Note: On macOS SO_REUSEPORT delivers everything to the socket that bound last. */ +static void listen_bind_all(void) { + size_t count, i; + ot_listen *entries = config_listen_entries(&count); + + for (i = 0; i < count; ++i) + listen_bind_one(entries + i); +} + +/* The socket the event loop was using is passed to the first worker. + If start up tests show that the kernel can spread connections over + a group bound to one address, every worker binds one of its own. + If not, all workers call accept on the first. */ +static void listen_start_tcp_workers(void) { + size_t count, i; + ot_listen *entries = config_listen_entries(&count); + + for (i = 0; i < count; ++i) { + ot_listen *entry = entries + i; + int balance = listen_sharing_for(entry) == SOCKET_BALANCED; + int worker; + + if (entry->proto != FLAG_TCP || entry->workers <= 0) + continue; + + event_del((int)entry->sock); + tcp_worker_install(entry->sock); + + for (worker = 1; worker < entry->workers; ++worker) { + int64_t sock = entry->sock; + + if (balance) { + sock = socket_tcp_listen(entry->ip, entry->port, SOCKET_BALANCED); + if (sock == -1) { + fprintf(stderr, "Warning: could not bind another socket for tcp worker %d: %s. Sharing the first instead.\n", worker, + strerror(errno)); + sock = entry->sock; + } + } + tcp_worker_install(sock); + } + } +} + +static void listen_start_udp_workers(void) { + size_t count, i; + ot_listen *entries = config_listen_entries(&count); + + udp_init(); + + for (i = 0; i < count; ++i) { + ot_listen *entry = entries + i; + int balance = listen_sharing_for(entry) == SOCKET_BALANCED; + int worker; + + if (entry->proto != FLAG_UDP || entry->workers <= 0) + continue; + + event_del((int)entry->sock); + udp_worker_install(entry->sock); + + for (worker = 1; worker < entry->workers; ++worker) { + int64_t sock = entry->sock; + + if (balance) { + sock = socket_udp_bind(entry->ip, entry->port, SOCKET_BALANCED); + if (sock == -1) { + fprintf(stderr, "Warning: could not bind another socket for udp worker %d: %s. Sharing the first instead.\n", worker, strerror(errno)); + sock = entry->sock; + } + } + udp_worker_install(sock); + } + } +} + +void load_state(const char *const state_filename) { + FILE *state_filehandle; + char inbuf[512]; + ot_hash infohash; + uint64_t base, downcount; + int consumed; + + state_filehandle = fopen(state_filename, "r"); + + if (state_filehandle == NULL) { + fprintf(stderr, "Warning: Can't open config file: %s.", state_filename); + return; + } + + /* We do ignore anything that is not of the form "^[:xdigit:]:\d+:\d+" */ + while (fgets(inbuf, sizeof(inbuf), state_filehandle)) { + size_t i; + + if (!hex_to_hash(infohash, inbuf)) + continue; + i = 2 * sizeof(ot_hash); + + if (inbuf[i++] != ':' || !(consumed = scan_uint(inbuf + i, &base, UINT64_MAX))) + continue; + i += consumed; + if (inbuf[i++] != ':' || !(consumed = scan_uint(inbuf + i, &downcount, UINT64_MAX))) + continue; + add_torrent_from_saved_state(infohash, base, downcount); + } + + fclose(state_filehandle); +} + +int drop_privileges(const char *const serveruser, const char *const serverdir) { + struct passwd *pws = NULL; + +#ifdef _DEBUG + if (!geteuid()) + fprintf(stderr, "Dropping to user %s.\n", serveruser); + if (serverdir) + fprintf(stderr, "ch%s'ing to directory %s.\n", geteuid() ? "dir" : "root", serverdir); +#endif + + /* Grab pws entry before chrooting */ + pws = getpwnam(serveruser); + endpwent(); + + if (geteuid() == 0) { + /* Running as root: chroot and drop privileges */ + if (serverdir && chroot(serverdir)) { + fprintf(stderr, "Could not chroot to %s, because: %s\n", serverdir, strerror(errno)); + return -1; + } + + if (chdir("/")) + panic("chdir() failed after chrooting: "); + + /* If we can't find server user, revert to nobody's default uid */ + if (!pws) { + fprintf(stderr, "Warning: Could not get password entry for %s. Reverting to uid -2.\n", serveruser); + if (setegid((gid_t)-2) || setgid((gid_t)-2) || setuid((uid_t)-2) || seteuid((uid_t)-2)) + panic("Could not set uid to value -2"); + } else { + if (setegid(pws->pw_gid) || setgid(pws->pw_gid) || setuid(pws->pw_uid) || seteuid(pws->pw_uid)) + panic("Could not set uid to specified value"); + } + + if (geteuid() == 0 || getegid() == 0) + panic("Still running with root privileges?!"); + } else { + /* Normal user, just chdir() */ + if (serverdir && chdir(serverdir)) { + fprintf(stderr, "Could not chroot to %s, because: %s\n", serverdir, strerror(errno)); + return -1; + } + } + + return 0; +} + +/* Maintain our copy of the clock. time() on BSDs is very expensive. */ +int main(int argc, char **argv) { + + random_init(); + if (fdstate_init() == -1) + panic("fdstate_init"); + if (event_init() == -1) + panic("event_init"); + config_parse(argc, argv); + + listen_bind_all(); + + defaul_signal_handlers(); + +#ifdef WANT_SYSLOGS + openlog("opentracker", 0, LOG_USER); + setlogmask(LOG_UPTO(LOG_INFO)); +#endif + + if (drop_privileges(g_serveruser ? g_serveruser : "nobody", g_serverdir) == -1) + panic("drop_privileges failed, exiting. Last error"); + + + /* Create our self pipe which allows us to interrupt mainloops + wait in case some data is available to send out */ + if (pipe(g_self_pipe) == -1) + panic("selfpipe failed: "); + /* Both ends have to be non blocking: the loop drains the read end until it + is empty, and a worker nudging a full pipe has already made its point */ + socket_set_blocking(g_self_pipe[0], 0); + socket_set_blocking(g_self_pipe[1], 0); + if (event_add(g_self_pipe[0], EVENT_IN, fdstate_open(g_self_pipe[0], (void *)FLAG_SELFPIPE, 0)) == -1) + panic("selfpipe event_add"); + + /* Init all sub systems. This call may fail with an exit() */ + trackerlogic_init(); + +#ifdef _DEBUG_RANDOMTORRENTS + fprintf(stderr, "DEBUG: Generating %d random peers on random torrents. This may take a while. (Setting RANDOMTORRENTS in trackerlogic.h)\n", RANDOMTORRENTS); + trackerlogic_add_random_torrents(RANDOMTORRENTS); + fprintf(stderr, "... done.\n"); +#endif + + if (g_statefile) + load_state(g_statefile); + + install_signal_handlers(); + + listen_start_tcp_workers(); + listen_start_udp_workers(); + + server_mainloop(0); + worker_stop_all(); + + /* No thread is serving any more, so the teardown can run where a debugger + and the sanitizers can follow it */ + trackerlogic_deinit(); +#ifdef WANT_SYSLOGS + closelog(); +#endif + + event_deinit(); + fdstate_deinit(); + return 0; +} diff --git a/src/ot_accesslist.c b/src/ot_accesslist.c new file mode 100644 index 0000000..86f9a7d --- /dev/null +++ b/src/ot_accesslist.c @@ -0,0 +1,601 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#if defined(WANT_ACCESSLIST_BLACK) || defined(WANT_ACCESSLIST_WHITE) +#include +#include +#include +#include +#include +#endif +#ifdef WANT_DYNAMIC_ACCESSLIST +#include +#endif + +/* Opentracker */ +#include "ot_accesslist.h" +#include "ot_hex.h" +#include "ot_ip.h" +#include "ot_vector.h" +#include "trackerlogic.h" + +/* GLOBAL VARIABLES */ +#ifdef WANT_ACCESSLIST +const char *g_accesslist_filename = NULL; +#ifdef WANT_DYNAMIC_ACCESSLIST +const char *g_accesslist_pipe_add = NULL; +const char *g_accesslist_pipe_delete = NULL; +#endif +static pthread_mutex_t g_accesslist_mutex; + +/* Accesslists are lock free linked lists. We can not make them locking, because every announce + would try to acquire the mutex, making it the most contested mutex in the whole of opentracker, + basically creating a central performance choke point. + + The idea is that updating the list heads happens under the g_accesslist_mutex guard and is + done atomically, while consumers might potentially still hold pointers deeper inside the list. + + Consumers (for now only via accesslist_hashisvalid) will always fetch the list head pointer + that is guaranteed to live for at least five minutes. This should be many orders of magnitudes + more than how long it will be needed by the bsearch done on the list. */ +struct ot_accesslist; +typedef struct ot_accesslist ot_accesslist; +struct ot_accesslist { + ot_hash *list; + size_t size; + ot_time base; + ot_accesslist *next; +}; +static ot_accesslist *_Atomic g_accesslist = NULL; +#ifdef WANT_DYNAMIC_ACCESSLIST +static ot_accesslist *_Atomic g_accesslist_add = NULL; +static ot_accesslist *_Atomic g_accesslist_delete = NULL; +#endif + +/* Helpers to work on access lists */ +static int vector_compare_hash(const void *hash1, const void *hash2) { return memcmp(hash1, hash2, OT_HASH_COMPARE_SIZE); } + +static ot_accesslist *accesslist_free(ot_accesslist *accesslist) { + while (accesslist) { + ot_accesslist *this_accesslist = accesslist; + accesslist = this_accesslist->next; + free(this_accesslist->list); + free(this_accesslist); + } + return NULL; +} + +static ot_accesslist *accesslist_make(ot_accesslist *next, size_t size) { + ot_accesslist *accesslist_new = malloc(sizeof(ot_accesslist)); + if (accesslist_new) { + accesslist_new->list = size ? malloc(sizeof(ot_hash) * size) : NULL; + accesslist_new->size = size; + accesslist_new->base = 0; + accesslist_new->next = next; + if (size && !accesslist_new->list) { + free(accesslist_new); + accesslist_new = NULL; + } + } + return accesslist_new; +} + +/* This must be called with g_accesslist_mutex held. + This will never delete head, because that might still be in use. */ +static void accesslist_clean(ot_accesslist *accesslist) { + while (accesslist && accesslist->next) { + if (accesslist->next->base + 5 < time_now_minutes()) + accesslist->next = accesslist_free(accesslist->next); + accesslist = accesslist->next; + } +} + +/* A hash needs forty bytes, and all but a final one are followed by at least + one separator, so n hashes need at least 41n - 1 bytes. (len + 1) / 41 is + therefore an exact upper bound on what a buffer of len bytes can hold. */ +size_t accesslist_parse_hashes(ot_hash *dest, size_t capacity, const char *buf, size_t len) { + const char *end = buf + len; + const char *offs = buf; + size_t count = 0; + + while (count < capacity && end - offs >= 40) { + if (hex_to_hash(dest[count], offs)) { + offs += 40; + + if (offs == end || !hex_isdigit((unsigned char)*offs)) + ++count; + } + + /* Find start of next line */ + while (offs < end && *(offs++) != '\n') {} + } + + return count; +} + +/* Map a file read only. Returns 0 and sets map and maplen on success, -1 if + the file cannot be opened or mapped. */ +static int accesslist_map(const char *filename, const char **map, size_t *maplen) { + struct stat st; + void *mapped; + int fd = open(filename, O_RDONLY); + + if (fd < 0) + return -1; + + if (fstat(fd, &st) < 0 || st.st_size < 0 || (uintmax_t)st.st_size > (uintmax_t)SIZE_MAX) { + close(fd); + return -1; + } + + if (st.st_size == 0) { + close(fd); + *map = NULL; + *maplen = 0; + return 0; + } + + mapped = mmap(NULL, (size_t)st.st_size, PROT_READ, MAP_SHARED, fd, 0); + close(fd); + + if (mapped == MAP_FAILED) + return -1; + + *map = mapped; + *maplen = (size_t)st.st_size; + return 0; +} + +static void accesslist_readfile(void) { + ot_accesslist *accesslist_new; + const char *map; + size_t maplen, capacity, found; + + if (!g_accesslist_filename) + return; + + if (accesslist_map(g_accesslist_filename, &map, &maplen) < 0) { + char *wd = getcwd(NULL, 0); + fprintf(stderr, "Warning: Can't open accesslist file: %s (but will try to create it later, if necessary and possible).\nPWD: %s\n", g_accesslist_filename, wd); + free(wd); + return; + } + + /* Make room for the most info_hashes this file could possibly hold */ + capacity = (maplen + 1) / 41; + accesslist_new = accesslist_make(g_accesslist, capacity); + if (!accesslist_new) { + fprintf(stderr, "Warning: Not enough memory to allocate %zd bytes for accesslist buffer. May succeed later.\n", capacity * sizeof(ot_hash)); + if (map) + munmap((void *)map, maplen); + return; + } + + found = maplen ? accesslist_parse_hashes(accesslist_new->list, capacity, map, maplen) : 0; +#ifdef _DEBUG + fprintf(stderr, "Added %zd info_hashes to accesslist\n", found); +#endif + + if (map) + munmap((void *)map, maplen); + + if (found) + qsort(accesslist_new->list, found, sizeof(ot_hash), vector_compare_hash); + accesslist_new->size = found; + + /* Now exchange the accesslist vector in the least race condition prone way */ + pthread_mutex_lock(&g_accesslist_mutex); + + /* Mark old accesslist as "needs to live another five minutes" */ + if (g_accesslist) + g_accesslist->base = time_now_minutes(); + accesslist_new->next = g_accesslist; + g_accesslist = accesslist_new; /* Only now set a new list */ + +#ifdef WANT_DYNAMIC_ACCESSLIST + /* If we have dynamic accesslists, reloading a new one will always void the add/delete lists. + Insert empty ones at the list head */ + if (g_accesslist_add && (accesslist_new = accesslist_make(g_accesslist_add, 0)) != NULL) { + g_accesslist_add->base = time_now_minutes(); + g_accesslist_add = accesslist_new; + } + if (g_accesslist_delete && (accesslist_new = accesslist_make(g_accesslist_delete, 0)) != NULL) { + g_accesslist_delete->base = time_now_minutes(); + g_accesslist_delete = accesslist_new; + } +#endif + + accesslist_clean(g_accesslist); + + pthread_mutex_unlock(&g_accesslist_mutex); +} + +int accesslist_hashisvalid(const ot_hash hash) { + /* Get working copy of current access list */ + ot_accesslist *accesslist = g_accesslist; +#ifdef WANT_DYNAMIC_ACCESSLIST + ot_accesslist *accesslist_add, *accesslist_delete; +#endif + void *exactmatch = NULL; + + if (accesslist && accesslist->size) + exactmatch = bsearch(hash, accesslist->list, accesslist->size, OT_HASH_COMPARE_SIZE, vector_compare_hash); + +#ifdef WANT_DYNAMIC_ACCESSLIST + /* If we had no match on the main list, scan the list of dynamically added hashes */ + accesslist_add = g_accesslist_add; + if ((exactmatch == NULL) && accesslist_add && accesslist_add->size) + exactmatch = bsearch(hash, accesslist_add->list, accesslist_add->size, OT_HASH_COMPARE_SIZE, vector_compare_hash); + + /* If we found a matching hash on the main list, scan the list of dynamically deleted hashes */ + accesslist_delete = g_accesslist_delete; + if ((exactmatch != NULL) && accesslist_delete && accesslist_delete->size && bsearch(hash, accesslist_delete->list, accesslist_delete->size, OT_HASH_COMPARE_SIZE, vector_compare_hash)) + exactmatch = NULL; +#endif + +#ifdef WANT_ACCESSLIST_BLACK + return exactmatch == NULL; +#else + return exactmatch != NULL; +#endif +} + +static void *accesslist_worker(void *args) { + int sig; + sigset_t signal_mask; + + sigemptyset(&signal_mask); + sigaddset(&signal_mask, SIGHUP); + + (void)args; + + while (1) { + if (!g_opentracker_running) + return NULL; + + /* Initial attempt to read accesslist */ + accesslist_readfile(); + + /* Wait for signals */ + while (sigwait(&signal_mask, &sig) != 0 && sig != SIGHUP) + ; + } + return NULL; +} + +#ifdef WANT_DYNAMIC_ACCESSLIST +static pthread_t thread_adder_id, thread_deleter_id; +static _Atomic int adder_left, deleter_left; +static void *accesslist_adddel_worker(const char *fifoname, ot_accesslist *_Atomic *adding_to, ot_accesslist *_Atomic *removing_from) { + struct stat st; + + if (!stat(fifoname, &st)) { + if (!S_ISFIFO(st.st_mode)) { + fprintf(stderr, "Error when starting dynamic accesslists: Found Non-FIFO file at %s.\nPlease remove it and restart opentracker.\n", fifoname); + return NULL; + } + } else { + int error = mkfifo(fifoname, 0755); + if (error && error != EEXIST) { + fprintf(stderr, "Error when starting dynamic accesslists: Couldn't create FIFO at %s, error: %s\n", fifoname, strerror(errno)); + return NULL; + } + } + + while (g_opentracker_running) { + FILE *fifo = fopen(fifoname, "r"); + char *line = NULL; + size_t linecap = 0; + ssize_t linelen; + + if (!fifo) { + /* Woken on purpose: the loop condition above decides what happens next */ + if (errno == EINTR) + continue; + fprintf(stderr, "Error when reading dynamic accesslists: Couldn't open FIFO at %s, error: %s\n", fifoname, strerror(errno)); + return NULL; + } + + while ((linelen = getline(&line, &linecap, fifo)) > 0) { + ot_hash info_hash; + + printf("Got line %*s", (int)linelen, line); + /* We do ignore anything that is not of the form "^[:xdigit:]{40}[^:xdigit:].*" + If there's not enough characters for an info_hash in the line, skip it. */ + if (linelen < 41) + continue; + + if (!hex_to_hash(info_hash, line)) + continue; + + /* From now on we modify g_accesslist_add and g_accesslist_delete, so prevent the + other worker threads from doing the same */ + pthread_mutex_lock(&g_accesslist_mutex); + + /* If the info hash is in the removing_from list, create a new head without that entry */ + if (*removing_from && (*removing_from)->list) { + ot_hash *exactmatch = bsearch(info_hash, (*removing_from)->list, (*removing_from)->size, OT_HASH_COMPARE_SIZE, vector_compare_hash); + if (exactmatch) { + ptrdiff_t off = exactmatch - (*removing_from)->list; + ot_accesslist *accesslist_new = accesslist_make(*removing_from, (*removing_from)->size - 1); + if (accesslist_new) { + size_t after = (*removing_from)->size - off - 1; + + /* Removing the only entry leaves a list with no room at all, and + memcpy may not be handed that even for nothing */ + if (off) + memcpy(accesslist_new->list, (*removing_from)->list, sizeof(ot_hash) * off); + if (after) + memcpy(accesslist_new->list + off, (*removing_from)->list + off + 1, sizeof(ot_hash) * after); + (*removing_from)->base = time_now_minutes(); + *removing_from = accesslist_new; + } + } + } + + /* Simple case: there's no adding_to list yet, create one with one member */ + if (!*adding_to) { + ot_accesslist *accesslist_new = accesslist_make(NULL, 1); + if (accesslist_new) { + memcpy(accesslist_new->list, info_hash, sizeof(ot_hash)); + *adding_to = accesslist_new; + } + } else { + int exactmatch = 0; + ot_hash *insert_point = vector_binary_search(info_hash, (*adding_to)->list, (*adding_to)->size, sizeof(ot_hash), OT_HASH_COMPARE_SIZE, &exactmatch); + + /* Only if the info hash is not in the adding_to list, create a new head with that entry */ + if (!exactmatch) { + ot_accesslist *accesslist_new = accesslist_make(*adding_to, (*adding_to)->size + 1); + ptrdiff_t off = insert_point - (*adding_to)->list; + if (accesslist_new) { + memcpy(accesslist_new->list, (*adding_to)->list, sizeof(ot_hash) * off); + memcpy(accesslist_new->list + off, info_hash, sizeof(info_hash)); + memcpy(accesslist_new->list + off + 1, (*adding_to)->list + off, sizeof(ot_hash) * ((*adding_to)->size - off)); + (*adding_to)->base = time_now_minutes(); + *adding_to = accesslist_new; + } + } + } + + pthread_mutex_unlock(&g_accesslist_mutex); + } + + free(line); + fclose(fifo); + } + return NULL; +} + +static void *accesslist_adder_worker(void *args) { + (void)args; + accesslist_adddel_worker(g_accesslist_pipe_add, &g_accesslist_add, &g_accesslist_delete); + atomic_store(&adder_left, 1); + return NULL; +} +static void *accesslist_deleter_worker(void *args) { + (void)args; + accesslist_adddel_worker(g_accesslist_pipe_delete, &g_accesslist_delete, &g_accesslist_add); + atomic_store(&deleter_left, 1); + return NULL; +} + +/* These sit in a blocking open or read on a FIFO, so they are woken the way + the other workers are, and the signal is repeated because it only + interrupts a call already under way. The pipe name is what says whether + the thread was ever started. */ +static void accesslist_stop_fifo_worker(pthread_t thread, _Atomic int *left, const char *pipe_name) { + if (!pipe_name) + return; + + while (!atomic_load(left)) { + pthread_kill(thread, OT_SIGNAL_WAKE); + usleep(1000); + } + pthread_join(thread, NULL); +} +#endif + +static pthread_t thread_id; +void accesslist_init(void) { + pthread_mutex_init(&g_accesslist_mutex, NULL); + pthread_create(&thread_id, NULL, accesslist_worker, NULL); +#ifdef WANT_DYNAMIC_ACCESSLIST + if (g_accesslist_pipe_add) + pthread_create(&thread_adder_id, NULL, accesslist_adder_worker, NULL); + if (g_accesslist_pipe_delete) + pthread_create(&thread_deleter_id, NULL, accesslist_deleter_worker, NULL); +#endif +} + +void accesslist_deinit(void) { + /* Wake the worker and wait for it to notice that the tracker is stopping. */ + pthread_kill(thread_id, SIGHUP); + pthread_join(thread_id, NULL); + +#ifdef WANT_DYNAMIC_ACCESSLIST + /* Before the lists they are reading are freed below */ + accesslist_stop_fifo_worker(thread_adder_id, &adder_left, g_accesslist_pipe_add); + accesslist_stop_fifo_worker(thread_deleter_id, &deleter_left, g_accesslist_pipe_delete); +#endif + + pthread_mutex_lock(&g_accesslist_mutex); + + g_accesslist = accesslist_free(g_accesslist); + +#ifdef WANT_DYNAMIC_ACCESSLIST + g_accesslist_add = accesslist_free(g_accesslist_add); + g_accesslist_delete = accesslist_free(g_accesslist_delete); +#endif + + pthread_mutex_unlock(&g_accesslist_mutex); + pthread_mutex_destroy(&g_accesslist_mutex); +} + +void accesslist_cleanup(void) { + pthread_mutex_lock(&g_accesslist_mutex); + + accesslist_clean(g_accesslist); +#if WANT_DYNAMIC_ACCESSLIST + accesslist_clean(g_accesslist_add); + accesslist_clean(g_accesslist_delete); +#endif + + pthread_mutex_unlock(&g_accesslist_mutex); +} +#endif + +int accesslist_address_in_net(const ot_ip6 address, const ot_net *net) { + int bits = net->bits, checkbits = (0x7f00 >> (bits & 7)); + int result = memcmp(address, &net->address, bits >> 3); + if (!result && (bits & 7)) + result = (checkbits & address[bits >> 3]) - (checkbits & net->address[bits >> 3]); + return result == 0; +} + +void *accesslist_set_value_for_net(const ot_net *net, ot_vector *vector, const void *value, const size_t member_size) { + size_t i; + int exactmatch; + + /* Caller must have a concept of ot_net in it's member */ + if (member_size < sizeof(ot_net)) + return 0; + + /* Check each net in vector for overlap */ + uint8_t *member = ((uint8_t *)vector->data); + for (i = 0; i < vector->size; ++i) { + if (accesslist_address_in_net(*(ot_ip6 *)member, net) || accesslist_address_in_net(net->address, (ot_net *)member)) + return 0; + member += member_size; + } + + member = vector_find_or_insert(vector, (void *)net, member_size, sizeof(ot_net), &exactmatch); + if (member) { + memcpy(member, net, sizeof(ot_net)); + memcpy(member + sizeof(ot_net), value, member_size - sizeof(ot_net)); + } + + return member; +} + +/* Takes a vector filled with { ot_net net, uint8_t[x] value }; + Returns value associated with the net, or NULL if not found */ +void *accesslist_get_value_for_net(const ot_ip6 address, const ot_vector *vector, const size_t member_size) { + int exactmatch; + /* This binary search will return a pointer to the first non-containing network... */ + ot_net *net = vector_binary_search(address, vector->data, vector->size, member_size, sizeof(ot_ip6), &exactmatch); + if (!net) + return NULL; + /* ... so we'll need to move back one step unless we've exactly hit the first address in network */ + if (!exactmatch && ((void *)net > vector->data)) + --net; + if (!accesslist_address_in_net(address, net)) + return NULL; + return (void *)net; +} + +#ifdef WANT_IP_FROM_PROXY +typedef struct { + ot_net *proxy; + ot_vector networks; +} ot_proxymap; + +static ot_vector g_proxies_list; +static pthread_mutex_t g_proxies_list_mutex = PTHREAD_MUTEX_INITIALIZER; + +int proxylist_add_network(const ot_net *proxy, const ot_net *net) { + ot_proxymap *map; + int exactmatch, result = 1; + pthread_mutex_lock(&g_proxies_list_mutex); + + /* If we have a direct hit, use and extend the vector there */ + map = vector_binary_search(proxy, g_proxies_list.data, g_proxies_list.size, sizeof(ot_proxymap), sizeof(ot_net), &exactmatch); + + if (!map || !exactmatch) { + /* else see, if we've got overlapping networks + and get a new empty vector if not */ + ot_vector empty; + memset(&empty, 0, sizeof(ot_vector)); + map = accesslist_set_value_for_net(proxy, &g_proxies_list, &empty, sizeof(ot_proxymap)); + } + + if (map && accesslist_set_value_for_net(net, &map->networks, NULL, sizeof(ot_net))) + result = 1; + + pthread_mutex_unlock(&g_proxies_list_mutex); + return result; +} + +int proxylist_check_proxy(const ot_ip6 proxy, const ot_ip6 address) { + int result = 0; + ot_proxymap *map; + + pthread_mutex_lock(&g_proxies_list_mutex); + + if ((map = accesslist_get_value_for_net(proxy, &g_proxies_list, sizeof(ot_proxymap)))) + if (!address || accesslist_get_value_for_net(address, &map->networks, sizeof(ot_net))) + result = 1; + + pthread_mutex_unlock(&g_proxies_list_mutex); + return result; +} + +#endif + +static ot_net g_admin_nets[OT_ADMINIP_MAX]; +static ot_permissions g_admin_nets_permissions[OT_ADMINIP_MAX]; +static size_t g_admin_nets_count = 0; + +int accesslist_bless_net(ot_net *net, ot_permissions permissions) { + if (g_admin_nets_count >= OT_ADMINIP_MAX) + return -1; + + memcpy(g_admin_nets + g_admin_nets_count, net, sizeof(ot_net)); + g_admin_nets_permissions[g_admin_nets_count++] = permissions; + +#ifdef _DEBUG + { + char _debug[512]; + int off = snprintf(_debug, sizeof(_debug), "Blessing ip net "); + off += ip_format(_debug + off, net->address); + if (net->bits < 128) { + _debug[off++] = '/'; + if (ip_is_v4mapped(net->address)) + off += snprintf(_debug + off, sizeof(_debug) - off, "%d", net->bits - 96); + else + off += snprintf(_debug + off, sizeof(_debug) - off, "%d", net->bits); + } + + if (permissions & OT_PERMISSION_MAY_STAT) + off += snprintf(_debug + off, 512 - off, " may_fetch_stats"); + if (permissions & OT_PERMISSION_MAY_LIVESYNC) + off += snprintf(_debug + off, 512 - off, " may_sync_live"); + if (permissions & OT_PERMISSION_MAY_FULLSCRAPE) + off += snprintf(_debug + off, 512 - off, " may_fetch_fullscrapes"); + if (permissions & OT_PERMISSION_MAY_PROXY) + off += snprintf(_debug + off, 512 - off, " may_proxy"); + if (!permissions) + off += snprintf(_debug + off, sizeof(_debug) - off, " nothing"); + _debug[off++] = '.'; + _debug[off++] = '\n'; + (void)write(2, _debug, off); + } +#endif + + return 0; +} + +int accesslist_is_blessed(ot_ip6 ip, ot_permissions permissions) { + size_t i; + for (i = 0; i < g_admin_nets_count; ++i) + if (accesslist_address_in_net(ip, g_admin_nets + i) && (g_admin_nets_permissions[i] & permissions)) + return 1; + return 0; +} diff --git a/src/ot_accesslist.h b/src/ot_accesslist.h new file mode 100644 index 0000000..68c88dc --- /dev/null +++ b/src/ot_accesslist.h @@ -0,0 +1,77 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_ACCESSLIST_H__ +#define OT_ACCESSLIST_H__ + +#include "trackerlogic.h" + +#if defined(WANT_ACCESSLIST_BLACK) && defined(WANT_ACCESSLIST_WHITE) +#error WANT_ACCESSLIST_BLACK and WANT_ACCESSLIST_WHITE are exclusive. +#endif + +#if defined(WANT_ACCESSLIST_BLACK) || defined(WANT_ACCESSLIST_WHITE) +#define WANT_ACCESSLIST +/* Starts a thread that rereads the list when sent SIGHUP, and is stopped by + being sent one, so the caller blocks SIGHUP process wide beforehand */ +void accesslist_init(void); +void accesslist_deinit(void); +int accesslist_hashisvalid(const ot_hash hash); +void accesslist_cleanup(void); + +/* Scan buf for up to capacity info_hashes in hexadecimal notation, one per + line, and return how many were written to dest */ +size_t accesslist_parse_hashes(ot_hash *dest, size_t capacity, const char *buf, size_t len); + +extern const char *g_accesslist_filename; +#ifdef WANT_DYNAMIC_ACCESSLIST +extern const char *g_accesslist_pipe_add; +extern const char *g_accesslist_pipe_delete; +#endif + +#else +#ifdef WANT_DYNAMIC_ACCESSLIST +#error WANT_DYNAMIC_ACCESSLIST needs either WANT_ACCESSLIST_BLACK or WANT_ACCESSLIST_WHITE +#endif + +#define accesslist_init(accesslist_filename) +#define accesslist_deinit() +#define accesslist_hashisvalid(hash) 1 +#endif + +/* Test if an address is subset of an ot_net, return value is considered a bool */ +int accesslist_address_in_net(const ot_ip6 address, const ot_net *net); + +/* Store a value into a vector of struct { ot_net net, uint8_t[x] value } member; + returns NULL + if member_size is too small, or + if one of the nets inside the vector are a subnet of _net_, or + if _net_ is a subnet of one of the nets inside the vector, or + if the vector could not be resized + returns pointer to new member in vector for success + member_size can be sizeof(ot_net) to reduce the lookup to a boolean mapping +*/ +void *accesslist_set_value_for_net(const ot_net *net, ot_vector *vector, const void *value, const size_t member_size); + +/* Takes a vector filled with struct { ot_net net, uint8_t[x] value } member; + Returns pointer to _member_ associated with the net, or NULL if not found + member_size can be sizeof(ot_net) to reduce the lookup to a boolean mapping +*/ +void *accesslist_get_value_for_net(const ot_ip6 address, const ot_vector *vector, const size_t member_size); + +#ifdef WANT_IP_FROM_PROXY +int proxylist_add_network(const ot_net *proxy, const ot_net *net); +int proxylist_check_network(const ot_ip6 *proxy, const ot_ip6 address /* can be NULL to only check proxy */); +#endif + +typedef enum { + OT_PERMISSION_MAY_FULLSCRAPE = 0x1, + OT_PERMISSION_MAY_STAT = 0x2, + OT_PERMISSION_MAY_LIVESYNC = 0x4, + OT_PERMISSION_MAY_PROXY = 0x8 +} ot_permissions; + +int accesslist_bless_net(ot_net *net, ot_permissions permissions); +int accesslist_is_blessed(ot_ip6 ip, ot_permissions permissions); + +#endif diff --git a/src/ot_batch.c b/src/ot_batch.c new file mode 100644 index 0000000..e4dda0b --- /dev/null +++ b/src/ot_batch.c @@ -0,0 +1,156 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_batch.h" + +#ifndef IOV_MAX +#define IOV_MAX 1024 +#endif + +#define OT_BATCH_MIN_ENTRIES 16 + +static int batch_room_for(ot_batch *batch, size_t wanted) { + size_t space = batch->space ? batch->space : OT_BATCH_MIN_ENTRIES; + struct iovec *grown; + + if (batch->count + wanted <= batch->space) + return 0; + + /* During a chunked transfer entries are appended while earlier ones are + still going out, so the front of the array frees up as we go. Slide the + rest down before asking for more. */ + if (batch->first && batch->count - batch->first + wanted <= batch->space) { + memmove(batch->entries, batch->entries + batch->first, (batch->count - batch->first) * sizeof(struct iovec)); + batch->count -= batch->first; + batch->first = 0; + return 0; + } + + while (space < batch->count + wanted) + space *= 2; + + if (!(grown = realloc(batch->entries, space * sizeof(struct iovec)))) + return -1; + + batch->entries = grown; + batch->space = space; + return 0; +} + +int batch_add(ot_batch *batch, void *buf, size_t len) { + if (batch_room_for(batch, 1)) { + free(buf); + return -1; + } + + batch->entries[batch->count].iov_base = buf; + batch->entries[batch->count].iov_len = len; + ++batch->count; + return 0; +} + +int batch_add_vector(ot_batch *batch, const struct iovec *entries, size_t count) { + if (batch_room_for(batch, count)) { + while (count--) + free(entries[count].iov_base); + return -1; + } + + memcpy(batch->entries + batch->count, entries, count * sizeof(struct iovec)); + batch->count += count; + return 0; +} + +/* Drop everything the kernel accepted, freeing each buffer as it goes */ +static void batch_consume(ot_batch *batch, size_t written) { + while (written) { + struct iovec *entry = batch->entries + batch->first; + size_t left = entry->iov_len - batch->offset; + + if (written < left) { + batch->offset += written; + return; + } + + free(entry->iov_base); + written -= left; + batch->offset = 0; + ++batch->first; + } + + /* Nothing is outstanding, so the array can start over from the front */ + if (batch->first == batch->count) + batch->first = batch->count = 0; +} + +/* A peer that has gone away must come back as an error rather than as a + signal that kills the tracker, so ask the kernel not to raise one where it + can be asked. macOS cannot, and relies on SIGPIPE being blocked. */ +static ssize_t batch_writev(int s, struct iovec *entries, int count) { +#ifdef MSG_NOSIGNAL + struct msghdr msg; + + memset(&msg, 0, sizeof(msg)); + msg.msg_iov = entries; + msg.msg_iovlen = count; + return sendmsg(s, &msg, MSG_NOSIGNAL); +#else + return writev(s, entries, count); +#endif +} + +ssize_t batch_send(int s, ot_batch *batch) { + size_t entries = batch->count - batch->first; + ssize_t written; + + if (!batch_pending(batch)) + return BATCH_DONE; + + if (entries > IOV_MAX) + entries = IOV_MAX; + + /* The first entry may be half way out already */ + batch->entries[batch->first].iov_base = (char *)batch->entries[batch->first].iov_base + batch->offset; + batch->entries[batch->first].iov_len -= batch->offset; + written = batch_writev(s, batch->entries + batch->first, (int)entries); + batch->entries[batch->first].iov_base = (char *)batch->entries[batch->first].iov_base - batch->offset; + batch->entries[batch->first].iov_len += batch->offset; + + if (written < 0) + return (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) ? BATCH_AGAIN : BATCH_FAILED; + if (!written) + return BATCH_FAILED; + + batch_consume(batch, (size_t)written); + return written; +} + +void batch_reset(ot_batch *batch) { + while (batch->first < batch->count) + free(batch->entries[batch->first++].iov_base); + + free(batch->entries); + batch->entries = NULL; + batch->count = batch->first = batch->offset = batch->space = 0; +} + +/* For the few constants the chunked encoding needs, which the batch frees + along with everything else */ +int batch_add_copy(ot_batch *batch, const void *buf, size_t len) { + void *copy = malloc(len); + + if (!copy) + return -1; + memcpy(copy, buf, len); + return batch_add(batch, copy, len); +} diff --git a/src/ot_batch.h b/src/ot_batch.h new file mode 100644 index 0000000..910bea7 --- /dev/null +++ b/src/ot_batch.h @@ -0,0 +1,46 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_BATCH_H__ +#define OT_BATCH_H__ + +#include +#include +#include + +/* A queue of buffers on their way to one socket. The batch owns every buffer + handed to it and frees each one as soon as it has left, so a client that + reads slowly costs memory only for what it has not received yet. + + A zeroed ot_batch is ready to use. */ +typedef struct { + struct iovec *entries; + size_t space; + size_t count; + size_t first; /* entries before this one are sent and freed */ + size_t offset; /* bytes of entries[first] already sent */ +} ot_batch; + +static inline int batch_pending(const ot_batch *batch) { return batch->first != batch->count; } + +/* What batch_send reports instead of a byte count */ +enum { BATCH_DONE = 0, BATCH_AGAIN = -1, BATCH_FAILED = -2 }; + +/* Queue buf and take ownership of it. Returns 0, or -1 having freed buf. */ +int batch_add(ot_batch *batch, void *buf, size_t len); + +/* Write as much as the socket takes. Returns the number of bytes written, or + BATCH_DONE, BATCH_AGAIN or BATCH_FAILED. */ +ssize_t batch_send(int s, ot_batch *batch); + +/* Queue every entry of a vector, taking ownership of each buffer but not of + the vector itself. Returns 0, or -1 having freed all of them. */ +int batch_add_vector(ot_batch *batch, const struct iovec *entries, size_t count); + +/* Queue a copy of buf, which the batch then owns. Returns 0, or -1. */ +int batch_add_copy(ot_batch *batch, const void *buf, size_t len); + +/* Free whatever has not left yet, along with the batch's own array */ +void batch_reset(ot_batch *batch); + +#endif diff --git a/src/ot_buf.c b/src/ot_buf.c new file mode 100644 index 0000000..ecd3f8d --- /dev/null +++ b/src/ot_buf.c @@ -0,0 +1,39 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include + +/* Opentracker */ +#include "ot_buf.h" + +int buf_cat(ot_buf *buf, const void *src, size_t len) { + char *data; + + if (!len) + return 0; + + if (buf->size + len < buf->size) + return -1; + + if (!(data = realloc(buf->data, buf->size + len))) + return -1; + + buf->data = data; + memcpy(data + buf->size, src, len); + buf->size += len; + + return 0; +} + +void buf_truncate(ot_buf *buf, size_t len) { + if (len < buf->size) + buf->size = len; +} + +void buf_reset(ot_buf *buf) { + free(buf->data); + buf->data = NULL; + buf->size = 0; +} diff --git a/src/ot_buf.h b/src/ot_buf.h new file mode 100644 index 0000000..7fb944e --- /dev/null +++ b/src/ot_buf.h @@ -0,0 +1,25 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_BUF_H__ +#define OT_BUF_H__ + +#include + +/* A growable byte buffer, holding an HTTP request that did not arrive in a + single packet. Requests that arrive whole are served straight from the + worker's input buffer and never come near this. */ +typedef struct { + char *data; + size_t size; +} ot_buf; + +/* Append len bytes from src. Returns 0, or -1 if the memory could not be + had, in which case the buffer keeps whatever it already held. */ +int buf_cat(ot_buf *buf, const void *src, size_t len); + +void buf_truncate(ot_buf *buf, size_t len); + +void buf_reset(ot_buf *buf); + +#endif diff --git a/src/ot_clean.c b/src/ot_clean.c new file mode 100644 index 0000000..f181e83 --- /dev/null +++ b/src/ot_clean.c @@ -0,0 +1,176 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_accesslist.h" +#include "ot_clean.h" +#include "ot_mutex.h" +#include "ot_stats.h" +#include "ot_vector.h" +#include "trackerlogic.h" + +/* Returns amount of removed peers */ +static ssize_t clean_single_bucket(ot_peer *peers, size_t peer_count, size_t peer_size, time_t timedout, size_t *removed_seeders) { + ot_peer *last_peer = peers + peer_count * peer_size, *insert_point; + + /* Two scan modes: unless there is one peer removed, just increase ot_peertime */ + while (peers < last_peer) { + time_t timediff = timedout + OT_PEERTIME(peers, peer_size); + if (timediff >= OT_PEER_TIMEOUT) + break; + OT_PEERTIME(peers, peer_size) = timediff; + peers += peer_size; + } + + /* If we at least remove one peer, we have to copy */ + for (insert_point = peers; peers < last_peer; peers += peer_size) { + time_t timediff = timedout + OT_PEERTIME(peers, peer_size); + + if (timediff < OT_PEER_TIMEOUT) { + OT_PEERTIME(peers, peer_size) = timediff; + memcpy(insert_point, peers, peer_size); + insert_point += peer_size; + } else if (OT_PEERFLAG_D(peers, peer_size) & PEER_FLAG_SEEDING) + (*removed_seeders)++; + } + + return (peers - insert_point) / peer_size; +} + +int clean_single_peer_list(ot_peerlist *peer_list, size_t peer_size) { + ot_vector *peer_vector = &peer_list->peers; + time_t timedout = (time_t)(time_now_minutes() - peer_list->base); + size_t num_buckets = 1, removed_seeders = 0; + + /* No need to clean empty torrent */ + if (!timedout) + return 0; + + /* Torrent has idled out */ + if (timedout > OT_TORRENT_TIMEOUT) + return 1; + + /* Nothing to be cleaned here? Test if torrent is worth keeping */ + if (timedout > OT_PEER_TIMEOUT) { + if (!peer_list->peer_count) + return peer_list->down_count ? 0 : 1; + timedout = OT_PEER_TIMEOUT; + } + + if (OT_PEERLIST_HASBUCKETS(peer_list)) { + num_buckets = peer_vector->size; + peer_vector = (ot_vector *)peer_vector->data; + } + + while (num_buckets--) { + size_t removed_peers = clean_single_bucket(peer_vector->data, peer_vector->size, peer_size, timedout, &removed_seeders); + peer_list->peer_count -= removed_peers; + peer_vector->size -= removed_peers; + if (removed_peers) + vector_fixup_peers(peer_vector, peer_size); + + /* Skip to next bucket, a vector containing peers */ + ++peer_vector; + } + + peer_list->seed_count -= removed_seeders; + + /* See if we need to convert a torrent from simple vector to bucket list */ + if ((peer_list->peer_count > OT_PEER_BUCKET_MINCOUNT) || OT_PEERLIST_HASBUCKETS(peer_list)) + vector_redistribute_buckets(peer_list, peer_size); + + if (peer_list->peer_count) + peer_list->base = time_now_minutes(); + else { + /* When we got here, the last time that torrent + has been touched is OT_PEER_TIMEOUT Minutes before */ + peer_list->base = time_now_minutes() - OT_PEER_TIMEOUT; + } + return 0; +} + +/* Clean a single torrent + return 1 if torrent timed out +*/ +int clean_single_torrent(ot_torrent *torrent) { + return clean_single_peer_list(torrent->peer_list6, OT_PEER_SIZE6) * clean_single_peer_list(torrent->peer_list4, OT_PEER_SIZE4); +} + +static pthread_mutex_t g_clean_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t g_clean_wakeup = PTHREAD_COND_INITIALIZER; + +static void clean_sleep(void) { + struct timespec deadline; + + clock_gettime(CLOCK_REALTIME, &deadline); + deadline.tv_sec += OT_CLEAN_SLEEP / 1000000; + deadline.tv_nsec += (OT_CLEAN_SLEEP % 1000000) * 1000; + if (deadline.tv_nsec >= 1000000000) { + ++deadline.tv_sec; + deadline.tv_nsec -= 1000000000; + } + + /* Both sides hold the mutex, so the flag cannot be cleared and signalled + in the window between testing it and waiting on it */ + pthread_mutex_lock(&g_clean_mutex); + while (g_opentracker_running) + if (pthread_cond_timedwait(&g_clean_wakeup, &g_clean_mutex, &deadline) == ETIMEDOUT) + break; + pthread_mutex_unlock(&g_clean_mutex); +} + +/* Clean up all peers in one bucket, remove timedout pools and torrents */ +static void clean_bucket(size_t bucket) { + ot_vector *torrents_list = mutex_bucket_lock((int)bucket); + size_t toffs; + int delta_torrentcount = 0; + + for (toffs = 0; toffs < torrents_list->size; ++toffs) { + ot_torrent *torrent = ((ot_torrent *)(torrents_list->data)) + toffs; + if (clean_single_torrent(torrent)) { + vector_remove_torrent(torrents_list, torrent); + --delta_torrentcount; + --toffs; + } + } + mutex_bucket_unlock((int)bucket, delta_torrentcount); +} + +static void *clean_worker(void *args) { + (void)args; + while (1) { + size_t bucket = OT_BUCKET_COUNT; + while (bucket) { + size_t todo = bucket < OT_CLEAN_BUCKETS_PER_WAKE ? bucket : OT_CLEAN_BUCKETS_PER_WAKE; + + /* One bucket is locked at a time, however many a wake gets through */ + while (todo--) + clean_bucket(--bucket); + + if (!g_opentracker_running) + return NULL; + clean_sleep(); + } + stats_cleanup(); +#ifdef WANT_ACCESSLIST + accesslist_cleanup(); +#endif + } + return NULL; +} + +static pthread_t thread_id; +void clean_init(void) { pthread_create(&thread_id, NULL, clean_worker, NULL); } + +void clean_deinit(void) { + pthread_mutex_lock(&g_clean_mutex); + pthread_cond_signal(&g_clean_wakeup); + pthread_mutex_unlock(&g_clean_mutex); + pthread_join(thread_id, NULL); +} diff --git a/src/ot_clean.h b/src/ot_clean.h new file mode 100644 index 0000000..0592791 --- /dev/null +++ b/src/ot_clean.h @@ -0,0 +1,28 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_CLEAN_H__ +#define OT_CLEAN_H__ + +#include "trackerlogic.h" + +/* The amount of time a clean cycle should take */ +#define OT_CLEAN_INTERVAL_MINUTES 2 + +/* How often the cleaner may wake. A full sweep takes OT_CLEAN_INTERVAL_MINUTES. + The clean operations per wake is goverend by OT_BUCKET_COUNT_BITS, + set by the Makefile. + At the default of 1024 buckets this works out at one bucket every + 117 ms. */ +#define OT_CLEAN_MAX_WAKES_PER_SECOND 10 + +#define OT_CLEAN_WAKE_BUDGET ((OT_CLEAN_INTERVAL_MINUTES) * 60 * (OT_CLEAN_MAX_WAKES_PER_SECOND)) +#define OT_CLEAN_BUCKETS_PER_WAKE (((OT_BUCKET_COUNT) + (OT_CLEAN_WAKE_BUDGET) - 1) / (OT_CLEAN_WAKE_BUDGET)) +#define OT_CLEAN_WAKES ((OT_BUCKET_COUNT) / (OT_CLEAN_BUCKETS_PER_WAKE)) +#define OT_CLEAN_SLEEP (((OT_CLEAN_INTERVAL_MINUTES) * 60 * 1000000) / (OT_CLEAN_WAKES)) + +void clean_init(void); +void clean_deinit(void); +int clean_single_torrent(ot_torrent *torrent); + +#endif diff --git a/src/ot_config.c b/src/ot_config.c new file mode 100644 index 0000000..95136b5 --- /dev/null +++ b/src/ot_config.c @@ -0,0 +1,586 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_accesslist.h" +#include "ot_event.h" +#include "ot_config.h" +#include "ot_http.h" +#include "ot_ip.h" +#include "ot_livesync.h" +#include "ot_scan.h" +#include "ot_stats.h" +#include "trackerlogic.h" + +const char *g_redirecturl; +const char *g_serverdir; +const char *g_serveruser; +const char *g_statefile; + +#define OT_LISTEN_INHERIT (-1) + +static ot_listen *g_listen; +static size_t g_listen_count; +static int g_listen_shared; +static int g_listen_workers_udp = 1; +static int g_listen_workers_tcp = 0; + +static void panic(const char *routine) __attribute__((noreturn)); +static void panic(const char *routine) { + fprintf(stderr, "%s: %s\n", routine, strerror(errno)); + exit(111); +} + +static void listen_append(const ot_ip6 ip, uint16_t port, PROTO_FLAG proto, int shared, int workers) { + ot_listen *grown = realloc(g_listen, (g_listen_count + 1) * sizeof(ot_listen)); + ot_listen *entry; + + if (!grown) + panic("listen_append"); + g_listen = grown; + + entry = g_listen + g_listen_count++; + memcpy(entry->ip, ip, sizeof(ot_ip6)); + entry->port = port; + entry->proto = proto; + entry->shared = shared; + entry->workers = workers; + entry->sock = -1; +} + +typedef enum { LISTEN_OPTION_UNKNOWN, LISTEN_OPTION_SHARED, LISTEN_OPTION_WORKERS } listen_option_kind; + +typedef struct { + const char *name; + size_t len; + listen_option_kind kind; + int takes_value; +} listen_option_key; + +#define LISTEN_OPTION_KEY(s, k, v) {s, sizeof(s) - 1, k, v} + +static const listen_option_key listen_option_keys[] = { + LISTEN_OPTION_KEY("shared", LISTEN_OPTION_SHARED, 0), + LISTEN_OPTION_KEY("workers", LISTEN_OPTION_WORKERS, 1), +}; +static const size_t listen_option_count = sizeof(listen_option_keys) / sizeof(listen_option_keys[0]); + +static listen_option_kind parse_listen_option(char **p) { + size_t i; + + for (i = 0; i < listen_option_count; ++i) { + const listen_option_key *key = &listen_option_keys[i]; + char delimiter = (*p)[key->len]; + + if (strncmp(*p, key->name, key->len)) + continue; + if (key->takes_value ? delimiter != '=' : (delimiter && !isspace((unsigned char)delimiter))) + continue; + + *p += key->len + (key->takes_value ? 1 : 0); + return key->kind; + } + return LISTEN_OPTION_UNKNOWN; +} + +/* Options trailing an address in a listen statement, in any order */ +static int listen_scan_options(char *p, int *shared, int *workers) { + uint64_t value; + + for (;;) { + while (isspace((unsigned char)*p)) + ++p; + if (!*p) + return 1; + + switch (parse_listen_option(&p)) { + case LISTEN_OPTION_SHARED: + *shared = 1; + break; + case LISTEN_OPTION_WORKERS: { + size_t off = scan_uint(p, &value, UINT_MAX); + + if (!off) + return 0; + *workers = (int)value; + p += off; + break; + } + default: + return 0; + } + } +} + + +static const char *set_config_option(const char **option, char *value) { +#ifdef _DEBUG + fprintf(stderr, "Setting config option: %s\n", value); +#endif + while (isspace(*value)) + ++value; + free((char *)*option); + return *option = strdup(value); +} + +static int scan_ip6_port(const char *src, ot_ip6 ip, uint16_t *port) { + const char *s = src; + uint64_t value; + int off, bracket = 0; + while (isspace(*s)) + ++s; + if (*s == '[') + ++s, ++bracket; /* for v6 style notation */ + if (!(off = ip_parse(s, ip))) + return 0; + s += off; + if (bracket && *s == ']') + ++s; + if (*s == 0 || isspace(*s)) + return s - src; + if (!ip_is_v4mapped(ip)) { + if (*s != ':' && *s != '.') + return 0; + if (!bracket && *(s) == ':') + return 0; + s++; + } else { + if (*(s++) != ':') + return 0; + } + if (!(off = scan_uint(s, &value, 65535))) + return 0; + *port = (uint16_t)value; + return off + s - src; +} + +static int scan_ip6_net(const char *src, ot_net *net) { + const char *s = src; + uint64_t value; + int off; + while (isspace(*s)) + ++s; + if (!(off = ip_parse(s, net->address))) + return 0; + s += off; + if (*s != '/') + net->bits = 128; + else { + s++; + if (!(off = scan_uint(s, &value, 128))) + return 0; + net->bits = (int)value; + if (ip_is_v4mapped(net->address)) + net->bits += 96; + if (net->bits > 128) + return 0; + s += off; + } + return off + s - src; +} + +typedef enum { + CONFIG_UNKNOWN, + CONFIG_TRACKER_ROOTDIR, + CONFIG_TRACKER_USER, + CONFIG_LISTEN_TCP_UDP, + CONFIG_LISTEN_TCP, + CONFIG_LISTEN_UDP, + CONFIG_LISTEN_UDP_WORKERS, + CONFIG_LISTEN_TCP_WORKERS, + CONFIG_LISTEN_SHARED, + CONFIG_ACCESS_WHITELIST, + CONFIG_ACCESS_BLACKLIST, + CONFIG_ACCESS_FIFO_ADD, + CONFIG_ACCESS_FIFO_DELETE, + CONFIG_ACCESS_STATS, + CONFIG_ACCESS_STATS_PATH, + CONFIG_ACCESS_PROXY, + CONFIG_TRACKER_REDIRECT_URL, + CONFIG_LIVESYNC_CLUSTER_NODE_IP, + CONFIG_LIVESYNC_CLUSTER_LISTEN, +} config_kind; + +typedef struct { + const char *name; + size_t len; + config_kind kind; +} config_key; + +#define CONFIG_KEY(s, k) { s, sizeof(s) - 1, k } + +static const config_key config_keys[] = { + CONFIG_KEY("tracker.rootdir", CONFIG_TRACKER_ROOTDIR), + CONFIG_KEY("tracker.user", CONFIG_TRACKER_USER), + CONFIG_KEY("listen.tcp_udp", CONFIG_LISTEN_TCP_UDP), + CONFIG_KEY("listen.tcp", CONFIG_LISTEN_TCP), + CONFIG_KEY("listen.udp", CONFIG_LISTEN_UDP), + CONFIG_KEY("listen.udp.workers", CONFIG_LISTEN_UDP_WORKERS), + CONFIG_KEY("listen.tcp.workers", CONFIG_LISTEN_TCP_WORKERS), + CONFIG_KEY("listen.shared", CONFIG_LISTEN_SHARED), +#ifdef WANT_ACCESSLIST_WHITE + CONFIG_KEY("access.whitelist", CONFIG_ACCESS_WHITELIST), +#elif defined(WANT_ACCESSLIST_BLACK) + CONFIG_KEY("access.blacklist", CONFIG_ACCESS_BLACKLIST), +#endif +#ifdef WANT_DYNAMIC_ACCESSLIST + CONFIG_KEY("access.fifo_add", CONFIG_ACCESS_FIFO_ADD), + CONFIG_KEY("access.fifo_delete", CONFIG_ACCESS_FIFO_DELETE), +#endif +#ifdef WANT_RESTRICT_STATS + CONFIG_KEY("access.stats", CONFIG_ACCESS_STATS), +#endif + CONFIG_KEY("access.stats_path", CONFIG_ACCESS_STATS_PATH), +#ifdef WANT_IP_FROM_PROXY + CONFIG_KEY("access.proxy", CONFIG_ACCESS_PROXY), +#endif + CONFIG_KEY("tracker.redirect_url", CONFIG_TRACKER_REDIRECT_URL), +#ifdef WANT_SYNC_LIVE + CONFIG_KEY("livesync.cluster.node_ip", CONFIG_LIVESYNC_CLUSTER_NODE_IP), + CONFIG_KEY("livesync.cluster.listen", CONFIG_LIVESYNC_CLUSTER_LISTEN), +#endif +}; +static const size_t config_entry_count = sizeof(config_keys) / sizeof(config_keys[0]); + +static config_kind parse_config_key(char **p) { + size_t i; + for (i = 0; i < config_entry_count; ++i) { + const config_key *key = &config_keys[i]; + if (strncmp(*p, key->name, key->len) == 0 && isspace((unsigned char)(*p)[key->len])) { + *p += key->len + 1; + /* Skip white spaces */ + while (isspace((unsigned char)**p)) + ++(*p); + return key->kind; + } + } + return CONFIG_UNKNOWN; +} + +static int parse_configfile(char *config_filename) { + FILE *config_file_handle; + char inbuf[512]; + ot_ip6 tmpip; +#if defined(WANT_RESTRICT_STATS) || defined(WANT_IP_FROM_PROXY) || defined(WANT_SYNC_LIVE) + ot_net tmpnet; +#endif + int bound = 0; + + config_file_handle = fopen(config_filename, "r"); + + if (config_file_handle == NULL) { + fprintf(stderr, "Warning: Can't open config file: %s.", config_filename); + return 0; + } + + while (fgets(inbuf, sizeof(inbuf), config_file_handle)) { + char *p = inbuf, *end; + uint64_t value; + uint16_t tmpport = 6969; + config_kind kind; + + /* Skip white spaces */ + while (isspace((unsigned char)*p)) + ++p; + + /* Ignore comments and empty lines */ + if ((*p == '#') || (*p == '\n') || (*p == 0)) + continue; + + /* consume trailing new lines and spaces */ + end = p + strlen(p); + while (end > p && isspace((unsigned char)end[-1])) + *--end = 0; + + /* Scan for commands */ + kind = parse_config_key(&p); + switch (kind) { + case CONFIG_TRACKER_ROOTDIR: + set_config_option(&g_serverdir, p); + break; + case CONFIG_TRACKER_USER: + set_config_option(&g_serveruser, p); + break; + case CONFIG_LISTEN_TCP_UDP: + case CONFIG_LISTEN_TCP: + case CONFIG_LISTEN_UDP: { + int shared = OT_LISTEN_INHERIT; + int workers = OT_LISTEN_INHERIT; + int off = scan_ip6_port(p, tmpip, &tmpport); + + if (!off || !listen_scan_options(p + off, &shared, &workers)) + goto parse_error; + if (kind != CONFIG_LISTEN_UDP) + listen_append(tmpip, tmpport, FLAG_TCP, shared, workers), ++bound; + if (kind != CONFIG_LISTEN_TCP) + listen_append(tmpip, tmpport, FLAG_UDP, shared, workers), ++bound; + break; + } + case CONFIG_LISTEN_UDP_WORKERS: + if (!scan_uint(p, &value, UINT_MAX)) + goto parse_error; + g_listen_workers_udp = (int)value; + break; + case CONFIG_LISTEN_TCP_WORKERS: + if (!scan_uint(p, &value, UINT_MAX)) + goto parse_error; + g_listen_workers_tcp = (int)value; + break; + case CONFIG_LISTEN_SHARED: + if (!scan_uint(p, &value, 1)) + goto parse_error; + g_listen_shared = (int)value; + break; +#ifdef WANT_ACCESSLIST_WHITE + case CONFIG_ACCESS_WHITELIST: + set_config_option(&g_accesslist_filename, p); + break; +#elif defined(WANT_ACCESSLIST_BLACK) + case CONFIG_ACCESS_BLACKLIST: + set_config_option(&g_accesslist_filename, p); + break; +#endif +#ifdef WANT_DYNAMIC_ACCESSLIST + case CONFIG_ACCESS_FIFO_ADD: + set_config_option(&g_accesslist_pipe_add, p); + break; + case CONFIG_ACCESS_FIFO_DELETE: + set_config_option(&g_accesslist_pipe_delete, p); + break; +#endif +#ifdef WANT_RESTRICT_STATS + case CONFIG_ACCESS_STATS: + if (!scan_ip6_net(p, &tmpnet)) + goto parse_error; + accesslist_bless_net(&tmpnet, OT_PERMISSION_MAY_STAT); + break; +#endif + case CONFIG_ACCESS_STATS_PATH: + set_config_option(&g_stats_path, p); + break; +#ifdef WANT_IP_FROM_PROXY + case CONFIG_ACCESS_PROXY: + if (!scan_ip6_net(p, &tmpnet)) + goto parse_error; + accesslist_bless_net(&tmpnet, OT_PERMISSION_MAY_PROXY); + break; +#endif + case CONFIG_TRACKER_REDIRECT_URL: + set_config_option(&g_redirecturl, p); + break; +#ifdef WANT_SYNC_LIVE + case CONFIG_LIVESYNC_CLUSTER_NODE_IP: + if (!scan_ip6_net(p, &tmpnet)) + goto parse_error; + accesslist_bless_net(&tmpnet, OT_PERMISSION_MAY_LIVESYNC); + break; + case CONFIG_LIVESYNC_CLUSTER_LISTEN: + tmpport = LIVESYNC_PORT; + { + int shared = OT_LISTEN_INHERIT, dummy = OT_LISTEN_INHERIT; + int off = scan_ip6_port(p, tmpip, &tmpport); + + if (!off || !listen_scan_options(p + off, &shared, &dummy)) + goto parse_error; + listen_append(tmpip, tmpport, FLAG_MCA, shared, OT_LISTEN_INHERIT); + } + break; +#endif + default: + fprintf(stderr, "Unhandled line in config file: %s\n", inbuf); + break; + parse_error: + fprintf(stderr, "Parse error in config file: %s\n", inbuf); + } + } + fclose(config_file_handle); + return bound; +} + +static void usage(char *name) { + fprintf(stderr, + "Usage: %s [-i ip] [-p port] [-P port] [-r redirect] [-d dir] [-u user] [-A ip[/bits]] [-f config] [-s livesyncport]" +#ifdef WANT_ACCESSLIST_BLACK + " [-b blacklistfile]" +#elif defined(WANT_ACCESSLIST_WHITE) + " [-w whitelistfile]" +#endif + "\n", + name); +} + +#define HELPLINE(opt, desc) fprintf(stderr, "\t%-10s%s\n", opt, desc) +static void help(char *name) { + usage(name); + + HELPLINE("-f config", "include and execute the config file"); + HELPLINE("-i ip", "specify ip to bind to with next -[pP] (default: any, overrides preceeding ones)"); + HELPLINE("-p port", "do bind to tcp port (default: 6969, you may specify more than one)"); + HELPLINE("-P port", "do bind to udp port (default: 6969, you may specify more than one)"); + HELPLINE("-r redirecturl", "specify url where / should be redirected to (default none)"); + HELPLINE("-d dir", "specify directory to try to chroot to (default: \".\")"); + HELPLINE("-u user", "specify user under whose privileges opentracker should run (default: \"nobody\")"); + HELPLINE("-A ip[/bits]", "bless an ip address or net as admin address (e.g. to allow syncs from this address)"); + HELPLINE("-v", "report version, event mechanism and the feature flags opentracker was built with"); +#ifdef WANT_ACCESSLIST_BLACK + HELPLINE("-b file", "specify blacklist file."); +#elif defined(WANT_ACCESSLIST_WHITE) + HELPLINE("-w file", "specify whitelist file."); +#endif + + fprintf(stderr, "\nExample: ./opentracker -i 127.0.0.1 -p 6969 -P 6969 -f ./opentracker.conf -i 10.1.1.23 -p 2710 -p 80\n"); + fprintf(stderr, " Here -i 127.0.0.1 selects the ip address for the next -p 6969 and -P 6969.\n"); + fprintf(stderr, " If no port is bound from config file or command line, the last address given\n"); + fprintf(stderr, " (or ::1 if none is set) will be used on port 6969.\n"); +} +#undef HELPLINE + + +static void listen_resolve_defaults(void) { + size_t i; + + for (i = 0; i < g_listen_count; ++i) { + ot_listen *entry = g_listen + i; + + if (entry->shared == OT_LISTEN_INHERIT) + entry->shared = g_listen_shared; + if (entry->workers == OT_LISTEN_INHERIT) + entry->workers = entry->proto == FLAG_TCP ? g_listen_workers_tcp : g_listen_workers_udp; + + /* A tcp worker hands a connection it cannot finish to the event loop, + which only one thread may touch where the set is ours rather than the + kernel's */ + if (entry->proto == FLAG_TCP && entry->workers > 1 && !event_can_share()) { + fprintf(stderr, "Warning: Multiple workers impossible on this platform with event mechanism %s chosen.\n", + event_mechanism()); + entry->workers = 1; + } + } +} + +ot_listen *config_listen_entries(size_t *count) { + *count = g_listen_count; + return g_listen; +} + +void config_parse(int argc, char **argv) { + ot_ip6 serverip; + ot_net tmpnet; + int bound = 0, scanon = 1; + uint64_t value; + + memset(serverip, 0, sizeof(ot_ip6)); +#ifdef WANT_V4_ONLY + serverip[10] = serverip[11] = -1; +#endif + + while (scanon) { + switch (getopt(argc, argv, + ":i:p:A:P:d:u:r:s:f:l:v" +#ifdef WANT_ACCESSLIST_BLACK + "b:" +#elif defined(WANT_ACCESSLIST_WHITE) + "w:" +#endif + "h")) { + case -1: + scanon = 0; + break; + case 'i': + if (!ip_parse(optarg, serverip)) { + usage(argv[0]); + exit(1); + } + break; +#ifdef WANT_ACCESSLIST_BLACK + case 'b': + set_config_option(&g_accesslist_filename, optarg); + break; +#elif defined(WANT_ACCESSLIST_WHITE) + case 'w': + set_config_option(&g_accesslist_filename, optarg); + break; +#endif + case 'p': + if (!scan_uint(optarg, &value, 65535)) { + usage(argv[0]); + exit(1); + } + listen_append(serverip, (uint16_t)value, FLAG_TCP, OT_LISTEN_INHERIT, OT_LISTEN_INHERIT); + bound++; + break; + case 'P': + if (!scan_uint(optarg, &value, 65535)) { + usage(argv[0]); + exit(1); + } + listen_append(serverip, (uint16_t)value, FLAG_UDP, OT_LISTEN_INHERIT, OT_LISTEN_INHERIT); + bound++; + break; +#ifdef WANT_SYNC_LIVE + case 's': + if (!scan_uint(optarg, &value, 65535)) { + usage(argv[0]); + exit(1); + } + listen_append(serverip, (uint16_t)value, FLAG_MCA, OT_LISTEN_INHERIT, OT_LISTEN_INHERIT); + break; +#endif + case 'd': + set_config_option(&g_serverdir, optarg); + break; + case 'u': + set_config_option(&g_serveruser, optarg); + break; + case 'r': + set_config_option(&g_redirecturl, optarg); + break; + case 'l': + g_statefile = optarg; + break; + case 'A': + if (!scan_ip6_net(optarg, &tmpnet)) { + usage(argv[0]); + exit(1); + } + accesslist_bless_net(&tmpnet, 0xffff); /* Allow everything for now */ + break; + case 'f': + bound += parse_configfile(optarg); + break; + case 'h': + help(argv[0]); + exit(0); + case 'v': { + char buffer[8192]; + char *cursor = buffer; + + stats_return_tracker_version(&cursor, buffer + sizeof(buffer) - 1); + stats_return_tracker_features(&cursor, buffer + sizeof(buffer) - 1); + *cursor = 0; + fputs(buffer, stderr); + exit(0); + } + default: + case '?': + usage(argv[0]); + exit(1); + } + } + + if (!bound) { + listen_append(serverip, 6969, FLAG_TCP, OT_LISTEN_INHERIT, OT_LISTEN_INHERIT); + listen_append(serverip, 6969, FLAG_UDP, OT_LISTEN_INHERIT, OT_LISTEN_INHERIT); + } + + listen_resolve_defaults(); +} diff --git a/src/ot_config.h b/src/ot_config.h new file mode 100644 index 0000000..79187ee --- /dev/null +++ b/src/ot_config.h @@ -0,0 +1,33 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_CONFIG_H__ +#define OT_CONFIG_H__ + +#include + +#include "trackerlogic.h" + +/* An address opentracker was asked to listen on. Options the statement did + not name have been filled in from the file wide defaults by the time these + are handed out. sock is filled in by whoever binds them. */ +typedef struct { + ot_ip6 ip; + uint16_t port; + PROTO_FLAG proto; + int shared; + int workers; + int64_t sock; +} ot_listen; + +extern const char *g_redirecturl; +extern const char *g_serverdir; +extern const char *g_serveruser; +extern const char *g_statefile; + +/* Reads the command line and every config file it names. Exits on error. */ +void config_parse(int argc, char **argv); + +ot_listen *config_listen_entries(size_t *count); + +#endif diff --git a/src/ot_event.c b/src/ot_event.c new file mode 100644 index 0000000..6062f70 --- /dev/null +++ b/src/ot_event.c @@ -0,0 +1,357 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_event.h" + +/* One caller waits, so the scratch the mechanism needs is kept here rather + than allocated per call. */ + +/* poll works everywhere and is what the BSDs get until kqueue is written. + Defining OT_EVENT_FORCE_POLL picks it on a platform that would otherwise + use something better, which is how both are exercised by the tests. */ +#if defined(OT_EVENT_FORCE_POLL) +#define OT_EVENT_POLL +#elif defined(OT_EVENT_FORCE_KQUEUE) +#define OT_EVENT_KQUEUE +#elif defined(__linux__) +#define OT_EVENT_EPOLL +#elif defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__DragonFly__) || defined(__APPLE__) +#define OT_EVENT_KQUEUE +#else +#define OT_EVENT_POLL +#endif + +#ifdef OT_EVENT_EPOLL +/* ------------------------------------------------------------- epoll */ + +#include + +static int g_epoll = -1; +static struct epoll_event *g_ready; +static int g_ready_space; + +const char *event_mechanism(void) { return "epoll"; } +/* The set lives in the kernel, and epoll_ctl during another thread's + epoll_wait is what it is for */ +int event_can_share(void) { return 1; } + +int event_init(void) { + g_ready = NULL; + g_ready_space = 0; + return (g_epoll = epoll_create1(EPOLL_CLOEXEC)) == -1 ? -1 : 0; +} + +void event_deinit(void) { + if (g_epoll != -1) + close(g_epoll); + free(g_ready); + g_epoll = -1; + g_ready = NULL; + g_ready_space = 0; +} + +static int event_ctl(int op, int fd, int what, uint32_t generation) { + struct epoll_event ev; + + memset(&ev, 0, sizeof(ev)); + ev.events = (what & EVENT_IN ? (uint32_t)EPOLLIN : 0) | (what & EVENT_OUT ? (uint32_t)EPOLLOUT : 0); + /* The generation rides along inside the kernel, so an event says which + connection it belongs to without anybody looking it up */ + ev.data.u64 = ((uint64_t)generation << 32) | (uint32_t)fd; + return epoll_ctl(g_epoll, op, fd, &ev); +} + +int event_add(int fd, int what, uint32_t generation) { return event_ctl(EPOLL_CTL_ADD, fd, what, generation); } +int event_modify(int fd, int what, uint32_t generation) { return event_ctl(EPOLL_CTL_MOD, fd, what, generation); } +int event_del(int fd) { return epoll_ctl(g_epoll, EPOLL_CTL_DEL, fd, NULL); } + +/* A descriptor leaves the interest list when the last reference to its open + file description is closed, and the tracker never duplicates one */ +int event_close(int fd) { return close(fd); } + +int event_wait(ot_event *out, int max, int timeout_ms) { + int count, i; + + if (max > g_ready_space) { + struct epoll_event *grown = realloc(g_ready, (size_t)max * sizeof(struct epoll_event)); + + if (!grown) + return -1; + g_ready = grown; + g_ready_space = max; + } + + /* An interrupted wait is reported as no events rather than retried here, + so that the caller gets to look at whatever the signal was about */ + if ((count = epoll_wait(g_epoll, g_ready, max, timeout_ms)) == -1) + return errno == EINTR ? 0 : -1; + for (i = 0; i < count; ++i) { + uint32_t events = g_ready[i].events; + + out[i].fd = (int)(uint32_t)g_ready[i].data.u64; + out[i].generation = (uint32_t)(g_ready[i].data.u64 >> 32); + /* A hangup or an error is reported as both, so whichever side the caller + was waiting on gets to notice */ + out[i].what = (events & (EPOLLIN | EPOLLHUP | EPOLLERR) ? EVENT_IN : 0) | + (events & (EPOLLOUT | EPOLLHUP | EPOLLERR) ? EVENT_OUT : 0); + } + return count; +} + +#endif /* OT_EVENT_EPOLL */ + +#ifdef OT_EVENT_KQUEUE +/* ------------------------------------------------------------ kqueue */ + +#include +#include +#include + +static int g_kqueue = -1; +static struct kevent *g_ready; +static int g_ready_space; + +const char *event_mechanism(void) { return "kqueue"; } +/* The set lives in the kernel, and kevent may be called from any thread */ +int event_can_share(void) { return 1; } + +int event_init(void) { + g_ready = NULL; + g_ready_space = 0; + return (g_kqueue = kqueue()) == -1 ? -1 : 0; +} + +void event_deinit(void) { + if (g_kqueue != -1) + close(g_kqueue); + free(g_ready); + g_kqueue = -1; + g_ready = NULL; + g_ready_space = 0; +} + +/* kqueue keeps one registration per filter, so reading and writing are two + entries. A change of interest is an add of the one wanted and a delete + of the one not. The generation travels in udata, which the kernel passes + back untouched. */ +static int event_change(int fd, int what, uint32_t generation, int deleting) { + struct kevent change[2], result[2]; + void *udata = (void *)(uintptr_t)(((uint64_t)generation << 32) | (uint32_t)fd); + int succeeded = 0, count, i; + + EV_SET(&change[0], (uintptr_t)fd, EVFILT_READ, + EV_RECEIPT | ((deleting || !(what & EVENT_IN)) ? EV_DELETE : EV_ADD), 0, 0, udata); + EV_SET(&change[1], (uintptr_t)fd, EVFILT_WRITE, + EV_RECEIPT | ((deleting || !(what & EVENT_OUT)) ? EV_DELETE : EV_ADD), 0, 0, udata); + + if ((count = kevent(g_kqueue, change, 2, result, 2, NULL)) == -1) + return -1; + + for (i = 0; i < count && i < 2; ++i) { + if (!result[i].data) { + ++succeeded; + continue; + } + /* Dropping interest we never registered is not a failure */ + if ((change[i].flags & EV_DELETE) && result[i].data == ENOENT) + continue; + errno = (int)result[i].data; + return -1; + } + + /* Nothing at all was registered or removed, so the descriptor is unknown */ + if (!succeeded) { + errno = ENOENT; + return -1; + } + return 0; +} + +int event_add(int fd, int what, uint32_t generation) { return what ? event_change(fd, what, generation, 0) : 0; } +int event_modify(int fd, int what, uint32_t generation) { return event_change(fd, what, generation, 0); } +int event_del(int fd) { return event_change(fd, 0, 0, 1); } + +/* Closing a descriptor removes every kevent that references it */ +int event_close(int fd) { return close(fd); } + +int event_wait(ot_event *out, int max, int timeout_ms) { + struct timespec timeout; + struct timespec *deadline = NULL; + int count, i; + + if (max > g_ready_space) { + struct kevent *grown = realloc(g_ready, (size_t)max * sizeof(struct kevent)); + + if (!grown) + return -1; + g_ready = grown; + g_ready_space = max; + } + + if (timeout_ms >= 0) { + timeout.tv_sec = timeout_ms / 1000; + timeout.tv_nsec = (long)(timeout_ms % 1000) * 1000000; + deadline = &timeout; + } + + /* An interrupted wait is reported as no events rather than retried here, + so that the caller gets to look at whatever the signal was about */ + if ((count = kevent(g_kqueue, NULL, 0, g_ready, max, deadline)) == -1) + return errno == EINTR ? 0 : -1; + for (i = 0; i < count; ++i) { + uint64_t udata = (uint64_t)(uintptr_t)g_ready[i].udata; + int eof = (g_ready[i].flags & (EV_EOF | EV_ERROR)) != 0; + + out[i].fd = (int)(uint32_t)udata; + out[i].generation = (uint32_t)(udata >> 32); + out[i].what = ((g_ready[i].filter == EVFILT_READ || eof) ? EVENT_IN : 0) | + ((g_ready[i].filter == EVFILT_WRITE || eof) ? EVENT_OUT : 0); + } + return count; +} + +#endif /* OT_EVENT_KQUEUE */ + +#ifdef OT_EVENT_POLL +/* -------------------------------------------------------------- poll */ + +#include + +/* poll wants its descriptors packed, so the registered ones live in one + array and a second maps a descriptor number to its place in it. */ +static struct pollfd *g_pfds; +static uint32_t *g_generation; +static int *g_where; +static int g_count, g_space, g_where_space; + +const char *event_mechanism(void) { return "poll"; } +/* The arrays below are ours and nothing guards them */ +int event_can_share(void) { return 0; } + +int event_init(void) { + g_pfds = NULL, g_generation = NULL, g_where = NULL; + g_count = g_space = g_where_space = 0; + return 0; +} + +void event_deinit(void) { + free(g_pfds), free(g_generation), free(g_where); + g_pfds = NULL, g_generation = NULL, g_where = NULL; + g_count = g_space = g_where_space = 0; +} + +static short event_to_poll(int what) { + return (short)((what & EVENT_IN ? POLLIN : 0) | (what & EVENT_OUT ? POLLOUT : 0)); +} + +static int event_room_for(int fd) { + if (fd >= g_where_space) { + int space = g_where_space ? g_where_space : 64; + int *grown; + + while (space <= fd) + space *= 2; + if (!(grown = realloc(g_where, (size_t)space * sizeof(int)))) + return -1; + while (g_where_space < space) + grown[g_where_space++] = -1; + g_where = grown; + } + + if (g_count == g_space) { + int space = g_space ? g_space * 2 : 64; + struct pollfd *pfds = realloc(g_pfds, (size_t)space * sizeof(struct pollfd)); + uint32_t *gens; + + if (!pfds) + return -1; + g_pfds = pfds; + if (!(gens = realloc(g_generation, (size_t)space * sizeof(uint32_t)))) + return -1; + g_generation = gens; + g_space = space; + } + return 0; +} + +int event_add(int fd, int what, uint32_t generation) { + if (fd < 0 || event_room_for(fd) == -1) + return -1; + if (g_where[fd] != -1) { + errno = EEXIST; + return -1; + } + + g_pfds[g_count].fd = fd; + g_pfds[g_count].events = event_to_poll(what); + g_pfds[g_count].revents = 0; + g_generation[g_count] = generation; + g_where[fd] = g_count++; + return 0; +} + +int event_modify(int fd, int what, uint32_t generation) { + if (fd < 0 || fd >= g_where_space || g_where[fd] == -1) { + errno = ENOENT; + return -1; + } + g_pfds[g_where[fd]].events = event_to_poll(what); + g_generation[g_where[fd]] = generation; + return 0; +} + +int event_del(int fd) { + int at; + + if (fd < 0 || fd >= g_where_space || (at = g_where[fd]) == -1) { + errno = ENOENT; + return -1; + } + + /* The last one moves into the gap, so the array stays packed */ + g_pfds[at] = g_pfds[--g_count]; + g_generation[at] = g_generation[g_count]; + g_where[g_pfds[at].fd] = at; + g_where[fd] = -1; + return 0; +} + +/* Nothing here lives in the kernel, so the entry has to come out by hand + before the descriptor goes, or poll reports POLLNVAL for it forever */ +int event_close(int fd) { + event_del(fd); + return close(fd); +} + +int event_wait(ot_event *out, int max, int timeout_ms) { + int ready, found = 0, i; + + /* An interrupted wait is reported as no events rather than retried here, + so that the caller gets to look at whatever the signal was about */ + if ((ready = poll(g_pfds, (nfds_t)g_count, timeout_ms)) == -1) + return errno == EINTR ? 0 : -1; + if (ready <= 0) + return ready; + + for (i = 0; i < g_count && found < max; ++i) { + short revents = g_pfds[i].revents; + + if (!revents) + continue; + out[found].fd = g_pfds[i].fd; + out[found].generation = g_generation[i]; + out[found].what = (revents & (POLLIN | POLLHUP | POLLERR | POLLNVAL) ? EVENT_IN : 0) | + (revents & (POLLOUT | POLLHUP | POLLERR | POLLNVAL) ? EVENT_OUT : 0); + ++found; + } + return found; +} + +#endif /* OT_EVENT_POLL */ diff --git a/src/ot_event.h b/src/ot_event.h new file mode 100644 index 0000000..cb84ca8 --- /dev/null +++ b/src/ot_event.h @@ -0,0 +1,47 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_EVENT_H__ +#define OT_EVENT_H__ + +#include + +/* Which descriptors have become readable or writable, asked of whichever + mechanism the kernel offers. The generation a descriptor was registered + with is handed back with every event, so a caller can tell an event for + the connection it is holding from one for a connection that has since been + closed and had its number reused. */ + +enum { EVENT_IN = 1, EVENT_OUT = 2 }; + +typedef struct { + int fd; + uint32_t generation; + int what; +} ot_event; + +int event_init(void); +void event_deinit(void); + +int event_add(int fd, int what, uint32_t generation); +int event_modify(int fd, int what, uint32_t generation); +int event_del(int fd); + +/* Stop watching a descriptor and close it. Where the kernel drops a closed + descriptor from its set by itself, this is one system call where doing the + two separately would be two. */ +int event_close(int fd); + +/* Waits at most timeout_ms, or indefinitely when that is negative, and fills + out with up to max ready descriptors. Returns how many, or -1. */ +int event_wait(ot_event *out, int max, int timeout_ms); + +/* Whether the registered set may be changed from another thread while this + one is waiting on it. True where the kernel keeps the set, false where + this module keeps it in memory of its own. */ +int event_can_share(void); + +/* Which mechanism was compiled in, for the version string */ +const char *event_mechanism(void); + +#endif diff --git a/src/ot_fdstate.c b/src/ot_fdstate.c new file mode 100644 index 0000000..8e4bd89 --- /dev/null +++ b/src/ot_fdstate.c @@ -0,0 +1,202 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_fdstate.h" + +enum { FD_FREE, FD_IDLE, FD_BUSY, FD_CLOSING }; + +/* Generation and state share one word so that taking the state is also a + check that the generation has not moved */ +#define FD_GEN(w) ((uint32_t)((w) >> 32)) +#define FD_ST(w) ((uint32_t)(((w) >> 24) & 0xff)) +#define FD_WORD(g, s) (((uint64_t)(g) << 32) | ((uint64_t)(s) << 24)) +#define FD_SET_ST(w, s) (((w) & ~(uint64_t)0xff000000) | ((uint64_t)(s) << 24)) + +typedef struct { + _Atomic uint64_t state; + _Atomic(void *) cookie; + _Atomic ot_time expiry; +} ot_fdslot; + +/* A descriptor the kernel can hand out is below the limit it was handed out + under, but the limit can be raised after we sized the table, so an out of + range descriptor is rejected */ +#define FD_SLOTS_MAX (1u << 20) + +static ot_fdslot *g_slots; +static size_t g_slot_count; +/* Descriptors are handed out lowest first, so a sweep only has to walk as + far as the highest one ever seen rather than the whole table. */ +static _Atomic size_t g_slot_highest; + +int fdstate_init(void) { + struct rlimit limit; + size_t count = FD_SLOTS_MAX; + + if (getrlimit(RLIMIT_NOFILE, &limit) == 0 && limit.rlim_cur != RLIM_INFINITY && limit.rlim_cur < FD_SLOTS_MAX) + count = (size_t)limit.rlim_cur; + + g_slots = mmap(NULL, count * sizeof(ot_fdslot), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (g_slots == MAP_FAILED) { + g_slots = NULL; + return -1; + } + + g_slot_count = count; + atomic_store(&g_slot_highest, 0); + return 0; +} + +void fdstate_deinit(void) { + if (g_slots) + munmap(g_slots, g_slot_count * sizeof(ot_fdslot)); + g_slots = NULL; + g_slot_count = 0; + atomic_store(&g_slot_highest, 0); +} + +size_t fdstate_slots(void) { return g_slot_count; } + +static ot_fdslot *fdstate_slot(int fd) { + if (fd < 0 || (size_t)fd >= g_slot_count) + return NULL; + return g_slots + fd; +} + +uint32_t fdstate_open(int fd, void *cookie, ot_time expiry) { + ot_fdslot *slot = fdstate_slot(fd); + uint64_t word; + + if (!slot) + return 0; + + { + size_t highest = atomic_load_explicit(&g_slot_highest, memory_order_relaxed); + + while ((size_t)fd > highest && + !atomic_compare_exchange_weak_explicit(&g_slot_highest, &highest, (size_t)fd, memory_order_relaxed, + memory_order_relaxed)) + ; + } + + word = atomic_load_explicit(&slot->state, memory_order_relaxed); + atomic_store_explicit(&slot->cookie, cookie, memory_order_release); + atomic_store_explicit(&slot->expiry, expiry, memory_order_relaxed); + /* Publishes the two above to whoever claims the slot next */ + atomic_store_explicit(&slot->state, FD_WORD(FD_GEN(word), FD_IDLE), memory_order_release); + + return FD_GEN(word); +} + +uint32_t fdstate_generation(int fd) { + ot_fdslot *slot = fdstate_slot(fd); + + return slot ? FD_GEN(atomic_load_explicit(&slot->state, memory_order_relaxed)) : 0; +} + +int fdstate_claim(int fd, uint32_t generation) { + ot_fdslot *slot = fdstate_slot(fd); + uint64_t word; + + if (!slot) + return 0; + + word = atomic_load_explicit(&slot->state, memory_order_acquire); + for (;;) { + if (FD_GEN(word) != generation || FD_ST(word) != FD_IDLE) + return 0; + if (atomic_compare_exchange_weak_explicit(&slot->state, &word, FD_SET_ST(word, FD_BUSY), memory_order_acquire, + memory_order_relaxed)) + return 1; + /* word now holds what was really there, so the test above runs again */ + } +} + +void fdstate_release(int fd) { + ot_fdslot *slot = fdstate_slot(fd); + uint64_t word; + + if (!slot) + return; + + word = atomic_load_explicit(&slot->state, memory_order_relaxed); + atomic_store_explicit(&slot->state, FD_SET_ST(word, FD_IDLE), memory_order_release); +} + +int fdstate_claim_expired(int fd, ot_time now) { + ot_fdslot *slot = fdstate_slot(fd); + uint64_t word; + ot_time expiry; + + if (!slot) + return 0; + + /* Reading before asking for the slot keeps the common case, a descriptor + that has not expired, off the bus entirely */ + word = atomic_load_explicit(&slot->state, memory_order_relaxed); + if (FD_ST(word) != FD_IDLE) + return 0; + expiry = atomic_load_explicit(&slot->expiry, memory_order_relaxed); + if (!expiry || expiry > now) + return 0; + + /* Nothing can move the expiry while the slot sits at FD_IDLE, so winning + the slot is the last word */ + return atomic_compare_exchange_strong_explicit(&slot->state, &word, FD_SET_ST(word, FD_CLOSING), memory_order_acquire, + memory_order_relaxed); +} + +void fdstate_close(int fd) { + ot_fdslot *slot = fdstate_slot(fd); + uint64_t word; + + if (!slot) + return; + + word = atomic_load_explicit(&slot->state, memory_order_relaxed); + atomic_store_explicit(&slot->cookie, NULL, memory_order_relaxed); + atomic_store_explicit(&slot->expiry, 0, memory_order_relaxed); + atomic_store_explicit(&slot->state, FD_WORD(FD_GEN(word) + 1, FD_FREE), memory_order_release); +} + +int fdstate_next_expired(ot_time now, int *cursor) { + size_t highest = atomic_load_explicit(&g_slot_highest, memory_order_relaxed); + + for (; (size_t)*cursor <= highest; ++*cursor) + if (fdstate_claim_expired(*cursor, now)) + return (*cursor)++; + + return -1; +} + +void *fdstate_cookie(int fd) { + ot_fdslot *slot = fdstate_slot(fd); + + /* Pairs with the release in fdstate_set_cookie and fdstate_open */ + return slot ? atomic_load_explicit(&slot->cookie, memory_order_acquire) : NULL; +} + +void fdstate_set_cookie(int fd, void *cookie) { + ot_fdslot *slot = fdstate_slot(fd); + + /* Publishes what the cookie points at to whoever reads it next, which may + be another thread entirely */ + if (slot) + atomic_store_explicit(&slot->cookie, cookie, memory_order_release); +} + +void fdstate_set_expiry(int fd, ot_time expiry) { + ot_fdslot *slot = fdstate_slot(fd); + + if (slot) + atomic_store_explicit(&slot->expiry, expiry, memory_order_relaxed); +} diff --git a/src/ot_fdstate.h b/src/ot_fdstate.h new file mode 100644 index 0000000..60fe070 --- /dev/null +++ b/src/ot_fdstate.h @@ -0,0 +1,62 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_FDSTATE_H__ +#define OT_FDSTATE_H__ + +#include + +#include "trackerlogic.h" + +/* What the tracker knows about a file descriptor: who owns it, when it + should be given up on, and whether anyone is working on it right now. + + A descriptor number says nothing about identity, since the kernel hands + the same number out again as soon as it is closed. Every slot therefore + carries a generation that is bumped when the descriptor is given up, and + every caller that was told about a descriptor has to say which generation + it was told about. A caller holding a stale generation is turned away + instead of being handed somebody else's connection. + + The table is sized once from RLIMIT_NOFILE and never moves, so looking a + descriptor up needs no synchronisation at all. */ + +int fdstate_init(void); +void fdstate_deinit(void); +size_t fdstate_slots(void); + +/* Take a freshly accepted descriptor into the table and return the + generation to quote when asking for it back. expiry of 0 never expires. */ +uint32_t fdstate_open(int fd, void *cookie, ot_time expiry); + +/* Which generation a descriptor currently carries, to check an event that + was handed out earlier against the connection living there now. */ +uint32_t fdstate_generation(int fd); + +/* Ask for exclusive use. Returns 0 if the descriptor has been recycled since + generation was handed out, or if somebody else is using it. */ +int fdstate_claim(int fd, uint32_t generation); +void fdstate_release(int fd); + +/* The cleaner's side of the same thing: takes the descriptor only if nobody + is working on it and it really has expired. Never waits. */ +int fdstate_claim_expired(int fd, ot_time now); + +/* Give the descriptor up. The generation is bumped here, which has to happen + before close(2) so that the kernel cannot reissue the number while events + quoting the old generation are still in flight. */ +void fdstate_close(int fd); + +/* Walks the table for descriptors that have expired, handing back one at a + time until it returns -1. The cursor belongs to the caller, so several + sweepers can run without sharing anything. */ +int fdstate_next_expired(ot_time now, int *cursor); + +/* Both only valid while the caller holds the descriptor. */ +void *fdstate_cookie(int fd); + +/* Hand the slot a different cookie, for a connection whose state has moved */ +void fdstate_set_cookie(int fd, void *cookie); +void fdstate_set_expiry(int fd, ot_time expiry); + +#endif diff --git a/src/ot_fmt.c b/src/ot_fmt.c new file mode 100644 index 0000000..362c9b5 --- /dev/null +++ b/src/ot_fmt.c @@ -0,0 +1,40 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include + +/* Opentracker */ +#include "ot_fmt.h" + +void fmt_append(char **cursor, const char *end, const char *format, ...) { + va_list arguments; + size_t room; + int written; + + if (*cursor >= end) { + *cursor = (char *)end; + return; + } + + room = (size_t)(end - *cursor); + va_start(arguments, format); + written = vsnprintf(*cursor, room, format, arguments); + va_end(arguments); + + if (written < 0) + return; + *cursor = (size_t)written >= room ? (char *)end : *cursor + written; +} + +void fmt_bytes(char **cursor, const char *end, const void *bytes, size_t length) { + if (*cursor >= end || (size_t)(end - *cursor) < length) { + *cursor = (char *)end; + return; + } + + memcpy(*cursor, bytes, length); + *cursor += length; +} diff --git a/src/ot_fmt.h b/src/ot_fmt.h new file mode 100644 index 0000000..214cbc4 --- /dev/null +++ b/src/ot_fmt.h @@ -0,0 +1,19 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_FMT_H__ +#define OT_FMT_H__ + +#include + +/* Appending to a cursor that walks a fixed buffer. Both take the cursor by + reference and advance it, so a reply is written as a run of statements and + no caller does pointer arithmetic of its own. */ + +void fmt_append(char **cursor, const char *end, const char *format, ...) __attribute__((format(printf, 3, 4))); + +/* Append raw bytes for the binary stretches of a bencoded reply. + Either everything is appended or nothing at all */ +void fmt_bytes(char **cursor, const char *end, const void *bytes, size_t length); + +#endif diff --git a/src/ot_fullscrape.c b/src/ot_fullscrape.c new file mode 100644 index 0000000..9afad92 --- /dev/null +++ b/src/ot_fullscrape.c @@ -0,0 +1,455 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifdef WANT_FULLSCRAPE + +/* System */ +#include +#include +#include +#include +#ifdef WANT_COMPRESSION_GZIP +#include +#endif +#ifdef WANT_COMPRESSION_ZSTD +#include +#endif + +/* Opentracker */ +#include "ot_fmt.h" +#include "ot_hex.h" +#include "ot_fullscrape.h" +#include "ot_iovec.h" +#include "ot_mutex.h" +#include "ot_workqueue.h" +#include "ot_urlencoded.h" +#include "trackerlogic.h" + +/* Fetch full scrape info for all torrents + Full scrapes usually are huge and one does not want to + allocate more memory. So lets get them in 512k units +*/ +#define OT_SCRAPE_CHUNK_SIZE (1024 * 1024) + +/* "d8:completei%zde10:downloadedi%zde10:incompletei%zdee" */ +#define OT_SCRAPE_MAXENTRYLEN 256 + +/* Forward declaration */ +static void fullscrape_make(int taskid, ot_tasktype mode); +#ifdef WANT_COMPRESSION_GZIP +static void fullscrape_make_gzip(int taskid, ot_tasktype mode); +#endif +#ifdef WANT_COMPRESSION_ZSTD +static void fullscrape_make_zstd(int taskid, ot_tasktype mode); +#endif + +/* Converter function from memory to human readable hex strings + XXX - Duplicated from ot_stats. Needs fix. */ +/* This is the entry point into this worker thread + It grabs tasks from mutex_tasklist and delivers results back +*/ +static void *fullscrape_worker(void *args) { + (void)args; + + while (g_opentracker_running) { + ot_tasktype tasktype = TASK_FULLSCRAPE; + ot_taskid taskid = workqueue_poptask(&tasktype); + if (!taskid) + break; +#ifdef WANT_COMPRESSION_ZSTD + if (tasktype & TASK_FLAG_ZSTD) + fullscrape_make_zstd(taskid, tasktype); + else +#endif +#ifdef WANT_COMPRESSION_GZIP + if (tasktype & TASK_FLAG_GZIP) + fullscrape_make_gzip(taskid, tasktype); + else +#endif + fullscrape_make(taskid, tasktype); + workqueue_pushchunked(taskid, NULL); + } + return NULL; +} + +static pthread_t thread_id; +void fullscrape_init(void) { + pthread_create( &thread_id, NULL, fullscrape_worker, NULL ); +} + +void fullscrape_deinit(void) { + workqueue_wakeall(); + pthread_join( thread_id, NULL ); +} + +void fullscrape_deliver( int64_t sock, ot_tasktype tasktype ) { + workqueue_pushtask( sock, TASK_FULLSCRAPE | tasktype ); +} + +static void fullscrape_write_one( ot_tasktype mode, char **cursor, ot_torrent *torrent, ot_hash *hash ) { + /* Callers guarantee this much room: the chunk writer keeps its low + watermark that far from the end, the compressors hand over a buffer of + exactly that size */ + char *r = *cursor; + const char *r_end = r + OT_SCRAPE_MAXENTRYLEN; + size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; + size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; + size_t down_count = torrent->peer_list6->down_count + torrent->peer_list4->down_count; + + switch (mode & TASK_TASK_MASK) { + case TASK_FULLSCRAPE: + default: + /* push hash as bencoded string */ + *r++ = '2'; + *r++ = '0'; + *r++ = ':'; + memcpy(r, hash, sizeof(ot_hash)); + r += sizeof(ot_hash); + /* push rest of the scrape string */ + fmt_append(&r, r_end, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", seed_count, down_count, peer_count - seed_count); + + break; + case TASK_FULLSCRAPE_TPB_ASCII: + hex_from_hash(r, *hash); + r += 2 * sizeof(ot_hash); + fmt_append(&r, r_end, ":%zd:%zd\n", seed_count, peer_count - seed_count); + break; + case TASK_FULLSCRAPE_TPB_ASCII_PLUS: + hex_from_hash(r, *hash); + r += 2 * sizeof(ot_hash); + fmt_append(&r, r_end, ":%zd:%zd:%zd\n", seed_count, peer_count - seed_count, down_count); + break; + case TASK_FULLSCRAPE_TPB_BINARY: + memcpy(r, *hash, sizeof(ot_hash)); + r += sizeof(ot_hash); + *(uint32_t *)(r + 0) = htonl((uint32_t)seed_count); + *(uint32_t *)(r + 4) = htonl((uint32_t)(peer_count - seed_count)); + r += 8; + break; + case TASK_FULLSCRAPE_TPB_URLENCODED: + r += urlencoded_format(r, *hash, sizeof(ot_hash)); + fmt_append(&r, r_end, ":%zd:%zd\n", seed_count, peer_count - seed_count); + break; + case TASK_FULLSCRAPE_TRACKERSTATE: + hex_from_hash(r, *hash); + r += 2 * sizeof(ot_hash); + fmt_append(&r, r_end, ":%zd:%zd\n", torrent->peer_list6->base, down_count); + break; + } + *cursor = r; +} + +static void fullscrape_make(int taskid, ot_tasktype mode) { + size_t bucket; + char *r, *re; + struct iovec iovector = {NULL, 0}; + + /* Setup return vector... */ + r = iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); + if (!r) + return; + + /* re points to low watermark */ + re = r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN; + + if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) + fmt_append(&r, (char *)iovector.iov_base + OT_SCRAPE_CHUNK_SIZE, "d5:filesd"); + + /* For each bucket... */ + for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { + /* Get exclusive access to that bucket */ + ot_vector *torrents_list = mutex_bucket_lock(bucket); + ot_torrent *torrents = (ot_torrent *)(torrents_list->data); + size_t i; + + /* For each torrent in this bucket.. */ + for (i = 0; i < torrents_list->size; ++i) { + fullscrape_write_one(mode, &r, torrents + i, &torrents[i].hash); + + if (r > re) { + iovector.iov_len = r - (char *)iovector.iov_base; + + if (workqueue_pushchunked(taskid, &iovector)) { + free(iovector.iov_base); + mutex_bucket_unlock(bucket, 0); + return; + } + /* Allocate a fresh output buffer */ + r = iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); + if (!r) { + mutex_bucket_unlock(bucket, 0); + return; + } + + /* re points to low watermark */ + re = r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN; + } + } + + /* All torrents done: release lock on current bucket */ + mutex_bucket_unlock(bucket, 0); + + /* Parent thread died? */ + if (!g_opentracker_running) + return; + } + + if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) + fmt_append(&r, (char *)iovector.iov_base + OT_SCRAPE_CHUNK_SIZE, "ee"); + + /* Send rest of data */ + iovector.iov_len = r - (char *)iovector.iov_base; + if (workqueue_pushchunked(taskid, &iovector)) + free(iovector.iov_base); +} + +#ifdef WANT_COMPRESSION_GZIP + +static void fullscrape_make_gzip(int taskid, ot_tasktype mode) { + size_t bucket; + char *r; + struct iovec iovector = {NULL, 0}; + int zres; + z_stream strm; + /* Setup return vector... */ + iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); + if (!iovector.iov_base) + return; + + memset(&strm, 0, sizeof(strm)); + strm.next_out = (uint8_t *)iovector.iov_base; + strm.avail_out = OT_SCRAPE_CHUNK_SIZE; + if (deflateInit2(&strm, 7, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY) != Z_OK) + fprintf(stderr, "not ok.\n"); + + if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) { + strm.next_in = (uint8_t *)"d5:filesd"; + strm.avail_in = strlen("d5:filesd"); + zres = deflate(&strm, Z_NO_FLUSH); + } + + /* For each bucket... */ + for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { + /* Get exclusive access to that bucket */ + ot_vector *torrents_list = mutex_bucket_lock(bucket); + ot_torrent *torrents = (ot_torrent *)(torrents_list->data); + size_t i; + + /* For each torrent in this bucket.. */ + for (i = 0; i < torrents_list->size; ++i) { + char compress_buffer[OT_SCRAPE_MAXENTRYLEN]; + r = compress_buffer; + fullscrape_write_one(mode, &r, torrents + i, &torrents[i].hash); + strm.next_in = (uint8_t *)compress_buffer; + strm.avail_in = r - compress_buffer; + zres = deflate(&strm, Z_NO_FLUSH); + if ((zres < Z_OK) && (zres != Z_BUF_ERROR)) + fprintf(stderr, "deflate() failed while in fullscrape_make().\n"); + + /* Check if there still is enough buffer left */ + while (!strm.avail_out) { + iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base; + + if (workqueue_pushchunked(taskid, &iovector)) { + free(iovector.iov_base); + mutex_bucket_unlock(bucket, 0); + return; + } + /* Allocate a fresh output buffer */ + iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); + if (!iovector.iov_base) { + fprintf(stderr, "Out of memory trying to claim ouput buffer\n"); + deflateEnd(&strm); + mutex_bucket_unlock(bucket, 0); + return; + } + strm.next_out = (uint8_t *)iovector.iov_base; + strm.avail_out = OT_SCRAPE_CHUNK_SIZE; + zres = deflate(&strm, Z_NO_FLUSH); + if ((zres < Z_OK) && (zres != Z_BUF_ERROR)) + fprintf(stderr, "deflate() failed while in fullscrape_make().\n"); + } + } + + /* All torrents done: release lock on current bucket */ + mutex_bucket_unlock(bucket, 0); + + /* Parent thread died? */ + if (!g_opentracker_running) { + deflateEnd(&strm); + return; + } + } + + if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) { + strm.next_in = (uint8_t *)"ee"; + strm.avail_in = strlen("ee"); + } + + if (deflate(&strm, Z_FINISH) < Z_OK) + fprintf(stderr, "deflate() failed while in fullscrape_make()'s endgame.\n"); + + iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base; + if (workqueue_pushchunked(taskid, &iovector)) { + free(iovector.iov_base); + deflateEnd(&strm); + return; + } + + /* Check if there's a last batch of data in the zlib buffer */ + if (!strm.avail_out) { + /* Allocate a fresh output buffer */ + iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); + + if (!iovector.iov_base) { + fprintf(stderr, "Problem with iovec_next_chunk\n"); + deflateEnd(&strm); + return; + } + strm.next_out = iovector.iov_base; + strm.avail_out = OT_SCRAPE_CHUNK_SIZE; + if (deflate(&strm, Z_FINISH) < Z_OK) + fprintf(stderr, "deflate() failed while in fullscrape_make()'s endgame.\n"); + + /* Only pass the new buffer if there actually was some data left in the buffer */ + iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base; + if (!iovector.iov_len || workqueue_pushchunked(taskid, &iovector)) + free(iovector.iov_base); + } + + deflateEnd(&strm); +} +/* WANT_COMPRESSION_GZIP */ +#endif + +#ifdef WANT_COMPRESSION_ZSTD + +static void fullscrape_make_zstd(int taskid, ot_tasktype mode) { + int bucket; + char *r; + struct iovec iovector = {NULL, 0}; + ZSTD_CCtx *zstream = ZSTD_createCCtx(); + ZSTD_inBuffer inbuf; + ZSTD_outBuffer outbuf; + size_t more_bytes; + + if (!zstream) + return; + + /* Setup return vector... */ + iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); + if (!iovector.iov_base) { + ZSTD_freeCCtx(zstream); + return; + } + + /* Working with a compression level 6 is half as fast as level 3, but + seems to be the last reasonable bump that's worth extra cpu */ + ZSTD_CCtx_setParameter(zstream, ZSTD_c_compressionLevel, 6); + + outbuf.dst = iovector.iov_base; + outbuf.size = OT_SCRAPE_CHUNK_SIZE; + outbuf.pos = 0; + + if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) { + inbuf.src = (const void *)"d5:filesd"; + inbuf.size = strlen("d5:filesd"); + inbuf.pos = 0; + ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue); + } + + /* For each bucket... */ + for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { + /* Get exclusive access to that bucket */ + ot_vector *torrents_list = mutex_bucket_lock(bucket); + ot_torrent *torrents = (ot_torrent *)(torrents_list->data); + size_t i; + + /* For each torrent in this bucket.. */ + for (i = 0; i < torrents_list->size; ++i) { + char compress_buffer[OT_SCRAPE_MAXENTRYLEN]; + r = compress_buffer; + fullscrape_write_one(mode, &r, torrents + i, &torrents[i].hash); + inbuf.src = compress_buffer; + inbuf.size = r - compress_buffer; + inbuf.pos = 0; + ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue); + + /* Check if there still is enough buffer left */ + while (outbuf.pos + OT_SCRAPE_MAXENTRYLEN > outbuf.size) { + iovector.iov_len = outbuf.size; + + if (workqueue_pushchunked(taskid, &iovector)) { + free(iovector.iov_base); + ZSTD_freeCCtx(zstream); + return mutex_bucket_unlock(bucket, 0); + } + /* Allocate a fresh output buffer */ + iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); + if (!iovector.iov_base) { + fprintf(stderr, "Out of memory trying to claim ouput buffer\n"); + ZSTD_freeCCtx(zstream); + return mutex_bucket_unlock(bucket, 0); + } + + outbuf.dst = iovector.iov_base; + outbuf.size = OT_SCRAPE_CHUNK_SIZE; + outbuf.pos = 0; + + ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue); + } + } + + /* All torrents done: release lock on current bucket */ + mutex_bucket_unlock(bucket, 0); + + /* Parent thread died? */ + if (!g_opentracker_running) + return; + } + + if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) { + inbuf.src = (const void *)"ee"; + inbuf.size = strlen("ee"); + inbuf.pos = 0; + } + + more_bytes = ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_end); + + iovector.iov_len = outbuf.pos; + if (workqueue_pushchunked(taskid, &iovector)) { + free(iovector.iov_base); + ZSTD_freeCCtx(zstream); + return; + } + + /* Check if there's a last batch of data in the zlib buffer */ + if (more_bytes) { + /* Allocate a fresh output buffer */ + iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); + + if (!iovector.iov_base) { + fprintf(stderr, "Problem with iovec_next_chunk\n"); + ZSTD_freeCCtx(zstream); + return; + } + + outbuf.dst = iovector.iov_base; + outbuf.size = OT_SCRAPE_CHUNK_SIZE; + outbuf.pos = 0; + + ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_end); + + /* Only pass the new buffer if there actually was some data left in the buffer */ + iovector.iov_len = outbuf.pos; + if (!iovector.iov_len || workqueue_pushchunked(taskid, &iovector)) + free(iovector.iov_base); + } + + ZSTD_freeCCtx(zstream); +} +/* WANT_COMPRESSION_ZSTD */ +#endif + +/* WANT_FULLSCRAPE */ +#endif diff --git a/src/ot_fullscrape.h b/src/ot_fullscrape.h new file mode 100644 index 0000000..8d11cd3 --- /dev/null +++ b/src/ot_fullscrape.h @@ -0,0 +1,23 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_FULLSCRAPE_H__ +#define OT_FULLSCRAPE_H__ + +#ifdef WANT_FULLSCRAPE + +#include "ot_mutex.h" +#include "ot_workqueue.h" + +void fullscrape_init(void); +void fullscrape_deinit(void); +void fullscrape_deliver(int64_t sock, ot_tasktype tasktype); + +#else + +#define fullscrape_init() +#define fullscrape_deinit() + +#endif + +#endif diff --git a/src/ot_hex.c b/src/ot_hex.c new file mode 100644 index 0000000..b537094 --- /dev/null +++ b/src/ot_hex.c @@ -0,0 +1,56 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include + +/* Opentracker */ +#include "ot_hex.h" + +int hex_value(int c) { + unsigned char u = (unsigned char)c; + + if (u >= '0' && u <= '9') + return u - '0'; + u |= 0x20; /* fold upper case onto lower, digits are already handled */ + if (u >= 'a' && u <= 'f') + return u - 'a' + 10; + return -1; +} + +int hex_isdigit(int c) { return hex_value(c) >= 0; } + +char *hex_from_hash(char *dest, const uint8_t *hash) { + static const char digits[] = "0123456789ABCDEF"; + char *d = dest; + size_t i; + + for (i = 0; i < sizeof(ot_hash); ++i) { + *d++ = digits[hash[i] >> 4]; + *d++ = digits[hash[i] & 15]; + } + *d = 0; + return dest; +} + +int hex_to_hash(ot_hash hash, const char *src) { + uint8_t decoded[sizeof(ot_hash)]; + size_t i; + + for (i = 0; i < sizeof(ot_hash); ++i) { + int high, low; + + /* Tested one character at a time so that a string ending early stops us + at its terminator rather than one byte past it */ + if ((high = hex_value((unsigned char)src[2 * i])) < 0) + return 0; + if ((low = hex_value((unsigned char)src[2 * i + 1])) < 0) + return 0; + + decoded[i] = (uint8_t)((high << 4) | low); + } + + memcpy(hash, decoded, sizeof(decoded)); + return 1; +} diff --git a/src/ot_hex.h b/src/ot_hex.h new file mode 100644 index 0000000..ddbbb44 --- /dev/null +++ b/src/ot_hex.h @@ -0,0 +1,27 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_HEX_H__ +#define OT_HEX_H__ + +#include + +#include "trackerlogic.h" + +/* Write hash as forty upper case hex digits followed by a terminating zero. + dest must have room for forty one bytes. Returns dest. */ +char *hex_from_hash(char *dest, const uint8_t *hash); + +/* Read forty hex digits from src into hash. Returns non-zero on success. + On failure hash is left untouched and no byte of src beyond the offending + character is read, so src need not be forty characters long. */ +int hex_to_hash(ot_hash hash, const char *src); + +/* Non-zero if c is a hexadecimal digit, in either case. */ +int hex_isdigit(int c); + +/* The value of a hexadecimal digit, or -1 if c is not one. Shared with the + address scanner so that both agree on what a digit is. */ +int hex_value(int c); + +#endif diff --git a/src/ot_http.c b/src/ot_http.c new file mode 100644 index 0000000..96b8f4e --- /dev/null +++ b/src/ot_http.c @@ -0,0 +1,810 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_accesslist.h" +#include "ot_fullscrape.h" +#include "ot_http.h" +#include "ot_iovec.h" +#include "ot_ip.h" +#include "ot_mutex.h" +#include "ot_workqueue.h" +#include "ot_scan.h" +#include "ot_event.h" +#include "ot_fdstate.h" +#include "ot_vector.h" +#include "ot_fmt.h" +#include "ot_stats.h" +#include "ot_urlencoded.h" +#include "trackerlogic.h" + +#define OT_MAXMULTISCRAPE_COUNT 64 +extern const char *g_redirecturl; + +const char *g_stats_path; +ssize_t g_stats_path_len; + +enum { SUCCESS_HTTP_HEADER_LENGTH = 80, SUCCESS_HTTP_SIZE_OFF = 17 }; + +struct http_data *http_adopt(const int64_t sock) { + struct http_data *cookie = fdstate_cookie(sock), *heap; + int interest; + + if (!cookie || !(cookie->flag & STRUCT_HTTP_FLAG_ONSTACK)) + return cookie; + + if (!(heap = malloc(sizeof(*heap)))) + return NULL; + + *heap = *cookie; + heap->flag &= ~STRUCT_HTTP_FLAG_ONSTACK; + + /* Publishing the cookie is what lets another thread reach this connection, + so nothing may touch it afterwards, not even to read what it asked for */ + interest = heap->interest; + fdstate_set_cookie((int)sock, heap); + + if (event_add((int)sock, interest, fdstate_generation((int)sock)) == -1) { + fdstate_set_cookie((int)sock, cookie); + free(heap); + return NULL; + } + return heap; +} + +/* Give up a connection: the one place that knows everything hanging off it. */ +void http_close(const int64_t sock) { + struct http_data *cookie = fdstate_cookie(sock); + + fdstate_close((int)sock); + + if (cookie) { + if (cookie->batch) + batch_reset(cookie->batch); + free(cookie->batch); + buf_reset(&cookie->request); + if (cookie->flag & (STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER)) + workqueue_canceltask(sock); + if (!(cookie->flag & STRUCT_HTTP_FLAG_ONSTACK)) + free(cookie); + } + + event_close((int)sock); +} + +static void http_senddata(const int64_t sock, struct ot_workstruct *ws) { + struct http_data *cookie = fdstate_cookie(sock); + ssize_t written_size; + + if (!cookie) { + http_close(sock); + return; + } + + /* whoever sends data is not interested in its input buffer */ + if (ws->keep_alive && ws->header_size != ws->request_size) { + size_t rest = ws->request_size - ws->header_size; + if (cookie->request.data) { + memmove(cookie->request.data, ws->request + ws->header_size, rest); + buf_truncate(&cookie->request, rest); + } else if (buf_cat(&cookie->request, ws->request + ws->header_size, rest)) { + /* Without room for what the client pipelined behind this request, answer + this one and let the connection close, rather than keeping it open + around a request we have silently dropped */ + buf_reset(&cookie->request); + ws->keep_alive = 0; + } + } else + buf_reset(&cookie->request); + + written_size = write(sock, ws->reply, ws->reply_size); + /* A send buffer with no room in it is not a failure: nothing went out, so + the whole reply goes into the batch below and leaves when the socket + drains. Only a real error ends the connection here. */ + if (written_size < 0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) + written_size = 0; + + if ((written_size < 0) || ((written_size == ws->reply_size) && !ws->keep_alive)) { + http_close(sock); + return; + } + + if (written_size < ws->reply_size) { + char *outbuf; + + if (!(outbuf = malloc(ws->reply_size - written_size))) { + http_close(sock); + return; + } + + memcpy(outbuf, ws->reply + written_size, ws->reply_size - written_size); + if (!cookie->batch) { + cookie->batch = calloc(1, sizeof(ot_batch)); + if (!cookie->batch) { + free(outbuf); + http_close(sock); + return; + } + } + + batch_add(cookie->batch, outbuf, ws->reply_size - written_size); + + /* writeable short data sockets just have a tcp timeout */ + if (!ws->keep_alive) + fdstate_set_expiry(sock, 0); + http_interest_set(sock, cookie, EVENT_OUT); + } +} + +#define HTTPERROR_302 return http_issue_error(sock, ws, CODE_HTTPERROR_302) +#define HTTPERROR_400 return http_issue_error(sock, ws, CODE_HTTPERROR_400) +#define HTTPERROR_400_PARAM return http_issue_error(sock, ws, CODE_HTTPERROR_400_PARAM) +#define HTTPERROR_400_COMPACT return http_issue_error(sock, ws, CODE_HTTPERROR_400_COMPACT) +#define HTTPERROR_400_DOUBLEHASH return http_issue_error(sock, ws, CODE_HTTPERROR_400_PARAM) +#define HTTPERROR_402_NOTMODEST return http_issue_error(sock, ws, CODE_HTTPERROR_402_NOTMODEST) +#define HTTPERROR_403_IP return http_issue_error(sock, ws, CODE_HTTPERROR_403_IP) +#define HTTPERROR_404 return http_issue_error(sock, ws, CODE_HTTPERROR_404) +#define HTTPERROR_500 return http_issue_error(sock, ws, CODE_HTTPERROR_500) +/* A connection is either being read or being written, never both, so what it + waits for is set outright rather than added to and taken away again. The + event layer only hears about it when the answer changes. */ +void http_interest_set(const int64_t sock, struct http_data *cookie, int what) { + if (cookie->interest == what) + return; + cookie->interest = what; + /* Still on a stack, so not in the event loop yet: whoever adopts this + connection registers whatever it asked for last */ + if (cookie->flag & STRUCT_HTTP_FLAG_ONSTACK) + return; + event_modify((int)sock, what, fdstate_generation((int)sock)); +} + +ssize_t http_issue_error(const int64_t sock, struct ot_workstruct *ws, int code) { + const char *error_code[] = {"302 Found", "400 Invalid Request", "400 Invalid Request", "400 Invalid Request", "402 Payment Required", + "403 Not Modest", "403 Access Denied", "404 Not Found", "500 Internal Server Error"}; + const char *title = error_code[code]; + + ws->reply = ws->outbuf; + if (code == CODE_HTTPERROR_302) + ws->reply_size = snprintf(ws->reply, G_OUTBUF_SIZE, "HTTP/1.0 302 Found\r\nContent-Length: 0\r\nLocation: %s\r\n\r\n", g_redirecturl); + else + ws->reply_size = snprintf(ws->reply, G_OUTBUF_SIZE, "HTTP/1.0 %s\r\nContent-Type: text/html\r\nContent-Length: %zd\r\n\r\n%s\n", title, + strlen(title) + 16 - 4, title + 4); + +#ifdef _DEBUG_HTTPERROR + fprintf(stderr, "DEBUG: invalid request was: %s\n", ws->debugbuf); +#endif + stats_issue_event(ws, EVENT_FAILED, FLAG_TCP, code); + http_senddata(sock, ws); + return ws->reply_size = -2; +} + +ssize_t http_sendiovecdata(const int64_t sock, struct ot_workstruct *ws, size_t iovec_entries, struct iovec *iovector, int is_partial) { + struct http_data *cookie = fdstate_cookie(sock); + char *header; + const char *encoding = ""; + size_t header_size, size = iovec_length(&iovec_entries, (const struct iovec **)&iovector); + + /* No cookie? Bad socket. Leave. */ + if (!cookie) { + iovec_free(&iovec_entries, &iovector); + HTTPERROR_500; + } + + /* Descriptor numbers get recycled, so a result can only belong to this + connection if this connection is in fact waiting for one. */ + if (!(cookie->flag & (STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER))) { + iovec_free(&iovec_entries, &iovector); + return 0; + } + + /* If this socket collected request in a buffer, free it now */ + buf_reset(&cookie->request); + + /* If we came here, wait for the answer is over */ + if (cookie->flag & STRUCT_HTTP_FLAG_WAITINGFORTASK) { + http_interest_set(sock, cookie, 0); + cookie->flag &= ~STRUCT_HTTP_FLAG_WAITINGFORTASK; + } + + if (iovec_entries) { + + if (cookie->flag & STRUCT_HTTP_FLAG_ZSTD) + encoding = "Content-Encoding: zstd\r\n"; + else if (cookie->flag & STRUCT_HTTP_FLAG_GZIP) + encoding = "Content-Encoding: gzip\r\n"; + else if (cookie->flag & STRUCT_HTTP_FLAG_BZIP2) + encoding = "Content-Encoding: bzip2\r\n"; + + if (!(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED)) + header_size = asprintf(&header, "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n%sContent-Length: %zd\r\n\r\n", encoding, size); + else { + if (!(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER)) { + header_size = + asprintf(&header, "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n%sTransfer-Encoding: chunked\r\n\r\n%zx\r\n", encoding, size); + cookie->flag |= STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER; + } else + header_size = asprintf(&header, "%zx\r\n", size); + } + if (!header) { + iovec_free(&iovec_entries, &iovector); + HTTPERROR_500; + } + + if (!cookie->batch) { + cookie->batch = calloc(1, sizeof(ot_batch)); + if (!cookie->batch) { + free(header); + iovec_free(&iovec_entries, &iovector); + HTTPERROR_500; + } + } + + batch_add(cookie->batch, header, header_size); + batch_add_vector(cookie->batch, iovector, iovec_entries); + free(iovector); + + if (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) + batch_add_copy(cookie->batch, "\r\n", 2); + } + + if ((cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) && cookie->batch && !is_partial) { + batch_add_copy(cookie->batch, "0\r\n\r\n", 5); + cookie->flag &= ~STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER; + } + + fdstate_set_expiry(sock, time_future_seconds(OT_CLIENT_TIMEOUT_SEND)); + http_interest_set(sock, cookie, EVENT_OUT); + return 0; +} + +static ssize_t http_handle_stats(const int64_t sock, struct ot_workstruct *ws, char *read_ptr) { + static const ot_keywords keywords_main[] = {{"mode", 1}, {"format", 2}, {"info_hash", 3}, {NULL, -3}}; + static const ot_keywords keywords_mode[] = {{"peer", TASK_STATS_PEERS}, + {"conn", TASK_STATS_CONNS}, + {"scrp", TASK_STATS_SCRAPE}, + {"udp4", TASK_STATS_UDP}, + {"tcp4", TASK_STATS_TCP}, + {"torr", TASK_STATS_TORRENTS}, + {"fscr", TASK_STATS_FULLSCRAPE}, + {"s24s", TASK_STATS_SLASH24S}, + {"tpbs", TASK_STATS_TPB}, + {"herr", TASK_STATS_HTTPERRORS}, + {"completed", TASK_STATS_COMPLETED}, + {"top100", TASK_STATS_TOP100}, + {"top10", TASK_STATS_TOP10}, + {"renew", TASK_STATS_RENEW}, + {"syncs", TASK_STATS_SYNCS}, + {"everything", TASK_STATS_EVERYTHING}, + {"statedump", TASK_FULLSCRAPE_TRACKERSTATE}, + {"woodpeckers", TASK_STATS_WOODPECKERS}, +#ifdef WANT_LOG_NUMWANT + {"numwants", TASK_STATS_NUMWANTS}, +#endif + {NULL, -3}}; + static const ot_keywords keywords_format[] = {{"bin", TASK_FULLSCRAPE_TPB_BINARY}, {"ben", TASK_FULLSCRAPE}, + {"url", TASK_FULLSCRAPE_TPB_URLENCODED}, {"txt", TASK_FULLSCRAPE_TPB_ASCII}, + {"txtp", TASK_FULLSCRAPE_TPB_ASCII_PLUS}, {NULL, -3}}; + + struct http_data *cookie = fdstate_cookie(sock); + char *reply; + int mode = TASK_STATS_PEERS, scanon = 1, format = 0; + + /* A socket without one is a dead socket, which is what http_sendiovecdata + already answers 500 to. */ + if (!cookie) + HTTPERROR_500; + +#ifdef WANT_RESTRICT_STATS + if (!accesslist_is_blessed(cookie->ip, OT_PERMISSION_MAY_STAT)) + HTTPERROR_403_IP; +#endif + + while (scanon) { + switch (urlencoded_match_keyword(keywords_main, &read_ptr, SCAN_SEARCHPATH_PARAM)) { + case -2: + scanon = 0; + break; /* TERMINATOR */ + case -1: + HTTPERROR_400_PARAM; /* PARSE ERROR */ + case -3: + urlencoded_skip_value(&read_ptr); + break; + case 1: /* matched "mode" */ + if ((mode = urlencoded_match_keyword(keywords_mode, &read_ptr, SCAN_SEARCHPATH_VALUE)) <= 0) + HTTPERROR_400_PARAM; + break; + case 2: /* matched "format" */ + if ((format = urlencoded_match_keyword(keywords_format, &read_ptr, SCAN_SEARCHPATH_VALUE)) <= 0) + HTTPERROR_400_PARAM; + break; + case 3: + HTTPERROR_400_PARAM; /* If the stats URL was mistakenly added as announce URL, return a 400 */ + } + } + +#ifdef WANT_FULLSCRAPE + if (mode == TASK_FULLSCRAPE_TRACKERSTATE) { + format = mode; + mode = TASK_STATS_TPB; + } + + if (mode == TASK_STATS_TPB) { +#ifdef WANT_COMPRESSION_GZIP + ws->request[ws->request_size - 1] = 0; +#ifndef WANT_COMPRESSION_GZIP_ALWAYS + if (strstr(read_ptr - 1, "gzip")) { +#endif + cookie->flag |= STRUCT_HTTP_FLAG_GZIP; + format |= TASK_FLAG_GZIP; +#ifndef WANT_COMPRESSION_GZIP_ALWAYS + } +#endif +#endif + cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED; + + /* Clients waiting for us should not easily timeout */ + fdstate_set_expiry(sock, 0); + http_interest_set(sock, cookie, 0); + + /* Everything this connection needs has been written, so it can be given + a home of its own and only then be made reachable by a worker thread */ + if (!http_adopt(sock)) + HTTPERROR_500; + fullscrape_deliver(sock, format); + return ws->reply_size = -2; + } +#endif + + /* default format for now */ + if ((mode & TASK_CLASS_MASK) == TASK_STATS) { + + cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK; + + /* Complex stats also include expensive memory debugging tools */ + fdstate_set_expiry(sock, 0); + + if (!http_adopt(sock)) + HTTPERROR_500; + stats_deliver(sock, mode); + return ws->reply_size = -2; + } + + /* Simple stats can be answerred immediately */ + reply = ws->reply; + return_stats_for_tracker(&reply, ws->outbuf + G_OUTBUF_SIZE, mode, 0); + return ws->reply_size = reply - ws->reply; +} + +#ifdef WANT_MODEST_FULLSCRAPES +static pthread_mutex_t g_modest_fullscrape_mutex = PTHREAD_MUTEX_INITIALIZER; +static ot_vector g_modest_fullscrape_timeouts; +typedef struct { + ot_ip6 ip; + ot_time last_fullscrape; +} ot_scrape_log; +#endif + +#ifdef WANT_FULLSCRAPE +static ssize_t http_handle_fullscrape(const int64_t sock, struct ot_workstruct *ws) { + struct http_data *cookie = fdstate_cookie(sock); + int format = 0; + +#ifdef WANT_MODEST_FULLSCRAPES + { + ot_scrape_log this_peer, *new_peer; + int exactmatch; + memcpy(this_peer.ip, cookie->ip, sizeof(ot_ip6)); + this_peer.last_fullscrape = time_now_seconds(); + pthread_mutex_lock(&g_modest_fullscrape_mutex); + new_peer = vector_find_or_insert(&g_modest_fullscrape_timeouts, &this_peer, sizeof(ot_scrape_log), sizeof(ot_ip6), &exactmatch); + if (!new_peer) { + pthread_mutex_unlock(&g_modest_fullscrape_mutex); + HTTPERROR_500; + } + if (exactmatch && (this_peer.last_fullscrape - new_peer->last_fullscrape) < OT_MODEST_PEER_TIMEOUT) { + pthread_mutex_unlock(&g_modest_fullscrape_mutex); + HTTPERROR_402_NOTMODEST; + } + memcpy(new_peer, &this_peer, sizeof(ot_scrape_log)); + pthread_mutex_unlock(&g_modest_fullscrape_mutex); + } +#endif + + +#if defined(WANT_COMPRESSION_GZIP) || defined(WANT_COMPRESSION_ZSTD) + ws->request[ws->request_size - 1] = 0; +#ifdef WANT_COMPRESSION_GZIP + if (strstr(ws->request, "gzip")) { + cookie->flag |= STRUCT_HTTP_FLAG_GZIP; + format |= TASK_FLAG_GZIP; + } +#endif +#ifdef WANT_COMPRESSION_ZSTD + if (strstr(ws->request, "zstd")) { + cookie->flag |= STRUCT_HTTP_FLAG_ZSTD; + format |= TASK_FLAG_ZSTD; + } +#endif + +#if defined(WANT_COMPRESSION_ZSTD) && defined(WANT_COMPRESSION_ZSTD_ALWAYS) + cookie->flag |= STRUCT_HTTP_FLAG_ZSTD; + format |= TASK_FLAG_ZSTD; +#endif + +#if defined(WANT_COMPRESSION_GZIP) && defined(WANT_COMPRESSION_GZIP_ALWAYS) + cookie->flag |= STRUCT_HTTP_FLAG_GZIP; + format |= TASK_FLAG_GZIP; +#endif +#endif + + stats_issue_event(ws, EVENT_FULLSCRAPE_REQUEST, 0, (uintptr_t)cookie->ip); + +#ifdef _DEBUG_HTTPERROR + fprintf(stderr, "%s", ws->debugbuf); +#endif + + /* Pass this task to the worker thread */ + cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED; + /* Clients waiting for us should not easily timeout */ + fdstate_set_expiry(sock, 0); + http_interest_set(sock, cookie, 0); + + if (!http_adopt(sock)) + HTTPERROR_500; + fullscrape_deliver(sock, format); + return ws->reply_size = -2; +} +#endif + +static int compare_hash(const void *left, const void *right) { return memcmp(left, right, sizeof(ot_hash)); } + +static ssize_t http_handle_scrape(const int64_t sock, struct ot_workstruct *ws, char *read_ptr) { + static const ot_keywords keywords_scrape[] = {{"info_hash", 1}, {NULL, -3}}; + + ot_hash *multiscrape_buf = (ot_hash *)ws->request; + char *reply = ws->reply; + int scanon = 1; + size_t i, unique, numwant = 0; + + /* This is to hack around stupid clients that send "scrape ?info_hash" */ + if (read_ptr[-1] != '?') { + while ((*read_ptr != '?') && (*read_ptr != '\n')) + ++read_ptr; + if (*read_ptr == '\n') + HTTPERROR_400_PARAM; + ++read_ptr; + } + + while (scanon) { + switch (urlencoded_match_keyword(keywords_scrape, &read_ptr, SCAN_SEARCHPATH_PARAM)) { + case -2: + scanon = 0; + break; /* TERMINATOR */ + default: + HTTPERROR_400_PARAM; /* PARSE ERROR */ + case -3: + urlencoded_skip_value(&read_ptr); + break; + case 1: /* matched "info_hash" */ + /* ignore this, when we have less than 20 bytes */ + if (urlencoded_parse(&read_ptr, (char *)(multiscrape_buf + numwant++), SCAN_SEARCHPATH_VALUE) != (ssize_t)sizeof(ot_hash)) + HTTPERROR_400_PARAM; + break; + } + } + + /* No info_hash found? Inform user */ + if (!numwant) + HTTPERROR_400_PARAM; + + /* Limit number of hashes to process */ + if (numwant > OT_MAXMULTISCRAPE_COUNT) + numwant = OT_MAXMULTISCRAPE_COUNT; + + /* A bencoded dictionary has its keys in order and names each one once, so + the answer cannot just follow the order the client asked in */ + qsort(multiscrape_buf, numwant, sizeof(ot_hash), compare_hash); + for (i = unique = 1; i < numwant; ++i) + if (memcmp(multiscrape_buf[i], multiscrape_buf[unique - 1], sizeof(ot_hash))) + memcpy(multiscrape_buf[unique++], multiscrape_buf[i], sizeof(ot_hash)); + numwant = unique; + + /* Enough for http header + whole scrape string */ + return_tcp_scrape_for_torrent((const ot_hash *)multiscrape_buf, numwant, &reply, ws->outbuf + G_OUTBUF_SIZE); + ws->reply_size = reply - ws->reply; + stats_issue_event(ws, EVENT_SCRAPE, FLAG_TCP, ws->reply_size); + return ws->reply_size; +} + +#ifdef WANT_LOG_NUMWANT +unsigned long long numwants[201]; +#endif + +#if defined(WANT_KEEPALIVE) || defined(WANT_IP_FROM_PROXY) +static char *http_header(char *data, size_t byte_count, const char *header) { + size_t i; + long sl = strlen(header); + for (i = 0; i + sl + 2 < byte_count; ++i) { + if (data[i] != '\n' || data[i + sl + 1] != ':') + continue; + if (strncasecmp(data + i + 1, header, sl)) + continue; + data += i + sl + 2; + while (*data == ' ' || *data == '\t') + ++data; + return data; + } + return 0; +} +#endif + +static ot_keywords keywords_announce[] = {{"port", 1}, {"left", 2}, {"event", 3}, {"numwant", 4}, {"compact", 5}, {"compact6", 5}, {"info_hash", 6}, +#ifdef WANT_IP_FROM_QUERY_STRING + {"ip", 7}, +#endif + {"peer_id", 8}, {NULL, -3}}; +static ot_keywords keywords_announce_event[] = {{"completed", 1}, {"stopped", 2}, {NULL, -3}}; + +/* For the replies that are one fixed string and nothing else, where the only + thing the caller wants back is how long it turned out */ +static ssize_t reply_literal(struct ot_workstruct *ws, const char *text) { + char *reply = ws->reply; + + fmt_bytes(&reply, ws->outbuf + G_OUTBUF_SIZE, text, strlen(text)); + return reply - ws->reply; +} + +static ssize_t http_handle_announce(const int64_t sock, struct ot_workstruct *ws, char *read_ptr) { + size_t numwant; + int64_t tmp; + int scanon; + unsigned short port = 0; + char *write_ptr; + ssize_t len; + struct http_data *cookie = fdstate_cookie(sock); + + /* This is to hack around stupid clients that send "announce ?info_hash" */ + if (read_ptr[-1] != '?') { + while ((*read_ptr != '?') && (*read_ptr != '\n')) + ++read_ptr; + if (*read_ptr == '\n') + HTTPERROR_400_PARAM; + ++read_ptr; + } + +#ifdef WANT_IP_FROM_PROXY + if (accesslist_is_blessed(cookie->ip, OT_PERMISSION_MAY_PROXY)) { + ot_ip6 proxied_ip; + char *fwd = http_header(ws->request, ws->header_size, "x-forwarded-for"); + if (fwd && ip_parse(fwd, proxied_ip)) { + OT_SETIP(ws->peer, proxied_ip); + } else + OT_SETIP(ws->peer, cookie->ip); + } else +#endif + OT_SETIP(ws->peer, cookie->ip); + + ws->peer_id = NULL; + ws->hash = NULL; + + OT_SETPORT(ws->peer, &port); + OT_PEERFLAG(ws->peer) = 0; + numwant = 50; + scanon = 1; + + while (scanon) { + switch (urlencoded_match_keyword(keywords_announce, &read_ptr, SCAN_SEARCHPATH_PARAM)) { + case -2: + scanon = 0; + break; /* TERMINATOR */ + case -1: + HTTPERROR_400_PARAM; /* PARSE ERROR */ + case -3: + urlencoded_skip_value(&read_ptr); + break; + case 1: /* matched "port" */ + len = urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE); + if ((len <= 0) || scan_int(write_ptr, len, &tmp) || (tmp > 0xffff)) + HTTPERROR_400_PARAM; + port = htons((uint16_t)tmp); + OT_SETPORT(&ws->peer, &port); + break; + case 2: /* matched "left" */ + if ((len = urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE)) <= 0) + HTTPERROR_400_PARAM; + if (scan_int(write_ptr, len, &tmp)) + tmp = 0; + if (!tmp) + OT_PEERFLAG(&ws->peer) |= PEER_FLAG_SEEDING; + break; + case 3: /* matched "event" */ + switch (urlencoded_match_keyword(keywords_announce_event, &read_ptr, SCAN_SEARCHPATH_VALUE)) { + case -1: + HTTPERROR_400_PARAM; + case 1: /* matched "completed" */ + OT_PEERFLAG(&ws->peer) |= PEER_FLAG_COMPLETED; + break; + case 2: /* matched "stopped" */ + OT_PEERFLAG(&ws->peer) |= PEER_FLAG_STOPPED; + break; + default: + break; + } + break; + case 4: /* matched "numwant" */ + len = urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE); + if ((len <= 0) || scan_int(write_ptr, len, &tmp)) + HTTPERROR_400_PARAM; + if (tmp < 0) + tmp = 50; + if (tmp > 200) + tmp = 200; + numwant = (size_t)tmp; + break; + case 5: /* matched "compact" */ + len = urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE); + if ((len <= 0) || scan_int(write_ptr, len, &tmp)) + HTTPERROR_400_PARAM; + if (!tmp) + HTTPERROR_400_COMPACT; + break; + case 6: /* matched "info_hash" */ + if (ws->hash) + HTTPERROR_400_DOUBLEHASH; + /* ignore this, when we have less than 20 bytes */ + if (urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE) != 20) + HTTPERROR_400_PARAM; + ws->hash = (ot_hash *)write_ptr; + break; +#ifdef WANT_IP_FROM_QUERY_STRING + case 7: /* matched "ip" */ + { + char *tmp_buf1 = ws->reply, *tmp_buf2 = ws->reply + 16; + len = urlencoded_parse(&read_ptr, tmp_buf2, SCAN_SEARCHPATH_VALUE); + tmp_buf2[len] = 0; + if ((len <= 0) || ip_parse(tmp_buf2, tmp_buf1) != (size_t)len) + HTTPERROR_400_PARAM; + OT_SETIP(&ws->peer, tmp_buf1); + } break; +#endif + case 8: /* matched "peer_id" */ + /* ignore this, when we have less than 20 bytes */ + if (urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE) != 20) + HTTPERROR_400_PARAM; + ws->peer_id = write_ptr; + break; + } + } + +#ifdef WANT_LOG_NUMWANT + numwants[numwant]++; +#endif + + /* XXX DEBUG + stats_issue_event(ws, EVENT_ACCEPT, FLAG_TCP, (uintptr_t)ws->reply ); + */ + + /* Scanned whole query string */ + if (!ws->hash) + return ws->reply_size = reply_literal(ws, "d14:failure reason80:Your client forgot to send your torrent's info_hash. Please upgrade your client.e"); + + if (OT_PEERFLAG(&ws->peer) & PEER_FLAG_STOPPED) + ws->reply_size = remove_peer_from_torrent(FLAG_TCP, ws); + else + ws->reply_size = add_peer_to_torrent_and_return_peers(FLAG_TCP, ws, numwant); + + stats_issue_event(ws, EVENT_ANNOUNCE, FLAG_TCP, ws->reply_size); + return ws->reply_size; +} + +ssize_t http_handle_request(const int64_t sock, struct ot_workstruct *ws) { + ssize_t reply_off, len; + char *read_ptr = ws->request, *write_ptr; + +#ifdef _DEBUG_HTTPERROR + reply_off = ws->request_size; + if (ws->request_size >= G_DEBUGBUF_SIZE) + reply_off = G_DEBUGBUF_SIZE - 1; + memcpy(ws->debugbuf, ws->request, reply_off); + ws->debugbuf[reply_off] = 0; +#endif + + /* Tell subroutines where to put reply data */ + ws->reply = ws->outbuf + SUCCESS_HTTP_HEADER_LENGTH; + + /* Find out if the client wants to keep this connection alive, before any + subroutine gets to answer: an error reply finishes with the connection + on its way out, and ws outlives the connection it was serving */ + ws->keep_alive = 0; +#ifdef WANT_KEEPALIVE + if ((write_ptr = http_header(ws->request, ws->header_size, "connection")) && (*write_ptr == 'K' || *write_ptr == 'k')) + ws->keep_alive = 1; +#endif + { + struct http_data *connection = fdstate_cookie(sock); + if (connection) { + if (ws->keep_alive) + connection->flag |= STRUCT_HTTP_FLAG_KEEPALIVE; + else + connection->flag &= ~STRUCT_HTTP_FLAG_KEEPALIVE; + } + } + + /* This one implicitely tests strlen < 5, too -- remember, it is \n terminated */ + if (memcmp(read_ptr, "GET /", 5)) + HTTPERROR_400; + + /* Skip leading '/' */ + for (read_ptr += 4; *read_ptr == '/'; ++read_ptr) + ; + + /* Try to parse the request. + In reality we abandoned requiring the url to be correct. This now + only decodes url encoded characters, we check for announces and + scrapes by looking for "a*" or "sc" */ + len = urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_PATH); + + /* If parsing returned an error, leave with not found */ + if (g_redirecturl && (len == -2)) + HTTPERROR_302; + if (len <= 0) + HTTPERROR_404; + + /* This is the hardcore match for announce*/ + if ((*write_ptr == 'a') || (*write_ptr == '?')) + http_handle_announce(sock, ws, read_ptr); +#ifdef WANT_FULLSCRAPE + else if (!memcmp(write_ptr, "scrape HTTP/", 12)) + http_handle_fullscrape(sock, ws); +#endif + /* This is the hardcore match for scrape */ + else if (!memcmp(write_ptr, "sc", 2)) + http_handle_scrape(sock, ws, read_ptr); + /* All the rest is matched the standard way */ + else if (len == g_stats_path_len && !memcmp(write_ptr, g_stats_path, len)) + http_handle_stats(sock, ws, read_ptr); + else + HTTPERROR_404; + + /* If routines handled sending themselves, just return */ + if (ws->reply_size == -2) + return 0; + /* If routine failed, let http error take over */ + if (ws->reply_size <= 0) + HTTPERROR_500; + + /* This one is rather ugly, so I take you step by step through it. + + 1. In order to avoid having two buffers, one for header and one for content, we allow all above functions from trackerlogic to + write to a fixed location, leaving SUCCESS_HTTP_HEADER_LENGTH bytes in our work buffer, which is enough for the static string + plus dynamic space needed to expand our Content-Length value. We reserve SUCCESS_HTTP_SIZE_OFF for its expansion and calculate + the space NOT needed to expand in reply_off + */ + reply_off = SUCCESS_HTTP_SIZE_OFF - snprintf(ws->outbuf, 0, "%zd", ws->reply_size); + ws->reply = ws->outbuf + reply_off; + + /* 2. Now we write the header so that its terminating '\0' lands exactly one byte before the content starts. The bound is the + space step 1 reserved, so the '\0' falls where step 3 expects it. Complete packet size is increased by size of header plus + one byte '\n', we will copy over '\0' in next step */ + ws->reply_size += 1 + snprintf(ws->reply, (size_t)(SUCCESS_HTTP_HEADER_LENGTH - reply_off), + "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: %zd\r\n\r", ws->reply_size); + + /* 3. Finally we join both blocks neatly */ + ws->outbuf[SUCCESS_HTTP_HEADER_LENGTH - 1] = '\n'; + + http_senddata(sock, ws); + return ws->reply_size; +} diff --git a/src/ot_http.h b/src/ot_http.h new file mode 100644 index 0000000..fb56c64 --- /dev/null +++ b/src/ot_http.h @@ -0,0 +1,53 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_HTTP_H__ +#define OT_HTTP_H__ + +#include +#include +#include + +#include "ot_batch.h" +#include "ot_event.h" +#include "ot_buf.h" +#include "trackerlogic.h" + +typedef enum { + STRUCT_HTTP_FLAG_WAITINGFORTASK = 1, + STRUCT_HTTP_FLAG_GZIP = 2, + STRUCT_HTTP_FLAG_BZIP2 = 4, + STRUCT_HTTP_FLAG_ZSTD = 8, + STRUCT_HTTP_FLAG_CHUNKED = 16, + STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER = 32, + STRUCT_HTTP_FLAG_KEEPALIVE = 64, + STRUCT_HTTP_FLAG_ONSTACK = 128 +} STRUCT_HTTP_FLAG; + +struct http_data { + ot_buf request; + int interest; /* EVENT_IN and EVENT_OUT as asked for */ + ot_batch *batch; + ot_ip6 ip; + STRUCT_HTTP_FLAG flag; +}; + +/* What this connection is waiting for: EVENT_IN, EVENT_OUT, or nothing */ +void http_interest_set(const int64_t s, struct http_data *cookie, int what); + +/* Close a connection and release everything that belongs to it */ +void http_close(const int64_t s); + +/* Promote a stack cookie to a heap one, for persistence and for handing it + over to the event loop or to another thread. Does nothing to one that is + already on the heap. Returns the connection's state, or NULL. */ +struct http_data *http_adopt(const int64_t s); + +ssize_t http_handle_request(const int64_t s, struct ot_workstruct *ws); +ssize_t http_sendiovecdata(const int64_t s, struct ot_workstruct *ws, size_t iovec_entries, struct iovec *iovector, int is_partial); +ssize_t http_issue_error(const int64_t s, struct ot_workstruct *ws, int code); + +extern const char *g_stats_path; +extern ssize_t g_stats_path_len; + +#endif diff --git a/src/ot_iovec.c b/src/ot_iovec.c new file mode 100644 index 0000000..328f160 --- /dev/null +++ b/src/ot_iovec.c @@ -0,0 +1,87 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include + +/* Opentracker */ +#include "ot_iovec.h" + +void *iovec_new_chunk(size_t *iovec_entries, struct iovec **iovector, size_t new_alloc) { + void *new_data; + size_t new_entries = 1 + *iovec_entries; + struct iovec *new_vec = realloc(*iovector, new_entries * sizeof(struct iovec)); + + if (!new_vec) + return NULL; + + *iovector = new_vec; + + /* Only allocate after we have a place to store the pointer */ + new_data = malloc(new_alloc); + if (!new_data) + return NULL; + + new_vec[new_entries - 1].iov_base = new_data; + new_vec[new_entries - 1].iov_len = new_alloc; + + ++*iovec_entries; + return new_data; +} + +void *iovec_append(size_t *iovec_entries, struct iovec **iovector, struct iovec *append_iovector) { + size_t new_entries = *iovec_entries + 1; + struct iovec *new_vec = realloc(*iovector, new_entries * sizeof(struct iovec)); + if (!new_vec) + return NULL; + + /* Take over data from appended iovec */ + new_vec[*iovec_entries].iov_base = append_iovector->iov_base; + new_vec[*iovec_entries].iov_len = append_iovector->iov_len; + + append_iovector->iov_base = NULL; + append_iovector->iov_len = 0; + + *iovector = new_vec; + *iovec_entries = new_entries; + + return new_vec; +} + +void iovec_free(size_t *iovec_entries, struct iovec **iovector) { + size_t i; + for (i = 0; i < *iovec_entries; ++i) + free(((*iovector)[i]).iov_base); + free(*iovector); + *iovector = NULL; + *iovec_entries = 0; +} + +void iovec_trim_last(size_t *iovec_entries, struct iovec **iovector, void *last_ptr) { + if (*iovec_entries) { + char *base = (char *)((*iovector)[*iovec_entries - 1]).iov_base; + size_t new_alloc = ((char *)last_ptr) - base; + + ((*iovector)[*iovec_entries - 1]).iov_base = realloc(base, new_alloc); + ((*iovector)[*iovec_entries - 1]).iov_len = new_alloc; + } +} + +void *iovec_next_chunk(size_t *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc) { + void *new_data; + + iovec_trim_last(iovec_entries, iovector, last_ptr); + + if (!(new_data = iovec_new_chunk(iovec_entries, iovector, new_alloc))) + iovec_free(iovec_entries, iovector); + + return new_data; +} + +size_t iovec_length(const size_t *iovec_entries, const struct iovec **iovector) { + size_t i, length = 0; + for (i = 0; i < *iovec_entries; ++i) + length += ((*iovector)[i]).iov_len; + return length; +} diff --git a/src/ot_iovec.h b/src/ot_iovec.h new file mode 100644 index 0000000..4a8abbb --- /dev/null +++ b/src/ot_iovec.h @@ -0,0 +1,18 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_IOVEC_H__ +#define OT_IOVEC_H__ + +#include + +void *iovec_new_chunk(size_t *iovec_entries, struct iovec **iovector, size_t new_alloc); +void *iovec_append(size_t *iovec_entries, struct iovec **iovector, struct iovec *append_iovector); +void iovec_trim_last(size_t *iovec_entries, struct iovec **iovector, void *last_ptr); +void iovec_free(size_t *iovec_entries, struct iovec **iovector); + +size_t iovec_length(const size_t *iovec_entries, const struct iovec **iovector); + +void *iovec_next_chunk(size_t *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc); + +#endif diff --git a/src/ot_ip.c b/src/ot_ip.c new file mode 100644 index 0000000..9c76290 --- /dev/null +++ b/src/ot_ip.c @@ -0,0 +1,199 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include + +/* Opentracker */ +#include "ot_hex.h" +#include "ot_ip.h" + +const char ip_v4mapped_prefix[12] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff}; + +/* The dotted quad is only ever reached through the v4 mapped form, so both + halves of it stay in here */ + +static size_t ip4_format(char *dest, const uint8_t *ip) { + char *d = dest; + int i; + + for (i = 0; i < 4; ++i) { + unsigned int octet = ip[i]; + + if (i) + *d++ = '.'; + if (octet >= 100) + *d++ = (char)('0' + octet / 100); + if (octet >= 10) + *d++ = (char)('0' + (octet / 10) % 10); + *d++ = (char)('0' + octet % 10); + } + + return (size_t)(d - dest); +} + +static size_t ip4_scan(const char *src, uint8_t ip[4]) { + uint8_t octets[4]; + const char *s = src; + int i; + + for (i = 0; i < 4; ++i) { + unsigned int value = 0; + int digits = 0; + + if (i && *s++ != '.') + return 0; + + /* Leading zeros are tolerated, a value beyond one octet is not */ + while (*s >= '0' && *s <= '9' && digits < 4) { + value = value * 10 + (unsigned int)(*s++ - '0'); + ++digits; + } + if (!digits || value > 255) + return 0; + + octets[i] = (uint8_t)value; + } + + memcpy(ip, octets, sizeof(octets)); + return (size_t)(s - src); +} + +static unsigned int group_at(const char *ip, int n) { return ((unsigned int)(uint8_t)ip[2 * n] << 8) | (uint8_t)ip[2 * n + 1]; } + +size_t ip_format(char *dest, const ot_ip6 ip) { + static const char hexdigit[] = "0123456789abcdef"; + char *d = dest; + int best_start = 0, best_len = 0, i = 0; + + if (ip_is_v4mapped(ip)) + return ip4_format(dest, (const uint8_t *)ip + sizeof(ip_v4mapped_prefix)); + + /* RFC 5952 4.2: the longest run of zero groups becomes "::", never a run + of one, and the first of two equal runs wins */ + while (i < 8) { + int j = i; + + if (group_at(ip, i)) { + ++i; + continue; + } + while (j < 8 && !group_at(ip, j)) + ++j; + if (j - i > best_len) { + best_len = j - i; + best_start = i; + } + i = j; + } + if (best_len < 2) + best_len = 0; + + for (i = 0; i < 8; ++i) { + unsigned int group; + int shift = 12; + + if (best_len && i == best_start) { + *d++ = ':'; + if (i + best_len == 8) + *d++ = ':'; /* nothing follows, so close the run */ + i += best_len - 1; + continue; + } + + if (i) + *d++ = ':'; + + group = group_at(ip, i); + while (shift && !(group >> shift)) + shift -= 4; /* RFC 5952 4.1: no leading zeros */ + for (;; shift -= 4) { + *d++ = hexdigit[(group >> shift) & 15]; + if (!shift) + break; + } + } + + return (size_t)(d - dest); +} + +size_t ip_parse(const char *src, ot_ip6 ip) { + uint8_t prefix[16], suffix[16]; + size_t prefixlen = 0, suffixlen = 0; + const char *s = src; + int have_gap = 0; + int need_colon = 0; + + /* A plain IPv4 address is an address too, stored v4 mapped */ + { + size_t consumed = ip4_scan(src, prefix); + if (consumed) { + memcpy(ip, ip_v4mapped_prefix, sizeof(ip_v4mapped_prefix)); + memcpy(ip + sizeof(ip_v4mapped_prefix), prefix, 4); + return consumed; + } + } + + for (;;) { + unsigned int value = 0; + int digits = 0, nibble; + uint8_t *into = have_gap ? suffix : prefix; + size_t *fill = have_gap ? &suffixlen : &prefixlen; + + /* RFC 4291 2.2: one run of zero groups may be written "::" */ + if (s[0] == ':' && s[1] == ':') { + if (have_gap) + return 0; + have_gap = 1; + need_colon = 0; + s += 2; + continue; + } + + if (need_colon) { + if (*s != ':') + break; + ++s; + } + + /* RFC 4291 2.2 form 3: the final two groups may be a dotted quad */ + if (prefixlen + suffixlen + 4 <= 16) { + size_t consumed = ip4_scan(s, into + *fill); + if (consumed) { + *fill += 4; + s += consumed; + break; + } + } + + while (digits < 4 && (nibble = hex_value(*s)) >= 0) { + value = value * 16 + (unsigned int)nibble; + ++digits; + ++s; + } + if (!digits) { + if (need_colon) + --s; /* leave the separator we ate for the caller */ + break; + } + + if (prefixlen + suffixlen + 2 > 16) + return 0; + + into[(*fill)++] = (uint8_t)(value >> 8); + into[(*fill)++] = (uint8_t)value; + need_colon = 1; + } + + if (!prefixlen && !suffixlen && !have_gap) + return 0; + if (!have_gap && prefixlen + suffixlen != 16) + return 0; /* without "::" every group has to be spelled out */ + + memset(ip, 0, 16); + memcpy(ip, prefix, prefixlen); + memcpy(ip + 16 - suffixlen, suffix, suffixlen); + + return (size_t)(s - src); +} diff --git a/src/ot_ip.h b/src/ot_ip.h new file mode 100644 index 0000000..533403e --- /dev/null +++ b/src/ot_ip.h @@ -0,0 +1,31 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_IP_H__ +#define OT_IP_H__ + +#include +#include + +#include "trackerlogic.h" + +/* opentracker holds every address as sixteen bytes, writing IPv4 as the v4 + mapped form of RFC 4291 section 2.5.5.2. */ +extern const char ip_v4mapped_prefix[12]; + +static inline int ip_is_v4mapped(const void *ip) { return !memcmp(ip, ip_v4mapped_prefix, sizeof(ip_v4mapped_prefix)); } + +/* Read an address and return how many characters it took, or zero if src + does not start with one. A plain IPv4 address is accepted and stored in + its v4 mapped form. + + The count lets the caller carry on where the address ended, which is how + "address:port" and "address/bits" are read. */ +size_t ip_parse(const char *src, ot_ip6 ip); + +/* Write an address and return how many characters it took. Nothing is + terminated, so a caller may keep appending. A v4 mapped address is written + as the plain dotted quad rather than "::ffff:1.2.3.4". */ +size_t ip_format(char *dest, const ot_ip6 ip); + +#endif diff --git a/src/ot_livesync.c b/src/ot_livesync.c new file mode 100644 index 0000000..083fe1a --- /dev/null +++ b/src/ot_livesync.c @@ -0,0 +1,241 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_accesslist.h" +#include "ot_ip.h" +#include "ot_socket.h" +#include "ot_livesync.h" +#include "ot_mutex.h" +#include "ot_stats.h" +#include "trackerlogic.h" + +#ifdef WANT_SYNC_LIVE + +ot_ip6 groupip_1 = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 224, 0, 23, 5}; + +#define LIVESYNC_INCOMING_BUFFSIZE (256 * 256) + +#define LIVESYNC_OUTGOING_BUFFSIZE_PEERS 1480 +#define LIVESYNC_OUTGOING_WATERMARK_PEERS (sizeof(ot_peer) + sizeof(ot_hash)) + +#define LIVESYNC_MAXDELAY 15 /* seconds */ + +enum { OT_SYNC_PEER4, OT_SYNC_PEER6 }; + +/* Forward declaration */ +static void *livesync_worker(void *args); + +/* For incoming packets */ +static int g_socket_in = -1; + +/* For outgoing packets */ +static int g_socket_out = -1; + +static pthread_mutex_t g_outbuf_mutex = PTHREAD_MUTEX_INITIALIZER; +typedef struct { + uint8_t data[LIVESYNC_OUTGOING_BUFFSIZE_PEERS]; + size_t fill; + ot_time next_packet_time; +} sync_buffer; + +static sync_buffer g_v6_buf; +static sync_buffer g_v4_buf; + +static pthread_t thread_id; +static int thread_started; +static _Atomic int thread_left; +void livesync_init(void) { + uint32_t packet_type6 = htonl(OT_SYNC_PEER6); + uint32_t packet_type4 = htonl(OT_SYNC_PEER4); + + if (g_socket_in == -1) + exerr("No socket address for live sync specified."); + + /* Prepare outgoing peers buffer */ + memcpy(g_v6_buf.data, &g_tracker_id, sizeof(g_tracker_id)); + memcpy(g_v4_buf.data, &g_tracker_id, sizeof(g_tracker_id)); + + memcpy(g_v6_buf.data + sizeof(g_tracker_id), &packet_type6, sizeof(packet_type6)); + memcpy(g_v4_buf.data + sizeof(g_tracker_id), &packet_type4, sizeof(packet_type4)); + + g_v6_buf.fill = sizeof(g_tracker_id) + sizeof(uint32_t); + g_v4_buf.fill = sizeof(g_tracker_id) + sizeof(uint32_t); + + g_v6_buf.next_packet_time = time_future_seconds(LIVESYNC_MAXDELAY); + g_v4_buf.next_packet_time = time_future_seconds(LIVESYNC_MAXDELAY); + + thread_started = !pthread_create(&thread_id, NULL, livesync_worker, NULL); +} + +void livesync_deinit(void) { + if (thread_started) { + /* Repeats so it interrupts a read already under way */ + while (!atomic_load(&thread_left)) { + pthread_kill(thread_id, OT_SIGNAL_WAKE); + usleep(1000); + } + pthread_join(thread_id, NULL); + thread_started = 0; + } + + if (g_socket_in != -1) + close(g_socket_in); + if (g_socket_out != -1) + close(g_socket_out); +} + +void livesync_bind_mcast(ot_ip6 ip, uint16_t port) { + if (!ip_is_v4mapped(ip)) + exerr("v6 mcast support not yet available."); + + if (g_socket_in != -1) + exerr("Error: Livesync listen ip specified twice."); + + if ((g_socket_in = socket_mcast_join(groupip_1, ip, port)) == -1) + exerr("Error: Cant bind live sync incoming socket."); + + if ((g_socket_out = socket_mcast_bind(ip, port, 0)) == -1) + exerr("Error: Cant bind live sync outgoing socket."); +} + +/* Caller MUST hold g_outbuf_mutex. Returns with g_outbuf_mutex unlocked */ +static void livesync_issue_peersync(sync_buffer *buf) { + char mycopy[LIVESYNC_OUTGOING_BUFFSIZE_PEERS]; + size_t fill = buf->fill; + + memcpy(mycopy, buf->data, fill); + buf->fill = sizeof(g_tracker_id) + sizeof(uint32_t); + buf->next_packet_time = time_future_seconds(LIVESYNC_MAXDELAY); + + /* From now this thread has a local copy of the buffer and + has modified the protected element */ + pthread_mutex_unlock(&g_outbuf_mutex); + + socket_mcast_send(g_socket_out, mycopy, fill, groupip_1, LIVESYNC_PORT); +} + +static void livesync_handle_peersync(struct ot_workstruct *ws, size_t peer_size) { + size_t off = sizeof(g_tracker_id) + sizeof(uint32_t); + + /* Prepare peer record for v4mapped expansion */ + if (peer_size == OT_PEER_SIZE4) + memcpy(&ws->peer, ip_v4mapped_prefix, sizeof(ip_v4mapped_prefix)); + + /* Now basic sanity checks have been done on the live sync packet + We might add more testing and logging. */ + while ((ssize_t)(off + sizeof(ot_hash) + peer_size) <= ws->request_size) { + char *peer_ip = ws->request + off + sizeof(ot_hash); + if (peer_size == OT_PEER_SIZE4) + memcpy(&ws->peer[0] + sizeof(ip_v4mapped_prefix), peer_ip, peer_size); + else + memcpy(&ws->peer, peer_ip, peer_size); + ws->hash = (ot_hash *)(ws->request + off); + + if (!g_opentracker_running) + return; + + if (OT_PEERFLAG(ws->peer) & PEER_FLAG_STOPPED) + remove_peer_from_torrent(FLAG_MCA, ws); + else + add_peer_to_torrent_and_return_peers(FLAG_MCA, ws, /* amount = */ 0); + + off += sizeof(ot_hash) + peer_size; + } + + stats_issue_event(ws, EVENT_SYNC, 0, (ws->request_size - sizeof(g_tracker_id) - sizeof(uint32_t)) / ((ssize_t)sizeof(ot_hash) + peer_size)); +} + +/* Tickle the live sync module from time to time, so no events get + stuck when there's not enough traffic to fill udp packets fast + enough */ +void livesync_ticker(void) { + /* livesync_issue_peersync sets g_next_packet_time */ + pthread_mutex_lock(&g_outbuf_mutex); + if (time_now_seconds() > g_v6_buf.next_packet_time && g_v6_buf.fill > sizeof(g_tracker_id) + sizeof(uint32_t)) + livesync_issue_peersync(&g_v6_buf); + else + pthread_mutex_unlock(&g_outbuf_mutex); + + pthread_mutex_lock(&g_outbuf_mutex); + if (time_now_seconds() > g_v4_buf.next_packet_time && g_v4_buf.fill > sizeof(g_tracker_id) + sizeof(uint32_t)) + livesync_issue_peersync(&g_v4_buf); + else + pthread_mutex_unlock(&g_outbuf_mutex); +} + +/* Inform live sync about whats going on. */ +void livesync_tell(struct ot_workstruct *ws) { + size_t peer_size; /* initialized in next line */ + ot_peer *peer_src = peer_from_peer6(&ws->peer, &peer_size); + sync_buffer *dest_buf = peer_size == OT_PEER_SIZE6 ? &g_v6_buf : &g_v4_buf; + + pthread_mutex_lock(&g_outbuf_mutex); + + memcpy(dest_buf->data + dest_buf->fill, ws->hash, sizeof(ot_hash)); + dest_buf->fill += sizeof(ot_hash); + + memcpy(dest_buf->data + dest_buf->fill, peer_src, peer_size); + dest_buf->fill += peer_size; + + if (dest_buf->fill >= LIVESYNC_OUTGOING_BUFFSIZE_PEERS - LIVESYNC_OUTGOING_WATERMARK_PEERS) + livesync_issue_peersync(dest_buf); + else + pthread_mutex_unlock(&g_outbuf_mutex); +} + +static void *livesync_worker(void *args) { + struct ot_workstruct ws; + ot_ip6 in_ip; + uint16_t in_port; + + (void)args; + + /* Initialize our "thread local storage" */ + ws.inbuf = ws.request = malloc(LIVESYNC_INCOMING_BUFFSIZE); + ws.outbuf = ws.reply = 0; + + while (g_opentracker_running) { + uint32_t packet_type; + + ws.request_size = socket_recv(g_socket_in, ws.inbuf, LIVESYNC_INCOMING_BUFFSIZE, in_ip, &in_port); + + /* Expect at least tracker id and packet type */ + if (ws.request_size <= (ssize_t)(sizeof(g_tracker_id) + sizeof(uint32_t))) + continue; + if (!accesslist_is_blessed(in_ip, OT_PERMISSION_MAY_LIVESYNC)) + continue; + if (!memcmp(ws.inbuf, &g_tracker_id, sizeof(g_tracker_id))) { + /* TODO: log packet coming from ourselves */ + continue; + } + + memcpy(&packet_type, ws.inbuf + sizeof(g_tracker_id), sizeof(packet_type)); + + switch (ntohl(packet_type)) { + case OT_SYNC_PEER6: + livesync_handle_peersync(&ws, OT_PEER_SIZE6); + break; + case OT_SYNC_PEER4: + livesync_handle_peersync(&ws, OT_PEER_SIZE4); + break; + default: + break; + } + } + + free(ws.inbuf); + atomic_store(&thread_left, 1); + return NULL; +} + +#endif diff --git a/src/ot_livesync.h b/src/ot_livesync.h new file mode 100644 index 0000000..4bd3af0 --- /dev/null +++ b/src/ot_livesync.h @@ -0,0 +1,75 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_LIVESYNC_H__ +#define OT_LIVESYNC_H__ + +#include "trackerlogic.h" + +/* + Syncing is done as udp packets in the multicast domain 224.0.42.5 port 9696 + + Each tracker should join the multicast group and send its live sync packets + to that group, using a ttl of 1 + + Format of all sync packets is straight forward, packet type determines + which kind of packet this is: + + 0x0000 0x04 id of tracker instance + 0x0004 0x04 packet type + + ######## + ######## PEER SYNC PROTOCOL ######## + ######## + + Each tracker instance accumulates announce requests until its buffer is + full or a timeout is reached. Then it broadcasts its live sync packer: + + packet type SYNC_LIVE4 + [ 0x0008 0x14 info_hash + 0x001c 0x04 peer's ipv4 address + 0x0020 0x02 peer's port + 0x0024 0x02 peer flags v1 ( SEEDING = 0x80, COMPLETE = 0x40, STOPPED = 0x20 ) + ]* + + packet type SYNC_LIVE6 + [ 0x0008 0x14 info_hash + 0x001c 0x10 peer's ipv6 address + 0x002c 0x02 peer's port + 0x002e 0x02 peer flags v1 ( SEEDING = 0x80, COMPLETE = 0x40, STOPPED = 0x20 ) + ]* +*/ + +#ifdef WANT_SYNC_LIVE + +#define LIVESYNC_PORT 9696 + +void livesync_init(void); +void livesync_deinit(void); + +/* Join multicast group for listening and create sending socket */ +void livesync_bind_mcast(ot_ip6 ip, uint16_t port); + +/* Inform live sync about whats going on. */ +void livesync_tell(struct ot_workstruct *ws); + +/* Tickle the live sync module from time to time, so no events get + stuck when there's not enough traffic to fill udp packets fast + enough */ +void livesync_ticker(void); + +/* Handle an incoming live sync packet */ +void handle_livesync(const int64_t sock); + +#else + +/* If no syncing is required, save calling code from #ifdef + constructions */ +#define livesync_deinit() +#define livesync_init() +#define livesync_ticker() +#define handle_livesync(a) + +#endif + +#endif diff --git a/src/ot_mutex.c b/src/ot_mutex.c new file mode 100644 index 0000000..884b74c --- /dev/null +++ b/src/ot_mutex.c @@ -0,0 +1,70 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#include +#include + + +/* Opentracker */ +#include "ot_iovec.h" +#include "ot_mutex.h" +#include "ot_stats.h" +#include "trackerlogic.h" + +/* #define MTX_DBG( STRING ) fprintf( stderr, STRING ) */ +#define MTX_DBG(STRING) + +/* Our global all torrents list */ +static ot_vector all_torrents[OT_BUCKET_COUNT]; +static pthread_mutex_t bucket_mutex[OT_BUCKET_COUNT]; +/* Every bucket's unlock adjusts this, and a bucket lock only keeps other + threads out of that one bucket, so the counter needs its own protection */ +static _Atomic size_t g_torrent_count; + +ot_vector *mutex_bucket_lock(int bucket) { + pthread_mutex_lock(bucket_mutex + bucket); + return all_torrents + bucket; +} + +/* Torrents are spread over buckets by the first four bytes of their info_hash, + which may sit at any alignment, hence the memcpy */ +static int bucket_from_hash(ot_hash const hash) { + uint32_t bucket; + memcpy(&bucket, hash, sizeof(bucket)); + return ntohl(bucket) >> OT_BUCKET_COUNT_SHIFT; +} + +ot_vector *mutex_bucket_lock_by_hash(ot_hash const hash) { return mutex_bucket_lock(bucket_from_hash(hash)); } + +void mutex_bucket_unlock(int bucket, int delta_torrentcount) { + pthread_mutex_unlock(bucket_mutex + bucket); + if (delta_torrentcount) + atomic_fetch_add_explicit(&g_torrent_count, (size_t)delta_torrentcount, memory_order_relaxed); +} + +void mutex_bucket_unlock_by_hash(ot_hash const hash, int delta_torrentcount) { + mutex_bucket_unlock(bucket_from_hash(hash), delta_torrentcount); +} + +size_t mutex_get_torrent_count(void) { return atomic_load_explicit(&g_torrent_count, memory_order_relaxed); } + +void mutex_init(void) { + size_t i; + for (i = 0; i < OT_BUCKET_COUNT; ++i) + pthread_mutex_init(bucket_mutex + i, NULL); + memset(all_torrents, 0, sizeof(all_torrents)); +} + +void mutex_deinit(void) { + size_t i; + for (i = 0; i < OT_BUCKET_COUNT; ++i) + pthread_mutex_destroy(bucket_mutex + i); + memset(all_torrents, 0, sizeof(all_torrents)); +} diff --git a/src/ot_mutex.h b/src/ot_mutex.h new file mode 100644 index 0000000..d7ded92 --- /dev/null +++ b/src/ot_mutex.h @@ -0,0 +1,21 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_MUTEX_H__ +#define OT_MUTEX_H__ + +#include +#include "trackerlogic.h" + +void mutex_init(void); +void mutex_deinit(void); + +ot_vector *mutex_bucket_lock(int bucket); +ot_vector *mutex_bucket_lock_by_hash(ot_hash const hash); + +void mutex_bucket_unlock(int bucket, int delta_torrentcount); +void mutex_bucket_unlock_by_hash(ot_hash const hash, int delta_torrentcount); + +size_t mutex_get_torrent_count(void); + +#endif diff --git a/src/ot_random.c b/src/ot_random.c new file mode 100644 index 0000000..43f91bb --- /dev/null +++ b/src/ot_random.c @@ -0,0 +1,51 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include + +/* Opentracker */ +#include "ot_random.h" + +#ifdef WANT_INSECURE_RANDOM_FALLBACK + +#warning WANT_INSECURE_RANDOM_FALLBACK builds without arc4random. UDP connection ids become predictable and announces can be forged. Do not ship this. + +#include + +void random_init(void) { +#ifdef WANT_DEV_RANDOM + srandomdev(); +#else + srandom(time(NULL)); +#endif +} + +/* random() yields 31 bits, so one draw can never set the top bit. Combine + two of them rather than hand out a value that is quietly half as wide as + its type claims. */ +uint32_t random_uint32(void) { return ((uint32_t)random() << 16) ^ (uint32_t)random(); } + +void random_bytes(void *buf, size_t len) { + uint8_t *out = buf; + /* random() leaves the top bit clear and its lowest bits are the weakest, + so take a byte from the middle of each draw */ + while (len--) + *out++ = (uint8_t)(random() >> 8); +} + +#else + +#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__) && !defined(__DragonFly__) && !defined(__APPLE__) && \ + !(defined(__GLIBC__) && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 36))) +#include +#endif + +void random_init(void) {} + +uint32_t random_uint32(void) { return arc4random(); } + +void random_bytes(void *buf, size_t len) { arc4random_buf(buf, len); } + +#endif diff --git a/src/ot_random.h b/src/ot_random.h new file mode 100644 index 0000000..c2d0387 --- /dev/null +++ b/src/ot_random.h @@ -0,0 +1,17 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_RANDOM_H__ +#define OT_RANDOM_H__ + +#include +#include + +/* Seed whatever generator the build settled on. A no-op when backed by + arc4random, which seeds itself from the kernel. */ +void random_init(void); + +uint32_t random_uint32(void); +void random_bytes(void *buf, size_t len); + +#endif diff --git a/src/ot_rijndael.c b/src/ot_rijndael.c new file mode 100644 index 0000000..3f36bde --- /dev/null +++ b/src/ot_rijndael.c @@ -0,0 +1,488 @@ +/** + * rijndael-alg-fst.c + * + * @version 3.0 (December 2000) + * + * Optimised ANSI C code for the Rijndael cipher (now AES) + * + * @author Vincent Rijmen + * @author Antoon Bosselaers + * @author Paulo Barreto + * + * This code is hereby placed in the public domain. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "ot_rijndael.h" + +static const uint32_t Te0[256] = { + 0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU, + 0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U, + 0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU, + 0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU, + 0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U, + 0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU, + 0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU, + 0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU, + 0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU, + 0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU, + 0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U, + 0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU, + 0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU, + 0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U, + 0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU, + 0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU, + 0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU, + 0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU, + 0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU, + 0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U, + 0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU, + 0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU, + 0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU, + 0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU, + 0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U, + 0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U, + 0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U, + 0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U, + 0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU, + 0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U, + 0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U, + 0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU, + 0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU, + 0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U, + 0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U, + 0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U, + 0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU, + 0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U, + 0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU, + 0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U, + 0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU, + 0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U, + 0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U, + 0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU, + 0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U, + 0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U, + 0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U, + 0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U, + 0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U, + 0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U, + 0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U, + 0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U, + 0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU, + 0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U, + 0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U, + 0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U, + 0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U, + 0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U, + 0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U, + 0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU, + 0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U, + 0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U, + 0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U, + 0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU, +}; +static const uint32_t Te1[256] = { + 0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU, + 0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U, + 0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU, + 0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U, + 0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU, + 0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U, + 0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU, + 0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U, + 0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U, + 0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU, + 0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U, + 0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U, + 0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U, + 0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU, + 0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U, + 0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U, + 0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU, + 0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U, + 0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U, + 0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U, + 0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU, + 0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU, + 0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U, + 0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU, + 0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU, + 0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U, + 0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU, + 0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U, + 0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU, + 0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U, + 0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U, + 0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U, + 0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU, + 0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U, + 0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU, + 0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U, + 0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU, + 0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U, + 0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U, + 0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU, + 0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU, + 0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU, + 0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U, + 0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U, + 0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU, + 0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U, + 0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU, + 0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U, + 0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU, + 0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U, + 0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU, + 0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU, + 0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U, + 0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU, + 0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U, + 0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU, + 0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U, + 0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U, + 0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U, + 0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU, + 0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU, + 0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U, + 0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU, + 0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U, +}; +static const uint32_t Te2[256] = { + 0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU, + 0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U, + 0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU, + 0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U, + 0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU, + 0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U, + 0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU, + 0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U, + 0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U, + 0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU, + 0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U, + 0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U, + 0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U, + 0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU, + 0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U, + 0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U, + 0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU, + 0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U, + 0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U, + 0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U, + 0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU, + 0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU, + 0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U, + 0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU, + 0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU, + 0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U, + 0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU, + 0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U, + 0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU, + 0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U, + 0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U, + 0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U, + 0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU, + 0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U, + 0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU, + 0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U, + 0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU, + 0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U, + 0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U, + 0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU, + 0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU, + 0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU, + 0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U, + 0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U, + 0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU, + 0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U, + 0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU, + 0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U, + 0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU, + 0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U, + 0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU, + 0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU, + 0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U, + 0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU, + 0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U, + 0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU, + 0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U, + 0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U, + 0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U, + 0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU, + 0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU, + 0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U, + 0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU, + 0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U, +}; +static const uint32_t Te3[256] = { + + 0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U, + 0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U, + 0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U, + 0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU, + 0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU, + 0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU, + 0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U, + 0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU, + 0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU, + 0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U, + 0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U, + 0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU, + 0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU, + 0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU, + 0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU, + 0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU, + 0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U, + 0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU, + 0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU, + 0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U, + 0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U, + 0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U, + 0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U, + 0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U, + 0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU, + 0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U, + 0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU, + 0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU, + 0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U, + 0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U, + 0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U, + 0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU, + 0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U, + 0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU, + 0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU, + 0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U, + 0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U, + 0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU, + 0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U, + 0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU, + 0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U, + 0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U, + 0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U, + 0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U, + 0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU, + 0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U, + 0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU, + 0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U, + 0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU, + 0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U, + 0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU, + 0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU, + 0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU, + 0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU, + 0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U, + 0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U, + 0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U, + 0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U, + 0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U, + 0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U, + 0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU, + 0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U, + 0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU, + 0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU, +}; +static const uint32_t Te4[256] = { + 0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU, + 0xf2f2f2f2U, 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U, + 0x30303030U, 0x01010101U, 0x67676767U, 0x2b2b2b2bU, + 0xfefefefeU, 0xd7d7d7d7U, 0xababababU, 0x76767676U, + 0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU, + 0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U, + 0xadadadadU, 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU, + 0x9c9c9c9cU, 0xa4a4a4a4U, 0x72727272U, 0xc0c0c0c0U, + 0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, 0x26262626U, + 0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU, + 0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U, + 0x71717171U, 0xd8d8d8d8U, 0x31313131U, 0x15151515U, + 0x04040404U, 0xc7c7c7c7U, 0x23232323U, 0xc3c3c3c3U, + 0x18181818U, 0x96969696U, 0x05050505U, 0x9a9a9a9aU, + 0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U, + 0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U, + 0x09090909U, 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU, + 0x1b1b1b1bU, 0x6e6e6e6eU, 0x5a5a5a5aU, 0xa0a0a0a0U, + 0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, 0xb3b3b3b3U, + 0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U, + 0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU, + 0x20202020U, 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU, + 0x6a6a6a6aU, 0xcbcbcbcbU, 0xbebebebeU, 0x39393939U, + 0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, 0xcfcfcfcfU, + 0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU, + 0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U, + 0x45454545U, 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU, + 0x50505050U, 0x3c3c3c3cU, 0x9f9f9f9fU, 0xa8a8a8a8U, + 0x51515151U, 0xa3a3a3a3U, 0x40404040U, 0x8f8f8f8fU, + 0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U, + 0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U, + 0x10101010U, 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U, + 0xcdcdcdcdU, 0x0c0c0c0cU, 0x13131313U, 0xececececU, + 0x5f5f5f5fU, 0x97979797U, 0x44444444U, 0x17171717U, + 0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU, + 0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U, + 0x60606060U, 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU, + 0x22222222U, 0x2a2a2a2aU, 0x90909090U, 0x88888888U, + 0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, 0x14141414U, + 0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU, + 0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU, + 0x49494949U, 0x06060606U, 0x24242424U, 0x5c5c5c5cU, + 0xc2c2c2c2U, 0xd3d3d3d3U, 0xacacacacU, 0x62626262U, + 0x91919191U, 0x95959595U, 0xe4e4e4e4U, 0x79797979U, + 0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU, + 0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U, + 0x6c6c6c6cU, 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU, + 0x65656565U, 0x7a7a7a7aU, 0xaeaeaeaeU, 0x08080808U, + 0xbabababaU, 0x78787878U, 0x25252525U, 0x2e2e2e2eU, + 0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U, + 0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU, + 0x4b4b4b4bU, 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU, + 0x70707070U, 0x3e3e3e3eU, 0xb5b5b5b5U, 0x66666666U, + 0x48484848U, 0x03030303U, 0xf6f6f6f6U, 0x0e0e0e0eU, + 0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U, + 0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU, + 0xe1e1e1e1U, 0xf8f8f8f8U, 0x98989898U, 0x11111111U, + 0x69696969U, 0xd9d9d9d9U, 0x8e8e8e8eU, 0x94949494U, + 0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, 0xe9e9e9e9U, + 0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU, + 0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU, + 0xbfbfbfbfU, 0xe6e6e6e6U, 0x42424242U, 0x68686868U, + 0x41414141U, 0x99999999U, 0x2d2d2d2dU, 0x0f0f0f0fU, + 0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, 0x16161616U, +}; +static const uint32_t rcon[] = { + 0x01000000, 0x02000000, 0x04000000, 0x08000000, + 0x10000000, 0x20000000, 0x40000000, 0x80000000, + 0x1B000000, 0x36000000, /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */ +}; + +#define GETU32(pt) (((uint32_t)(pt)[0] << 24) ^ ((uint32_t)(pt)[1] << 16) ^ ((uint32_t)(pt)[2] << 8) ^ ((uint32_t)(pt)[3])) +#define PUTU32(ct, st) { (ct)[0] = (uint8_t)((st) >> 24); (ct)[1] = (uint8_t)((st) >> 16); (ct)[2] = (uint8_t)((st) >> 8); (ct)[3] = (uint8_t)(st); } + +/** + * Expand the cipher key into the encryption key schedule. + * + * @return the number of rounds for the given cipher key size. + */ +int rijndaelKeySetupEnc128(uint32_t rk[44], const uint8_t cipherKey[] ) { + int i = 0; + uint32_t temp; + + rk[0] = GETU32(cipherKey ); + rk[1] = GETU32(cipherKey + 4); + rk[2] = GETU32(cipherKey + 8); + rk[3] = GETU32(cipherKey + 12); + for (;;) { + temp = rk[3]; + rk[4] = rk[0] ^ + (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ + (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ + (Te4[(temp ) & 0xff] & 0x0000ff00) ^ + (Te4[(temp >> 24) ] & 0x000000ff) ^ + rcon[i]; + rk[5] = rk[1] ^ rk[4]; + rk[6] = rk[2] ^ rk[5]; + rk[7] = rk[3] ^ rk[6]; + if (++i == 10) { + return 10; + } + rk += 4; + } +} + +void rijndaelEncrypt128(const uint32_t rk[44], const uint8_t pt[16], uint8_t ct[16]) { + uint32_t s0, s1, s2, s3, t0, t1, t2, t3; + + /* + * map byte array block to cipher state + * and add initial round key: + */ + s0 = GETU32(pt ) ^ rk[0]; + s1 = GETU32(pt + 4) ^ rk[1]; + s2 = GETU32(pt + 8) ^ rk[2]; + s3 = GETU32(pt + 12) ^ rk[3]; + /* round 1: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[ 4]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[ 5]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[ 6]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[ 7]; + /* round 2: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[ 8]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[ 9]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[10]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[11]; + /* round 3: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[12]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[13]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[14]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[15]; + /* round 4: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[16]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[17]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[18]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[19]; + /* round 5: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[20]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[21]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[22]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[23]; + /* round 6: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[24]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[25]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[26]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[27]; + /* round 7: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[28]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[29]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[30]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[31]; + /* round 8: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[32]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[33]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[34]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[35]; + /* round 9: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[36]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[37]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[38]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[39]; + + /* + * apply last round and + * map cipher state to byte array block: + */ + s0 = + (Te4[(t0 >> 24) ] & 0xff000000) ^ + (Te4[(t1 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t2 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(t3 ) & 0xff] & 0x000000ff) ^ + rk[40]; + PUTU32(ct , s0); + s1 = + (Te4[(t1 >> 24) ] & 0xff000000) ^ + (Te4[(t2 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t3 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(t0 ) & 0xff] & 0x000000ff) ^ + rk[41]; + PUTU32(ct + 4, s1); + s2 = + (Te4[(t2 >> 24) ] & 0xff000000) ^ + (Te4[(t3 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t0 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(t1 ) & 0xff] & 0x000000ff) ^ + rk[42]; + PUTU32(ct + 8, s2); + s3 = + (Te4[(t3 >> 24) ] & 0xff000000) ^ + (Te4[(t0 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t1 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(t2 ) & 0xff] & 0x000000ff) ^ + rk[43]; + PUTU32(ct + 12, s3); +} diff --git a/src/ot_rijndael.h b/src/ot_rijndael.h new file mode 100644 index 0000000..1235215 --- /dev/null +++ b/src/ot_rijndael.h @@ -0,0 +1,20 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. + + The rijndael implementation was taken from + + http://www.cs.ucdavis.edu/~rogaway/ocb/ocb-ref/rijndael-alg-fst.c + + and modified to work with 128 bits (this is 10 rounds) only. */ + +#ifndef OT_RIJNDAEL_H__ +#define OT_RIJNDAEL_H__ + +#include + +int rijndaelKeySetupEnc128(uint32_t rk[44], const uint8_t cipherKey[] ); +void rijndaelEncrypt128(const uint32_t rk[44], const uint8_t pt[16], uint8_t ct[16]); + +extern const char *g_version_rijndael_c; + +#endif diff --git a/src/ot_scan.c b/src/ot_scan.c new file mode 100644 index 0000000..3b8e87c --- /dev/null +++ b/src/ot_scan.c @@ -0,0 +1,65 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include + +/* Opentracker */ +#include "ot_scan.h" + +size_t scan_uint(const char *src, uint64_t *dest, uint64_t max) { + unsigned long long value; + char *end; + + if (*src < '0' || *src > '9') + return 0; + + errno = 0; + value = strtoull(src, &end, 10); + + if (errno == ERANGE || value > (unsigned long long)max) + return 0; + + *dest = (uint64_t)value; + return (size_t)(end - src); +} + +size_t scan_uint_bencode(const char *src, uint64_t *dest, uint64_t max) { + uint64_t value; + size_t consumed = scan_uint(src, &value, max); + + /* bencode writes no leading zeros, so anything longer than a single digit + must not begin with one. */ + if (consumed > 1 && *src == '0') + return 0; + + if (consumed) + *dest = value; + + return consumed; +} + +int scan_int(const char *src, size_t len, int64_t *dest) { + int64_t value = 0; + int negate = 0; + + if (len && *src == '-') + --len, ++src, negate = 1; + if (!len) + return -1; + + while (len--) { + if (*src < '0' || *src > '9') + return -1; + /* A twenty character field of nines does not fit, and silently wrapping + would magically heal a nonsense announce */ + if (value > (INT64_MAX - (*src - '0')) / 10) + return -1; + value = 10 * value + (*src++ - '0'); + } + + *dest = negate ? -value : value; + return 0; +} diff --git a/src/ot_scan.h b/src/ot_scan.h new file mode 100644 index 0000000..a79ec13 --- /dev/null +++ b/src/ot_scan.h @@ -0,0 +1,25 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_SCAN_H__ +#define OT_SCAN_H__ + +#include +#include +#include + +/* Read an unsigned decimal number from src into dest and return how many + characters were consumed. Returns zero and leaves dest untouched when src + does not begin with a digit, or when the value would exceed max. */ +size_t scan_uint(const char *src, uint64_t *dest, uint64_t max); + +/* As scan_uint, but following the bencode integer grammar, which permits a + leading zero on zero alone. */ +size_t scan_uint_bencode(const char *src, uint64_t *dest, uint64_t max); + +/* Read a signed decimal number that has to take up exactly len characters, + which is how a value lifted out of a query string arrives. Returns 0, or + -1 leaving dest untouched. */ +int scan_int(const char *src, size_t len, int64_t *dest); + +#endif diff --git a/src/ot_socket.c b/src/ot_socket.c new file mode 100644 index 0000000..79bb01a --- /dev/null +++ b/src/ot_socket.c @@ -0,0 +1,310 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#define _GNU_SOURCE /* accept4 on glibc */ +#include +#include +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_ip.h" +#include "ot_socket.h" + +static void sockaddr_from_ip(struct sockaddr_in6 *sa, const ot_ip6 ip, uint16_t port) { + memset(sa, 0, sizeof(*sa)); + sa->sin6_family = AF_INET6; + sa->sin6_port = htons(port); + memcpy(&sa->sin6_addr, ip, sizeof(ot_ip6)); +} + +/* Linux reports an IPv4 peer on a dual stack socket as a v4 mapped + sockaddr_in6, the BSDs and macOS hand back a plain sockaddr_in instead, so + both forms have to be read. */ +static void ip_from_sockaddr(ot_ip6 ip, uint16_t *port, const struct sockaddr_storage *ss) { + if (ss->ss_family == AF_INET) { + const struct sockaddr_in *sa = (const struct sockaddr_in *)ss; + + memcpy(ip, ip_v4mapped_prefix, sizeof(ip_v4mapped_prefix)); + memcpy(ip + sizeof(ip_v4mapped_prefix), &sa->sin_addr, 4); + *port = ntohs(sa->sin_port); + } else { + const struct sockaddr_in6 *sa = (const struct sockaddr_in6 *)ss; + + memcpy(ip, &sa->sin6_addr, sizeof(ot_ip6)); + *port = ntohs(sa->sin6_port); + } +} + +int socket_set_blocking(int s, int blocking) { + int flags = fcntl(s, F_GETFL, 0); + + if (flags == -1) + return -1; + return fcntl(s, F_SETFL, blocking ? (flags & ~O_NONBLOCK) : (flags | O_NONBLOCK)); +} + +static int set_nonblocking(int s) { return socket_set_blocking(s, 0); } + +/* Close without letting the failure we are about to report be overwritten */ +static int close_and_fail(int s) { + int saved = errno; + close(s); + errno = saved; + return -1; +} + +static int socket_v6(int type) { + int s = socket(PF_INET6, type, 0); + int zero = 0; + + if (s == -1) + return -1; + + /* A tracker that answered only one family would be half a tracker */ + if (setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &zero, sizeof(zero)) == -1) + return close_and_fail(s); + + if (set_nonblocking(s) == -1) + return close_and_fail(s); + + return s; +} + +/* SO_REUSEPORT means three different things. Linux spreads datagrams over + the group. FreeBSD permits the bind and spreads nothing, which is what + SO_REUSEPORT_LB was added for. Darwin hands every datagram to whichever + socket bound last, so a group there starves all but one reader. */ +#if defined(SO_REUSEPORT_LB) +#define OT_BALANCE_OPTION SO_REUSEPORT_LB +#elif defined(SO_REUSEPORT) && !defined(__APPLE__) +#define OT_BALANCE_OPTION SO_REUSEPORT +#endif + +int socket_can_balance(void) { +#ifdef OT_BALANCE_OPTION + return 1; +#else + return 0; +#endif +} + +/* The only place in the tree that needs to know what a kernel calls this */ +static void socket_set_sharing(int s, socket_sharing sharing) { + int one = 1; + + setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); +#ifdef OT_BALANCE_OPTION + if (sharing == SOCKET_BALANCED) { + setsockopt(s, SOL_SOCKET, OT_BALANCE_OPTION, &one, sizeof(one)); + return; + } +#endif +#ifdef SO_REUSEPORT + if (sharing != SOCKET_EXCLUSIVE) + setsockopt(s, SOL_SOCKET, SO_REUSEPORT, &one, sizeof(one)); +#endif +} + +static int socket_bind_v6(int s, const ot_ip6 ip, uint16_t port, socket_sharing sharing) { + struct sockaddr_in6 sa; + + socket_set_sharing(s, sharing); + + sockaddr_from_ip(&sa, ip, port); + return bind(s, (struct sockaddr *)&sa, sizeof(sa)); +} + +/* Defer the listening socket reporting it to be ready to accept until enough + sensible data is available for an accept+read pair. Where available. */ +int socket_defer_accept(int s) { +#if defined(TCP_DEFER_ACCEPT) + int patience = OT_CLIENT_TIMEOUT; + + return setsockopt(s, IPPROTO_TCP, TCP_DEFER_ACCEPT, &patience, sizeof(patience)); +#elif defined(SO_ACCEPTFILTER) + struct accept_filter_arg filter; + const char name[sizeof(filter.af_name)] = "dataready"; + + memset(&filter, 0, sizeof(filter)); + memcpy(filter.af_name, name, sizeof(name)); + return setsockopt(s, SOL_SOCKET, SO_ACCEPTFILTER, &filter, sizeof(filter)); +#else + (void)s; + return -1; +#endif +} + +int socket_tcp_listen(const ot_ip6 ip, uint16_t port, socket_sharing sharing) { + int s = socket_v6(SOCK_STREAM); + + if (s == -1) + return -1; + if (socket_bind_v6(s, ip, port, sharing) == -1 || listen(s, SOMAXCONN) == -1) + return close_and_fail(s); + + return s; +} + +int socket_udp_bind(const ot_ip6 ip, uint16_t port, socket_sharing sharing) { + int s = socket_v6(SOCK_DGRAM); + + if (s == -1) + return -1; + if (socket_bind_v6(s, ip, port, sharing) == -1) + return close_and_fail(s); + + return s; +} + +int socket_tcp_connect(const ot_ip6 srcip, uint16_t srcport, const ot_ip6 dstip, uint16_t dstport) { + struct sockaddr_in6 sa; + int s = socket_v6(SOCK_STREAM); + + if (s == -1) + return -1; + if (socket_bind_v6(s, srcip, srcport, SOCKET_EXCLUSIVE) == -1) + return close_and_fail(s); + + sockaddr_from_ip(&sa, dstip, dstport); + if (connect(s, (struct sockaddr *)&sa, sizeof(sa)) == -1 && errno != EINPROGRESS && errno != EWOULDBLOCK) + return close_and_fail(s); + + return s; +} + +int socket_is_connected(int s) { + int error = 0; + socklen_t len = sizeof(error); + + if (getsockopt(s, SOL_SOCKET, SO_ERROR, &error, &len) == -1) + return 0; + if (error) { + errno = error; + return 0; + } + return 1; +} + +/* Linux hands over a blocking socket whatever the listening socket was, the + BSDs and macOS inherit the listener's flags, so the accepted socket has to + be told either way. accept4 carries the flag itself, which saves the two + fcntl calls it otherwise takes. Darwin has neither accept4 nor the flag. */ +int socket_accept(int s, ot_ip6 ip, uint16_t *port) { + struct sockaddr_storage ss; + socklen_t len = sizeof(ss); + int fd; + + memset(&ss, 0, sizeof(ss)); +#ifdef SOCK_NONBLOCK + if ((fd = accept4(s, (struct sockaddr *)&ss, &len, SOCK_NONBLOCK)) == -1) + return -1; +#else + if ((fd = accept(s, (struct sockaddr *)&ss, &len)) == -1) + return -1; + if (set_nonblocking(fd) == -1) + return close_and_fail(fd); +#endif + + ip_from_sockaddr(ip, port, &ss); + return fd; +} + +ssize_t socket_recv(int s, void *buf, size_t len, ot_ip6 ip, uint16_t *port) { + struct sockaddr_storage ss; + socklen_t sslen = sizeof(ss); + ssize_t received; + + memset(&ss, 0, sizeof(ss)); + received = recvfrom(s, buf, len, 0, (struct sockaddr *)&ss, &sslen); + if (received == -1) + return -1; + + ip_from_sockaddr(ip, port, &ss); + return received; +} + +ssize_t socket_send(int s, const void *buf, size_t len, const ot_ip6 ip, uint16_t port) { + struct sockaddr_in6 sa; + + sockaddr_from_ip(&sa, ip, port); + return sendto(s, buf, len, 0, (struct sockaddr *)&sa, sizeof(sa)); +} + +#ifdef WANT_SYNC_LIVE +static void sockaddr4_from_ip(struct sockaddr_in *sa, const ot_ip6 ip, uint16_t port) { + memset(sa, 0, sizeof(*sa)); + sa->sin_family = AF_INET; + sa->sin_port = htons(port); + memcpy(&sa->sin_addr, ip + sizeof(ip_v4mapped_prefix), 4); +} + +/* A multicast group is meant to have several listeners, so these are always + shared */ +static int socket_v4_bind(const ot_ip6 ip, uint16_t port) { + struct sockaddr_in sa; + int s; + + if ((s = socket(PF_INET, SOCK_DGRAM, 0)) == -1) + return -1; + + socket_set_sharing(s, SOCKET_SHARED); + + sockaddr4_from_ip(&sa, ip, port); + if (bind(s, (struct sockaddr *)&sa, sizeof(sa)) == -1) + return close_and_fail(s); + + return s; +} + +int socket_mcast_join(const ot_ip6 group, const ot_ip6 interface, uint16_t port) { + struct ip_mreq mreq; + ot_ip6 any; + int s; + + /* The group is what is listened to, so the socket takes the wildcard and + the join names the interface to receive it on */ + memcpy(any, ip_v4mapped_prefix, sizeof(ip_v4mapped_prefix)); + memset(any + sizeof(ip_v4mapped_prefix), 0, 4); + + if ((s = socket_v4_bind(any, port)) == -1) + return -1; + + memset(&mreq, 0, sizeof(mreq)); + memcpy(&mreq.imr_multiaddr, group + sizeof(ip_v4mapped_prefix), 4); + memcpy(&mreq.imr_interface, interface + sizeof(ip_v4mapped_prefix), 4); + if (setsockopt(s, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) == -1) + return close_and_fail(s); + + return s; +} + +int socket_mcast_bind(const ot_ip6 interface, uint16_t port, int loopback) { + /* The BSDs and macOS want a byte for the IPv4 multicast options, Linux + accepts either that or an int. */ + unsigned char ttl = 1, loop = (unsigned char)(loopback != 0); + int s = socket_v4_bind(interface, port); + + if (s == -1) + return -1; + if (set_nonblocking(s) == -1) + return close_and_fail(s); + + if (setsockopt(s, IPPROTO_IP, IP_MULTICAST_TTL, &ttl, sizeof(ttl)) == -1 || + setsockopt(s, IPPROTO_IP, IP_MULTICAST_LOOP, &loop, sizeof(loop)) == -1) + return close_and_fail(s); + + return s; +} + +ssize_t socket_mcast_send(int s, const void *buf, size_t len, const ot_ip6 group, uint16_t port) { + struct sockaddr_in sa; + + sockaddr4_from_ip(&sa, group, port); + return sendto(s, buf, len, 0, (struct sockaddr *)&sa, sizeof(sa)); +} +#endif diff --git a/src/ot_socket.h b/src/ot_socket.h new file mode 100644 index 0000000..5c1d840 --- /dev/null +++ b/src/ot_socket.h @@ -0,0 +1,56 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_SOCKET_H__ +#define OT_SOCKET_H__ + +#include +#include + +#include "trackerlogic.h" + +/* Every socket opentracker opens is an AF_INET6 socket with IPV6_V6ONLY + cleared, so one descriptor serves both families and an IPv4 peer arrives + as the v4 mapped address */ + +/* How an address may be shared with other sockets. Which socket option says + so differs between kernels, which is socket_set_sharing's business. + + SHARED lets another socket bind the same address. BALANCED asks the kernel + to spread arriving datagrams over the sockets that did, which is a + different thing and not everywhere available: ask socket_can_balance + before binding a group that depends on it. */ +typedef enum { SOCKET_EXCLUSIVE, SOCKET_SHARED, SOCKET_BALANCED } socket_sharing; + +/* Whether this kernel spreads datagrams over a group of sockets bound to one + address. Where it does not, binding a group is worse than one socket with + several readers, so the caller wants to know first. */ +int socket_can_balance(void); + +/* Create, bind and, for TCP, listen. The returned socket is non blocking + Returns -1 on failure with errno set */ +int socket_tcp_listen(const ot_ip6 ip, uint16_t port, socket_sharing sharing); +int socket_udp_bind(const ot_ip6 ip, uint16_t port, socket_sharing sharing); + +int socket_tcp_connect(const ot_ip6 srcip, uint16_t srcport, const ot_ip6 dstip, uint16_t dstport); +int socket_is_connected(int s); +int socket_accept(int s, ot_ip6 ip, uint16_t *port); +int socket_set_blocking(int s, int blocking); +int socket_defer_accept(int s); + +ssize_t socket_recv(int s, void *buf, size_t len, ot_ip6 ip, uint16_t *port); +ssize_t socket_send(int s, const void *buf, size_t len, const ot_ip6 ip, uint16_t port); + +#ifdef WANT_SYNC_LIVE +/* + socket_mcast_join takes the wildcard address and names the interface in the + join, and hands back a blocking socket, since the thread that reads it does + nothing else. socket_mcast_bind hands back a non blocking one, because the + thread that sends from it is also serving clients. Incoming datagrams are + read with socket_recv */ +int socket_mcast_join(const ot_ip6 group, const ot_ip6 interface, uint16_t port); +int socket_mcast_bind(const ot_ip6 interface, uint16_t port, int loopback); +ssize_t socket_mcast_send(int s, const void *buf, size_t len, const ot_ip6 group, uint16_t port); +#endif + +#endif diff --git a/src/ot_stats.c b/src/ot_stats.c new file mode 100644 index 0000000..2d6335b --- /dev/null +++ b/src/ot_stats.c @@ -0,0 +1,866 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef WANT_SYSLOGS +#include +#endif + +/* Opentracker */ +#include "ot_accesslist.h" +#include "ot_event.h" +#include "ot_hex.h" +#include "ot_iovec.h" +#include "ot_ip.h" +#include "ot_mutex.h" +#include "ot_workqueue.h" +#include "ot_fmt.h" +#include "ot_stats.h" +#include "trackerlogic.h" + +#ifndef NO_FULLSCRAPE_LOGGING +#define LOG_TO_STDERR(...) fprintf(stderr, __VA_ARGS__) +#else +#define LOG_TO_STDERR(...) +#endif + +/* Forward declaration */ +static void stats_make(size_t *iovec_entries, struct iovec **iovector, ot_tasktype mode); +#define OT_STATS_TMPSIZE 8192 + +/* Each serving thread keeps its own statistics tally. The list is linked + through intrusive next pointers within the workstruct. + + Registration happens once per thread and the adding up once per statistics + request, so this lock is never on anybody's hot path. */ +static struct ot_workstruct *g_counter_contexts; +static ot_counter g_retired[OT_COUNTER_COUNT]; +static pthread_mutex_t g_counter_mutex = PTHREAD_MUTEX_INITIALIZER; + +void stats_counters_register(struct ot_workstruct *ws) { + pthread_mutex_lock(&g_counter_mutex); + ws->counters_next = g_counter_contexts; + g_counter_contexts = ws; + pthread_mutex_unlock(&g_counter_mutex); +} + +void stats_counters_retire(struct ot_workstruct *ws) { + struct ot_workstruct **link; + size_t i; + + pthread_mutex_lock(&g_counter_mutex); + for (link = &g_counter_contexts; *link; link = &(*link)->counters_next) + if (*link == ws) { + *link = ws->counters_next; + break; + } + + /* When retiring a worker thread, collect the counter tally in the global */ + for (i = 0; i < OT_COUNTER_COUNT; ++i) + atomic_fetch_add_explicit(g_retired + i, atomic_load_explicit(ws->counters + i, memory_order_relaxed), memory_order_relaxed); + pthread_mutex_unlock(&g_counter_mutex); +} + +/* What every thread has counted so far, added together */ +static void counters_total(unsigned long long *total) { + struct ot_workstruct *ws; + size_t i; + + pthread_mutex_lock(&g_counter_mutex); + for (i = 0; i < OT_COUNTER_COUNT; ++i) + total[i] = atomic_load_explicit(g_retired + i, memory_order_relaxed); + for (ws = g_counter_contexts; ws; ws = ws->counters_next) + for (i = 0; i < OT_COUNTER_COUNT; ++i) + total[i] += atomic_load_explicit(ws->counters + i, memory_order_relaxed); + pthread_mutex_unlock(&g_counter_mutex); +} + +#define COUNT(index, by) atomic_fetch_add_explicit(ws->counters + (index), (unsigned long long)(by), memory_order_relaxed) + +static const char *ot_failed_request_names[] = { + "302 Redirect", "400 Parse Error", "400 Invalid Parameter", "400 Invalid Parameter (compact=0)", "400 Not Modest", + "402 Payment Required", "403 Access Denied", "404 Not found", "500 Internal Server Error"}; + +static time_t ot_start_time; + +#define STATS_NETWORK_NODE_BITWIDTH 4 +#define STATS_NETWORK_NODE_COUNT (1 << STATS_NETWORK_NODE_BITWIDTH) + +#define __BYTE(P, D) (((uint8_t *)P)[D / 8]) +#define __MSK (STATS_NETWORK_NODE_COUNT - 1) +#define __SHFT(D) ((D ^ STATS_NETWORK_NODE_BITWIDTH) & STATS_NETWORK_NODE_BITWIDTH) + +#define __LDR(P, D) ((__BYTE((P), (D)) >> __SHFT((D))) & __MSK) +#define __STR(P, D, V) __BYTE((P), (D)) = (__BYTE((P), (D)) & ~(__MSK << __SHFT((D)))) | ((V) << __SHFT((D))) + +#if 0 +// XXX +#define STATS_NETWORK_NODE_MAXDEPTH (68 - STATS_NETWORK_NODE_BITWIDTH) +#define STATS_NETWORK_NODE_LIMIT (48 - STATS_NETWORK_NODE_BITWIDTH) +#endif +#define STATS_NETWORK_NODE_MAXDEPTH (28 - STATS_NETWORK_NODE_BITWIDTH) +#define STATS_NETWORK_NODE_LIMIT (24 - STATS_NETWORK_NODE_BITWIDTH) + +typedef union stats_network_node stats_network_node; +union stats_network_node { + size_t counters[STATS_NETWORK_NODE_COUNT]; + stats_network_node *children[STATS_NETWORK_NODE_COUNT]; +}; + +static int stat_increase_network_count(stats_network_node **pnode, int depth, uintptr_t ip) { + int foo = __LDR(ip, depth); + stats_network_node *node; + + if (!*pnode) { + *pnode = malloc(sizeof(stats_network_node)); + if (!*pnode) + return -1; + memset(*pnode, 0, sizeof(stats_network_node)); + } + node = *pnode; + + if (depth < STATS_NETWORK_NODE_MAXDEPTH) + return stat_increase_network_count(node->children + foo, depth + STATS_NETWORK_NODE_BITWIDTH, ip); + + node->counters[foo]++; + return 0; +} + +static int stats_shift_down_network_count(stats_network_node **node, int depth, int shift) { + size_t i, rest = 0; + + if (!*node) + return 0; + + for (i = 0; i < STATS_NETWORK_NODE_COUNT; ++i) + if (depth < STATS_NETWORK_NODE_MAXDEPTH) + rest += stats_shift_down_network_count((*node)->children + i, depth + STATS_NETWORK_NODE_BITWIDTH, shift); + else + rest += (*node)->counters[i] >>= shift; + + if (!rest) { + free(*node); + *node = NULL; + } + + return rest; +} + +static size_t stats_get_highscore_networks(stats_network_node *node, int depth, ot_ip6 node_value, size_t *scores, ot_ip6 *networks, int network_count, + int limit) { + size_t i, score = 0; + + if (!node) + return 0; + + if (depth < limit) { + for (i = 0; i < STATS_NETWORK_NODE_COUNT; ++i) + if (node->children[i]) { + __STR(node_value, depth, i); + score += stats_get_highscore_networks(node->children[i], depth + STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit); + } + return score; + } + + if (depth > limit && depth < STATS_NETWORK_NODE_MAXDEPTH) { + for (i = 0; i < STATS_NETWORK_NODE_COUNT; ++i) + if (node->children[i]) + score += stats_get_highscore_networks(node->children[i], depth + STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit); + return score; + } + + if (depth > limit && depth == STATS_NETWORK_NODE_MAXDEPTH) { + for (i = 0; i < STATS_NETWORK_NODE_COUNT; ++i) + score += node->counters[i]; + return score; + } + + /* if( depth == limit ) */ + for (i = 0; i < STATS_NETWORK_NODE_COUNT; ++i) { + int j = 1; + size_t node_score; + + if (depth == STATS_NETWORK_NODE_MAXDEPTH) + node_score = node->counters[i]; + else + node_score = stats_get_highscore_networks(node->children[i], depth + STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit); + + score += node_score; + + if (node_score <= scores[0]) + continue; + + __STR(node_value, depth, i); + while (j < network_count && node_score > scores[j]) + ++j; + --j; + + memmove(scores, scores + 1, j * sizeof(*scores)); + memmove(networks, networks + 1, j * sizeof(*networks)); + scores[j] = node_score; + memcpy(networks + j, node_value, sizeof(*networks)); + } + + return score; +} + +static void stats_return_busy_networks(char **reply, const char *reply_end, stats_network_node *tree, size_t amount, int limit) { + ot_ip6 networks[amount]; + ot_ip6 node_value; + size_t scores[amount]; + size_t i; + + memset(scores, 0, sizeof(scores)); + memset(networks, 0, sizeof(networks)); + memset(node_value, 0, sizeof(node_value)); + + stats_get_highscore_networks(tree, 0, node_value, scores, networks, amount, limit); + + fmt_append(reply, reply_end, "Networks, limit /%d:\n", limit + STATS_NETWORK_NODE_BITWIDTH); + for (i = amount; i > 0; --i) { + if (scores[i - 1]) { + char ip_readable[64]; + + ip_readable[ip_format(ip_readable, networks[i - 1])] = 0; + fmt_append(reply, reply_end, "%08zd: %s\n", scores[i - 1], ip_readable); + } + } + fmt_append(reply, reply_end, "\n"); +} + +static void stats_slash24s_txt(char **reply, const char *reply_end, size_t amount) { + stats_network_node *slash24s_network_counters_root = NULL; + size_t i, bucket, peer_size = OT_PEER_SIZE4; + + for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { + ot_vector *torrents_list = mutex_bucket_lock(bucket); + for (i = 0; i < torrents_list->size; ++i) { + ot_peerlist *peer_list = (((ot_torrent *)(torrents_list->data))[i]).peer_list4; + ot_vector *bucket_list = &peer_list->peers; + int num_buckets = 1; + + if (OT_PEERLIST_HASBUCKETS(peer_list)) { + num_buckets = bucket_list->size; + bucket_list = (ot_vector *)bucket_list->data; + } + + while (num_buckets--) { + ot_peer *peers = (ot_peer *)bucket_list->data; + size_t numpeers = bucket_list->size; + while (numpeers--) { + if (stat_increase_network_count(&slash24s_network_counters_root, 0, (uintptr_t)(peers))) + goto bailout_unlock; + peers += peer_size; + } + ++bucket_list; + } + } + mutex_bucket_unlock(bucket, 0); + if (!g_opentracker_running) + goto done; + } + + /* The tree is built. Now analyze */ + stats_return_busy_networks(reply, reply_end, slash24s_network_counters_root, amount, STATS_NETWORK_NODE_MAXDEPTH); + stats_return_busy_networks(reply, reply_end, slash24s_network_counters_root, amount, STATS_NETWORK_NODE_LIMIT); + goto done; + +bailout_unlock: + mutex_bucket_unlock(bucket, 0); +done: + stats_shift_down_network_count(&slash24s_network_counters_root, 0, sizeof(int) * 8 - 1); +} + +#ifdef WANT_SPOT_WOODPECKER +static stats_network_node *stats_woodpeckers_tree; +static pthread_mutex_t g_woodpeckers_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void stats_return_woodpeckers(char **reply, const char *reply_end, int amount) { + pthread_mutex_lock(&g_woodpeckers_mutex); + stats_return_busy_networks(reply, reply_end, stats_woodpeckers_tree, amount, STATS_NETWORK_NODE_MAXDEPTH); + pthread_mutex_unlock(&g_woodpeckers_mutex); +} +#endif + +typedef struct { + unsigned long long torrent_count; + unsigned long long peer_count; + unsigned long long seed_count; +} torrent_stats; + +static int torrent_statter(ot_torrent *torrent, uintptr_t data) { + torrent_stats *stats = (torrent_stats *)data; + stats->torrent_count++; + stats->peer_count += torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; + stats->seed_count += torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; + return 0; +} + +/* Converter function from memory to human readable hex strings */ +typedef struct { + size_t val; + ot_hash hash; +} ot_record; + +static inline void stats_insert_into_highscore(size_t val, ot_record* list, size_t amount, ot_hash* hash) { + size_t i; + for (i = 0; (i < amount) && (list[i].val > val); ++i) + {} + /* if we score better than a record in the list, we need to make room in the high + score array by moving lower scoring entries up */ + if (i < amount) { + memmove(list + i + 1, list + i, (amount - i - 1) * sizeof(ot_record)); + + memcpy(&list[i].hash, hash, sizeof(ot_hash)); + list[i].val = val; + } +} + + +/* Fetches stats from tracker */ +void stats_top_txt(char **reply, const char *reply_end, size_t amount) { + size_t j, idx, bucket; + ot_record top100s[100], top100c[100], top100l[100]; + char hex_out[42]; + + if (amount > 100) + amount = 100; + + memset(top100s, 0, sizeof(top100s)); + memset(top100c, 0, sizeof(top100c)); + memset(top100l, 0, sizeof(top100l)); + + /* Iterate over complete torrent list */ + for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { + ot_vector *torrents_list = mutex_bucket_lock(bucket); + for (j = 0; j < torrents_list->size; ++j) { + ot_torrent *torrent = (ot_torrent *)(torrents_list->data) + j; + size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; + size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; + size_t leech_count = peer_count - seed_count; + + stats_insert_into_highscore(peer_count, top100c, amount, &torrent->hash); + stats_insert_into_highscore(seed_count, top100s, amount, &torrent->hash); + stats_insert_into_highscore(leech_count, top100l, amount, &torrent->hash); + } + mutex_bucket_unlock(bucket, 0); + if (!g_opentracker_running) + return; + } + + fmt_append(reply, reply_end, "Top %zd torrents by peers:\n", amount); + for (idx = 0; idx < amount; ++idx) + if (top100c[idx].val) + fmt_append(reply, reply_end, "\t%zd\t%s\n", top100c[idx].val, hex_from_hash(hex_out, top100c[idx].hash)); + fmt_append(reply, reply_end, "Top %zd torrents by seeds:\n", amount); + for (idx = 0; idx < amount; ++idx) + if (top100s[idx].val) + fmt_append(reply, reply_end, "\t%zd\t%s\n", top100s[idx].val, hex_from_hash(hex_out, top100s[idx].hash)); + fmt_append(reply, reply_end, "Top %zd torrents by leechers:\n", amount); + for (idx = 0; idx < amount; ++idx) + if (top100l[idx].val) + fmt_append(reply, reply_end, "\t%zd\t%s\n", top100l[idx].val, hex_from_hash(hex_out, top100l[idx].hash)); +} + +static unsigned long events_per_time(unsigned long long events, time_t t) { return events / ((unsigned int)t ? (unsigned int)t : 1); } + +static void stats_connections_mrtg(char **reply, const char *reply_end) { + ot_time t = time(NULL) - ot_start_time; + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker connections, %lu conns/s :: %lu success/s.", + total[OT_COUNTER_TCP_CONNECTIONS] + total[OT_COUNTER_UDP_CONNECTIONS], + total[OT_COUNTER_TCP_ANNOUNCES] + total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], (int)t, (int)(t / 3600), + events_per_time(total[OT_COUNTER_TCP_CONNECTIONS] + total[OT_COUNTER_UDP_CONNECTIONS], t), + events_per_time(total[OT_COUNTER_TCP_ANNOUNCES] + total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], t)); +} + +static void stats_udpconnections_mrtg(char **reply, const char *reply_end) { + ot_time t = time(NULL) - ot_start_time; + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker udp4 stats, %lu conns/s :: %lu success/s.", total[OT_COUNTER_UDP_CONNECTIONS], + total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], (int)t, (int)(t / 3600), + events_per_time(total[OT_COUNTER_UDP_CONNECTIONS], t), + events_per_time(total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], t)); +} + +static void stats_tcpconnections_mrtg(char **reply, const char *reply_end) { + time_t t = time(NULL) - ot_start_time; + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker tcp4 stats, %lu conns/s :: %lu success/s.", total[OT_COUNTER_TCP_CONNECTIONS], + total[OT_COUNTER_TCP_ANNOUNCES], (int)t, (int)(t / 3600), events_per_time(total[OT_COUNTER_TCP_CONNECTIONS], t), + events_per_time(total[OT_COUNTER_TCP_ANNOUNCES], t)); +} + +static void stats_scrape_mrtg(char **reply, const char *reply_end) { + time_t t = time(NULL) - ot_start_time; + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker scrape stats, %lu scrape/s (tcp and udp)", + total[OT_COUNTER_TCP_SCRAPES], total[OT_COUNTER_UDP_SCRAPES], (int)t, (int)(t / 3600), + events_per_time((total[OT_COUNTER_TCP_SCRAPES] + total[OT_COUNTER_UDP_SCRAPES]), t)); +} + +static void stats_fullscrapes_mrtg(char **reply, const char *reply_end) { + ot_time t = time(NULL) - ot_start_time; + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker full scrape stats, %lu conns/s :: %lu bytes/s.", + total[OT_COUNTER_FULLSCRAPES] * 1000, total[OT_COUNTER_FULLSCRAPE_BYTES], (int)t, (int)(t / 3600), + events_per_time(total[OT_COUNTER_FULLSCRAPES], t), + events_per_time(total[OT_COUNTER_FULLSCRAPE_BYTES], t)); +} + +static void stats_peers_mrtg(char **reply, const char *reply_end) { + torrent_stats stats = {0, 0, 0}; + + iterate_all_torrents(torrent_statter, (uintptr_t)&stats); + + fmt_append(reply, reply_end, "%llu\n%llu\nopentracker serving %llu torrents\nopentracker", stats.peer_count, stats.seed_count, + stats.torrent_count); +} + +static void stats_torrents_mrtg(char **reply, const char *reply_end) { + size_t torrent_count = mutex_get_torrent_count(); + + fmt_append(reply, reply_end, "%zd\n%zd\nopentracker serving %zd torrents\nopentracker", torrent_count, (size_t)0, torrent_count); +} + +static void stats_httperrors_txt(char **reply, const char *reply_end) { + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + fmt_append(reply, reply_end, "302 RED %llu\n400 ... %llu\n400 PAR %llu\n400 COM %llu\n403 IP %llu\n404 INV %llu\n500 SRV %llu\n", + total[OT_COUNTER_FAILED + 0], total[OT_COUNTER_FAILED + 1], total[OT_COUNTER_FAILED + 2], total[OT_COUNTER_FAILED + 3], + total[OT_COUNTER_FAILED + 4], total[OT_COUNTER_FAILED + 5], total[OT_COUNTER_FAILED + 6]); +} + +static void stats_return_renew_bucket(char **reply, const char *reply_end) { + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + int i; + + for (i = 0; i < OT_PEER_TIMEOUT; ++i) + fmt_append(reply, reply_end, "%02i %llu\n", i, total[OT_COUNTER_RENEWED + i]); +} + +static void stats_return_sync_mrtg(char **reply, const char *reply_end) { + ot_time t = time(NULL) - ot_start_time; + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker connections, %lu conns/s :: %lu success/s.", total[OT_COUNTER_SYNCS], 0LL, + (int)t, (int)(t / 3600), events_per_time(total[OT_COUNTER_TCP_CONNECTIONS] + total[OT_COUNTER_UDP_CONNECTIONS], t), + events_per_time(total[OT_COUNTER_TCP_ANNOUNCES] + total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], t)); +} + +static void stats_return_completed_mrtg(char **reply, const char *reply_end) { + ot_time t = time(NULL) - ot_start_time; + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + + fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker, %lu completed/h.", total[OT_COUNTER_COMPLETED], 0LL, (int)t, (int)(t / 3600), + events_per_time(total[OT_COUNTER_COMPLETED], t / 3600)); +} + +#ifdef WANT_LOG_NUMWANT +extern unsigned long long numwants[201]; +static void stats_return_numwants(char **reply, const char *reply_end) { + int i; + for (i = 0; i <= 200; ++i) + fmt_append(reply, reply_end, "%03d => %lld\n", i, numwants[i]); +} +#endif + +static void stats_return_everything(char **reply, const char *reply_end) { + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + torrent_stats stats = {0, 0, 0}; + int i; + + iterate_all_torrents(torrent_statter, (uintptr_t)&stats); + + fmt_append(reply, reply_end, "\n"); + fmt_append(reply, reply_end, "\n"); + fmt_append(reply, reply_end, " %llu\n", (unsigned long long)(time(NULL) - ot_start_time)); + fmt_append(reply, reply_end, " \n"); + fmt_append(reply, reply_end, " %zd\n", mutex_get_torrent_count()); + fmt_append(reply, reply_end, " %llu\n", stats.torrent_count); + fmt_append(reply, reply_end, " \n"); + fmt_append(reply, reply_end, " \n %llu\n \n", stats.peer_count); + fmt_append(reply, reply_end, " \n %llu\n \n", stats.seed_count); + fmt_append(reply, reply_end, " \n %llu\n \n", total[OT_COUNTER_COMPLETED]); + fmt_append(reply, reply_end, " \n"); + fmt_append(reply, reply_end, " \n %llu\n %llu\n %llu\n \n", + total[OT_COUNTER_TCP_CONNECTIONS], total[OT_COUNTER_TCP_ANNOUNCES], total[OT_COUNTER_TCP_SCRAPES]); + fmt_append(reply, reply_end, " \n %llu\n %llu\n %llu\n %llu\n %llu\n \n", + total[OT_COUNTER_UDP_CONNECTIONS], total[OT_COUNTER_UDP_CONNECTS], total[OT_COUNTER_UDP_ANNOUNCES], total[OT_COUNTER_UDP_SCRAPES], + total[OT_COUNTER_UDP_CONNID_MISSMATCH]); + fmt_append(reply, reply_end, " \n %llu\n \n", total[OT_COUNTER_SYNCS]); + fmt_append(reply, reply_end, " \n"); + fmt_append(reply, reply_end, " \n"); + fmt_append(reply, reply_end, " \n"); + for (i = 0; i < OT_PEER_TIMEOUT; ++i) + fmt_append(reply, reply_end, " %llu\n", i, total[OT_COUNTER_RENEWED + i]); + fmt_append(reply, reply_end, " \n"); + fmt_append(reply, reply_end, " \n"); + for (i = 0; i < CODE_HTTPERROR_COUNT; ++i) + fmt_append(reply, reply_end, " %llu\n", ot_failed_request_names[i], total[OT_COUNTER_FAILED + i]); + fmt_append(reply, reply_end, " \n"); + fmt_append(reply, reply_end, " \n %llu\n \n", total[OT_COUNTER_STALLS]); + fmt_append(reply, reply_end, " \n"); + fmt_append(reply, reply_end, ""); +} + +/* The names as the Makefile spells them, so that what a binary reports maps + straight back onto the FEATURES lines it was built from */ +static const char *const build_features[] = { +#ifdef WANT_V4_ONLY + "WANT_V4_ONLY", +#endif +#ifdef WANT_ACCESSLIST_BLACK + "WANT_ACCESSLIST_BLACK", +#endif +#ifdef WANT_ACCESSLIST_WHITE + "WANT_ACCESSLIST_WHITE", +#endif +#ifdef WANT_DYNAMIC_ACCESSLIST + "WANT_DYNAMIC_ACCESSLIST", +#endif +#ifdef WANT_SYNC_LIVE + "WANT_SYNC_LIVE", +#endif +#ifdef WANT_IP_FROM_QUERY_STRING + "WANT_IP_FROM_QUERY_STRING", +#endif +#ifdef WANT_IP_FROM_PROXY + "WANT_IP_FROM_PROXY", +#endif +#ifdef WANT_COMPRESSION_GZIP + "WANT_COMPRESSION_GZIP", +#endif +#ifdef WANT_COMPRESSION_GZIP_ALWAYS + "WANT_COMPRESSION_GZIP_ALWAYS", +#endif +#ifdef WANT_COMPRESSION_ZSTD + "WANT_COMPRESSION_ZSTD", +#endif +#ifdef WANT_COMPRESSION_ZSTD_ALWAYS + "WANT_COMPRESSION_ZSTD_ALWAYS", +#endif +#ifdef WANT_FULLSCRAPE + "WANT_FULLSCRAPE", +#endif +#ifdef WANT_MODEST_FULLSCRAPES + "WANT_MODEST_FULLSCRAPES", +#endif +#ifdef WANT_KEEPALIVE + "WANT_KEEPALIVE", +#endif +#ifdef WANT_RESTRICT_STATS + "WANT_RESTRICT_STATS", +#endif +#ifdef WANT_LOG_NUMWANT + "WANT_LOG_NUMWANT", +#endif +#ifdef WANT_SPOT_WOODPECKER + "WANT_SPOT_WOODPECKER", +#endif +#ifdef WANT_SYSLOGS + "WANT_SYSLOGS", +#endif +#ifdef WANT_DEV_RANDOM + "WANT_DEV_RANDOM", +#endif +#ifdef WANT_INSECURE_RANDOM_FALLBACK + "WANT_INSECURE_RANDOM_FALLBACK", +#endif +#ifdef _DEBUG + "_DEBUG", +#endif +#ifdef _DEBUG_HTTPERROR + "_DEBUG_HTTPERROR", +#endif +#ifdef _DEBUG_RANDOMTORRENTS + "_DEBUG_RANDOMTORRENTS", +#endif + NULL}; + +void stats_return_tracker_features(char **reply, const char *reply_end) { + size_t i; + + fmt_append(reply, reply_end, "events: %s\n", event_mechanism()); + fmt_append(reply, reply_end, "features:"); + for (i = 0; build_features[i]; ++i) + fmt_append(reply, reply_end, " %s", build_features[i]); + if (!build_features[0]) + fmt_append(reply, reply_end, " none"); + fmt_append(reply, reply_end, "\n"); +} + +void stats_return_tracker_version(char **reply, const char *reply_end) { + fmt_append(reply, reply_end, "opentracker %s", OT_VERSION); + if (*GIT_VERSION && strcmp(GIT_VERSION, OT_TAG)) + fmt_append(reply, reply_end, " (%s)", GIT_VERSION); + fmt_append(reply, reply_end, "\n"); + if (*OT_TAG) + fmt_append(reply, reply_end, "https://erdgeist.org/gitweb/opentracker/tag/?h=%s\n", OT_TAG); +} + +void return_stats_for_tracker(char **reply, const char *reply_end, int mode, int format) { + (void)format; + switch (mode & TASK_TASK_MASK) { + case TASK_STATS_CONNS: + stats_connections_mrtg(reply, reply_end); + break; + case TASK_STATS_SCRAPE: + stats_scrape_mrtg(reply, reply_end); + break; + case TASK_STATS_UDP: + stats_udpconnections_mrtg(reply, reply_end); + break; + case TASK_STATS_TCP: + stats_tcpconnections_mrtg(reply, reply_end); + break; + case TASK_STATS_FULLSCRAPE: + stats_fullscrapes_mrtg(reply, reply_end); + break; + case TASK_STATS_COMPLETED: + stats_return_completed_mrtg(reply, reply_end); + break; + case TASK_STATS_HTTPERRORS: + stats_httperrors_txt(reply, reply_end); + break; + case TASK_STATS_RENEW: + stats_return_renew_bucket(reply, reply_end); + break; + case TASK_STATS_SYNCS: + stats_return_sync_mrtg(reply, reply_end); + break; +#ifdef WANT_LOG_NUMWANT + case TASK_STATS_NUMWANTS: + stats_return_numwants(reply, reply_end); + break; +#endif + default: + break; + } +} + +static void stats_make(size_t *iovec_entries, struct iovec **iovector, ot_tasktype mode) { + char *r; + const char *r_end; + + *iovec_entries = 0; + *iovector = NULL; + if (!(r = iovec_new_chunk(iovec_entries, iovector, OT_STATS_TMPSIZE))) + return; + r_end = r + OT_STATS_TMPSIZE; + + switch (mode & TASK_TASK_MASK) { + case TASK_STATS_TORRENTS: + stats_torrents_mrtg(&r, r_end); + break; + case TASK_STATS_PEERS: + stats_peers_mrtg(&r, r_end); + break; + case TASK_STATS_SLASH24S: + stats_slash24s_txt(&r, r_end, 128); + break; + case TASK_STATS_TOP10: + stats_top_txt(&r, r_end, 10); + break; + case TASK_STATS_TOP100: + r = iovec_next_chunk(iovec_entries, iovector, r, 4 * OT_STATS_TMPSIZE); + if (!r) + return; + r_end = r + 4 * OT_STATS_TMPSIZE; + stats_top_txt(&r, r_end, 100); + break; + case TASK_STATS_EVERYTHING: + r = iovec_next_chunk(iovec_entries, iovector, r, OT_STATS_TMPSIZE + 64 * OT_PEER_TIMEOUT); + if (!r) + return; + r_end = r + OT_STATS_TMPSIZE + 64 * OT_PEER_TIMEOUT; + stats_return_everything(&r, r_end); + break; +#ifdef WANT_SPOT_WOODPECKER + case TASK_STATS_WOODPECKERS: + stats_return_woodpeckers(&r, r_end, 128); + break; +#endif + default: + iovec_free(iovec_entries, iovector); + return; + } + iovec_trim_last(iovec_entries, iovector, r); +} + +void stats_issue_event(struct ot_workstruct *ws, ot_status_event event, PROTO_FLAG proto, uintptr_t event_data) { + if (!ws) + return; + + switch (event) { + case EVENT_ACCEPT: + if (proto == FLAG_TCP) + COUNT(OT_COUNTER_TCP_CONNECTIONS, 1); + else + COUNT(OT_COUNTER_UDP_CONNECTIONS, 1); + break; + case EVENT_ANNOUNCE: + if (proto == FLAG_TCP) + COUNT(OT_COUNTER_TCP_ANNOUNCES, 1); + else + COUNT(OT_COUNTER_UDP_ANNOUNCES, 1); + break; + case EVENT_CONNECT: + if (proto == FLAG_TCP) + COUNT(OT_COUNTER_TCP_CONNECTS, 1); + else + COUNT(OT_COUNTER_UDP_CONNECTS, 1); + break; + case EVENT_COMPLETED: +#ifdef WANT_SYSLOGS + { + char timestring[64]; + char hash_hex[42], peerid_hex[42], ip_readable[64]; + struct tm time_now; + time_t ttt; + + time(&ttt); + localtime_r(&ttt, &time_now); + strftime(timestring, sizeof(timestring), "%FT%T%z", &time_now); + + hex_from_hash(hash_hex, *ws->hash); + if (ws->peer_id) + hex_from_hash(peerid_hex, (uint8_t *)ws->peer_id); + else { + *peerid_hex = 0; + } + + ip_readable[ip_format(ip_readable, (char *)&ws->peer)] = 0; + syslog(LOG_INFO, "time=%s event=completed info_hash=%s peer_id=%s ip=%s", timestring, hash_hex, peerid_hex, ip_readable); + } +#endif + COUNT(OT_COUNTER_COMPLETED, 1); + break; + case EVENT_SCRAPE: + if (proto == FLAG_TCP) + COUNT(OT_COUNTER_TCP_SCRAPES, 1); + else + COUNT(OT_COUNTER_UDP_SCRAPES, 1); + break; + case EVENT_FULLSCRAPE: + COUNT(OT_COUNTER_FULLSCRAPES, 1); + COUNT(OT_COUNTER_FULLSCRAPE_BYTES, event_data); + break; + case EVENT_FULLSCRAPE_REQUEST: { + ot_ip6 *ip = (ot_ip6 *)event_data; /* ugly hack to transfer ip to stats */ + char _debug[512]; + int off = snprintf(_debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(time_now_seconds() - ot_start_time) / 60); + off += ip_format(_debug + off, *ip); + off += snprintf(_debug + off, sizeof(_debug) - off, " - FULL SCRAPE\n"); + (void)write(2, _debug, off); + COUNT(OT_COUNTER_FULLSCRAPE_REQUESTS, 1); + } break; + case EVENT_FULLSCRAPE_REQUEST_GZIP: { + ot_ip6 *ip = (ot_ip6 *)event_data; /* ugly hack to transfer ip to stats */ + char _debug[512]; + int off = snprintf(_debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(time_now_seconds() - ot_start_time) / 60); + off += ip_format(_debug + off, *ip); + off += snprintf(_debug + off, sizeof(_debug) - off, " - FULL SCRAPE\n"); + (void)write(2, _debug, off); + COUNT(OT_COUNTER_FULLSCRAPE_REQUESTS, 1); + } break; + case EVENT_FAILED: + COUNT(OT_COUNTER_FAILED + event_data, 1); + break; + case EVENT_RENEW: + COUNT(OT_COUNTER_RENEWED + event_data, 1); + break; + case EVENT_SYNC: + COUNT(OT_COUNTER_SYNCS, event_data); + break; + case EVENT_BUCKET_LOCKED: + COUNT(OT_COUNTER_STALLS, 1); + break; +#ifdef WANT_SPOT_WOODPECKER + case EVENT_WOODPECKER: + pthread_mutex_lock(&g_woodpeckers_mutex); + stat_increase_network_count(&stats_woodpeckers_tree, 0, event_data); + pthread_mutex_unlock(&g_woodpeckers_mutex); + break; +#endif + case EVENT_CONNID_MISSMATCH: + COUNT(OT_COUNTER_UDP_CONNID_MISSMATCH, 1); + default: + break; + } +} + +void stats_cleanup(void) { +#ifdef WANT_SPOT_WOODPECKER + pthread_mutex_lock(&g_woodpeckers_mutex); + stats_shift_down_network_count(&stats_woodpeckers_tree, 0, 1); + pthread_mutex_unlock(&g_woodpeckers_mutex); +#endif +} + +static void *stats_worker(void *args) { + size_t iovec_entries; + struct iovec *iovector; + + (void)args; + + while (g_opentracker_running) { + ot_tasktype tasktype = TASK_STATS; + ot_taskid taskid = workqueue_poptask(&tasktype); + if (!taskid) + break; + stats_make(&iovec_entries, &iovector, tasktype); + if (workqueue_pushresult(taskid, iovec_entries, iovector)) + iovec_free(&iovec_entries, &iovector); + } + return NULL; +} + +void stats_deliver(int64_t sock, int tasktype) { workqueue_pushtask(sock, tasktype); } + +static pthread_t thread_id; +void stats_init(void) { + ot_start_time = time_now_seconds(); + pthread_create(&thread_id, NULL, stats_worker, NULL); +} + +void stats_deinit(void) { + workqueue_wakeall(); + pthread_join(thread_id, NULL); +} diff --git a/src/ot_stats.h b/src/ot_stats.h new file mode 100644 index 0000000..45a547e --- /dev/null +++ b/src/ot_stats.h @@ -0,0 +1,40 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_STATS_H__ +#define OT_STATS_H__ + +#include "trackerlogic.h" + +typedef enum { + EVENT_ACCEPT, + EVENT_READ, + EVENT_CONNECT, /* UDP only */ + EVENT_ANNOUNCE, + EVENT_COMPLETED, + EVENT_RENEW, + EVENT_SYNC, + EVENT_SCRAPE, + EVENT_FULLSCRAPE_REQUEST, + EVENT_FULLSCRAPE_REQUEST_GZIP, + EVENT_FULLSCRAPE_REQUEST_ZSTD, + EVENT_FULLSCRAPE, /* TCP only */ + EVENT_FAILED, + EVENT_BUCKET_LOCKED, + EVENT_WOODPECKER, + EVENT_CONNID_MISSMATCH +} ot_status_event; + +void stats_counters_register(struct ot_workstruct *ws); +void stats_counters_retire(struct ot_workstruct *ws); + +void stats_issue_event(struct ot_workstruct *ws, ot_status_event event, PROTO_FLAG proto, uintptr_t event_data); +void stats_deliver(int64_t sock, int tasktype); +void stats_cleanup(void); +void return_stats_for_tracker(char **reply, const char *reply_end, int mode, int format); +void stats_return_tracker_version(char **reply, const char *reply_end); +void stats_return_tracker_features(char **reply, const char *reply_end); +void stats_init(void); +void stats_deinit(void); + +#endif diff --git a/src/ot_tcp.c b/src/ot_tcp.c new file mode 100644 index 0000000..1351b92 --- /dev/null +++ b/src/ot_tcp.c @@ -0,0 +1,180 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_batch.h" +#include "ot_buf.h" +#include "ot_event.h" +#include "ot_fdstate.h" +#include "ot_http.h" +#include "ot_mutex.h" +#include "ot_socket.h" +#include "ot_worker.h" +#include "ot_stats.h" +#include "ot_tcp.h" +#include "trackerlogic.h" + +static ssize_t header_complete(char *request, ssize_t byte_count) { + ssize_t i = 0, state = 0; + + for (i = 1; i < byte_count; i += 2) + if (request[i] <= 13) { + i--; + for (state = 0; i < byte_count; ++i) { + char c = request[i]; + if (c == '\r' || c == '\n') + state = (state >> 2) | ((c << 6) & 0xc0); + else + break; + if (state >= 0xa0 || state == 0x99) + return i + 1; + } + } + return 0; +} + +void tcp_read(const int64_t sock, struct ot_workstruct *ws) { + struct http_data *cookie = fdstate_cookie(sock); + ssize_t byte_count = read((int)sock, ws->inbuf, G_INBUF_SIZE); + + if (byte_count == 0 || (byte_count == -1 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR)) { + http_close(sock); + return; + } + + if (byte_count == -1) + return; + + ws->request = NULL; + ws->request_size = 0; + ws->header_size = 0; + ws->keep_alive = 0; + + /* If we get the whole request in one packet, handle it without copying */ + if (!cookie->request.data) { + if ((ws->header_size = header_complete(ws->inbuf, byte_count))) { + ws->request = ws->inbuf; + ws->request_size = byte_count; + http_handle_request(sock, ws); + } else if (buf_cat(&cookie->request, ws->inbuf, (size_t)byte_count)) + http_close(sock); + return; + } + + if (buf_cat(&cookie->request, ws->inbuf, (size_t)byte_count) || cookie->request.size > G_INBUF_SIZE) { + http_issue_error(sock, ws, CODE_HTTPERROR_500); + return; + } + + while ((ws->header_size = header_complete(cookie->request.data, (ssize_t)cookie->request.size))) { + ws->request = cookie->request.data; + ws->request_size = (ssize_t)cookie->request.size; + http_handle_request(sock, ws); +#ifdef WANT_KEEPALIVE + if (!ws->keep_alive) +#endif + return; + } +} + +void tcp_write(const int64_t sock) { + struct http_data *cookie = fdstate_cookie(sock); + int chunked = 0; + int keep_alive = 0; + + if (cookie) { + chunked = (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) != 0; + keep_alive = (cookie->flag & STRUCT_HTTP_FLAG_KEEPALIVE) != 0; + + if (cookie->batch) { + if (batch_send(sock, cookie->batch) == BATCH_FAILED) { + http_close(sock); + return; + } + if (batch_pending(cookie->batch)) + return; + } + } + + /* In a chunked transfer, wait for the next piece once everything queued has gone out */ + if (chunked) + http_interest_set(sock, cookie, 0); + else if (keep_alive) { + /* The reply is out and the client asked to keep the connection, so go + back to waiting for the next request */ + http_interest_set(sock, cookie, EVENT_IN); + fdstate_set_expiry(sock, time_future_seconds(OT_CLIENT_TIMEOUT)); + } else + http_close(sock); +} + +/* The fast path, assuming that the complete request has usually arrived by the time a + connection is accepted, so it is answered here outside the event loop. The more load + a tracker has, the more likely it is that by the time it accepts, the roundtrip to + the peer has completed. + If the request could not be completely handled, the stack cookie is promoted to heap + and the connection handed over to the event loop to finish */ +void tcp_accept(const int64_t serversocket, struct ot_workstruct *ws) { + int64_t sock; + ot_ip6 ip; + uint16_t port; + + while ((sock = socket_accept(serversocket, ip, &port)) != -1) { + struct http_data cookie; + uint32_t generation; + + memset(&cookie, 0, sizeof(cookie)); + memcpy(cookie.ip, ip, sizeof(ot_ip6)); + cookie.interest = EVENT_IN; + cookie.flag = STRUCT_HTTP_FLAG_ONSTACK; + + generation = fdstate_open(sock, &cookie, time_future_seconds(OT_CLIENT_TIMEOUT)); + if (!fdstate_claim((int)sock, generation)) { + fdstate_close(sock); + event_close((int)sock); + continue; + } + + stats_issue_event(ws, EVENT_ACCEPT, FLAG_TCP, (uintptr_t)ip); + + tcp_read(sock, ws); + + /* Gone already if it was answered and closed inline */ + if (fdstate_generation((int)sock) != generation) + continue; + + /* Handling it may already have given it a heap home and handed it to + another thread, and then it is no longer this frame's to touch */ + if (fdstate_cookie((int)sock) != &cookie) + fdstate_release((int)sock); + else if (!http_adopt(sock)) + http_close(sock); + else + fdstate_release((int)sock); + } +} + +/* A worker blocks in accept and nowhere else: the connection it is handed is + non blocking, so a client that stops talking is given to the event loop + rather than keeping this thread. */ +static void tcp_run(struct ot_worker *me) { + while (g_opentracker_running) + tcp_accept(me->sock, &me->ws); +} + +void tcp_worker_install(int64_t sock) { + socket_set_blocking((int)sock, 1); + socket_defer_accept((int)sock); + + if (worker_start(tcp_run, sock) == -1) + fprintf(stderr, "Warning: could not install a tcp worker, this socket stays with the workers already running.\n"); +} diff --git a/src/ot_tcp.h b/src/ot_tcp.h new file mode 100644 index 0000000..afed142 --- /dev/null +++ b/src/ot_tcp.h @@ -0,0 +1,24 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_TCP_H__ +#define OT_TCP_H__ + +#include + +#include "trackerlogic.h" + +/* Take whatever connections are waiting on a listening socket and answer + them, each on this thread's stack, handing a connection that cannot be + finished to the event loop */ +void tcp_accept(const int64_t serversocket, struct ot_workstruct *ws); + +/* Handler connections in the event loop has become readable or writable */ +void tcp_read(const int64_t sock, struct ot_workstruct *ws); +void tcp_write(const int64_t sock); + +/* One thread accepting on one socket. Called once per worker. */ +void tcp_worker_install(int64_t sock); + + +#endif diff --git a/src/ot_udp.c b/src/ot_udp.c new file mode 100644 index 0000000..644b71d --- /dev/null +++ b/src/ot_udp.c @@ -0,0 +1,226 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_ip.h" +#include "ot_random.h" +#include "ot_rijndael.h" +#include "ot_socket.h" +#include "ot_worker.h" +#include "ot_stats.h" +#include "ot_udp.h" +#include "trackerlogic.h" + +static uint32_t g_rijndael_round_key[44] = {0}; +static volatile uint32_t g_key_of_the_hour[2] = {0}; +static volatile ot_time g_hour_of_the_key; + + +static void udp_generate_rijndael_round_key(void) { + uint32_t key[16]; + random_bytes(key, sizeof(key)); + rijndaelKeySetupEnc128(g_rijndael_round_key, (uint8_t *)key); + + g_key_of_the_hour[0] = random_uint32(); + g_hour_of_the_key = time_now_minutes(); +} + +/* Generate current and previous connection id for ip */ +static void udp_make_connectionid(uint32_t connid[2], const ot_ip6 remoteip, int age) { + uint32_t plain[4], crypt[4]; + int i; + uint32_t current_key_of_the_hour; + + if (time_now_minutes() - g_hour_of_the_key >= 60) { + uint32_t old_key_of_the_hour = g_key_of_the_hour[0]; + uint32_t new_key_of_the_hour = random_uint32(); + /* If in the meantime another thread has performed + key rotation, do not overwrite their results */ + if (time_now_minutes() - g_hour_of_the_key >= 60) { + /* Upgrade en bloc */ + g_hour_of_the_key = time_now_minutes(); + g_key_of_the_hour[0] = new_key_of_the_hour; + g_key_of_the_hour[1] = old_key_of_the_hour; + } + } + + memcpy(plain, remoteip, sizeof(plain)); + current_key_of_the_hour = g_key_of_the_hour[age]; + for (i = 0; i < 4; ++i) + plain[i] ^= current_key_of_the_hour; + rijndaelEncrypt128(g_rijndael_round_key, (uint8_t *)plain, (uint8_t *)crypt); + connid[0] = crypt[0] ^ crypt[1]; + connid[1] = crypt[2] ^ crypt[3]; +} + +/* UDP implementation according to http://xbtt.sourceforge.net/udp_tracker_protocol.html */ +int handle_udp6(int64_t serversocket, struct ot_workstruct *ws) { + ot_ip6 remoteip; + uint32_t *inpacket = (uint32_t *)ws->inbuf; + uint32_t *outpacket = (uint32_t *)ws->outbuf; + uint32_t left, event; + uint32_t connid[2]; + uint32_t action; + uint16_t port, remoteport; + size_t byte_count, scrape_count; + + ssize_t result = socket_recv(serversocket, ws->inbuf, G_INBUF_SIZE, remoteip, &remoteport); + if (result <= 0) + return 0; + byte_count = (size_t)result; + + stats_issue_event(ws, EVENT_ACCEPT, FLAG_UDP, (uintptr_t)remoteip); + stats_issue_event(ws, EVENT_READ, FLAG_UDP, byte_count); + + /* Minimum udp tracker packet size, also catches error */ + if (byte_count < 16) + return 1; + + /* Get action to take. Ignore error messages and broken packets */ + action = ntohl(inpacket[2]); + if (action > 2) + return 1; + + /* Generate the connection id we give out and expect to and from + the requesting ip address, this prevents udp spoofing */ + udp_make_connectionid(connid, remoteip, 0); + + /* Initialise hash pointer */ + ws->hash = NULL; + ws->peer_id = NULL; + + /* If action is not 0 (connect), then we expect the derived + connection id in first 64 bit */ + if ((action > 0) && (inpacket[0] != connid[0] || inpacket[1] != connid[1])) { + /* If connection id does not match, try the one that was + valid in the previous hour. Only if this also does not + match, return an error packet */ + udp_make_connectionid(connid, remoteip, 1); + if (inpacket[0] != connid[0] || inpacket[1] != connid[1]) { + const char err_idmiss[] = "Connection ID missmatch."; + size_t s = sizeof(err_idmiss); + outpacket[0] = htonl(3); + outpacket[1] = inpacket[3]; + memcpy(&outpacket[2], &err_idmiss[0], s); + socket_send(serversocket, ws->outbuf, 8 + s, remoteip, remoteport); + stats_issue_event(ws, EVENT_CONNID_MISSMATCH, FLAG_UDP, 8 + s); + return 1; + } + } + + switch (action) { + case 0: /* This is a connect action */ + /* look for udp bittorrent magic id */ + if ((ntohl(inpacket[0]) != 0x00000417) || (ntohl(inpacket[1]) != 0x27101980)) + return 1; + + outpacket[0] = 0; + outpacket[1] = inpacket[3]; + outpacket[2] = connid[0]; + outpacket[3] = connid[1]; + + socket_send(serversocket, ws->outbuf, 16, remoteip, remoteport); + stats_issue_event(ws, EVENT_CONNECT, FLAG_UDP, 16); + break; + case 1: /* This is an announce action */ + /* Minimum udp announce packet size */ + if (byte_count < 98) + return 1; + + /* We do only want to know, if it is zero */ + left = inpacket[64 / 4] | inpacket[68 / 4]; + + event = ntohl(inpacket[80 / 4]); + port = *(uint16_t *)(((char *)inpacket) + 96); + ws->hash = (ot_hash *)(((char *)inpacket) + 16); + + OT_SETIP(ws->peer, remoteip); + OT_SETPORT(ws->peer, &port); + OT_PEERFLAG(ws->peer) = 0; + + switch (event) { + case 1: + OT_PEERFLAG(ws->peer) |= PEER_FLAG_COMPLETED; + break; + case 3: + OT_PEERFLAG(ws->peer) |= PEER_FLAG_STOPPED; + break; + default: + break; + } + + if (!left) + OT_PEERFLAG(ws->peer) |= PEER_FLAG_SEEDING; + + outpacket[0] = htonl(1); /* announce action */ + outpacket[1] = inpacket[12 / 4]; + + if (OT_PEERFLAG(ws->peer) & PEER_FLAG_STOPPED) { /* Peer is gone. */ + ws->reply = ws->outbuf; + ws->reply_size = remove_peer_from_torrent(FLAG_UDP, ws); + } else { + /* Limit amount of peers to OT_MAX_PEERS_UDP */ + uint32_t numwant = ntohl(inpacket[92 / 4]); + size_t max_peers = ip_is_v4mapped(remoteip) ? OT_MAX_PEERS_UDP4 : OT_MAX_PEERS_UDP6; + if (numwant > max_peers) + numwant = max_peers; + + ws->reply = ws->outbuf + 8; + ws->reply_size = add_peer_to_torrent_and_return_peers(FLAG_UDP, ws, numwant); + + if (!ws->reply_size) { + const char err_unauth[] = "Requested download is not authorized for use with this tracker."; + size_t s = sizeof(err_unauth); + outpacket[0] = htonl(3); + memcpy(&outpacket[2], &err_unauth[0], s); + socket_send(serversocket, ws->outbuf, 8 + s, remoteip, remoteport); + return 1; + } + ws->reply_size += 8; + } + + socket_send(serversocket, ws->outbuf, ws->reply_size, remoteip, remoteport); + stats_issue_event(ws, EVENT_ANNOUNCE, FLAG_UDP, ws->reply_size); + break; + + case 2: /* This is a scrape action */ + outpacket[0] = htonl(2); /* scrape action */ + outpacket[1] = inpacket[12 / 4]; + + for (scrape_count = 0; (scrape_count * 20 < byte_count - 16) && (scrape_count <= 74); scrape_count++) + return_udp_scrape_for_torrent(*(ot_hash *)(((char *)inpacket) + 16 + 20 * scrape_count), ((char *)outpacket) + 8 + 12 * scrape_count); + + socket_send(serversocket, ws->outbuf, 8 + 12 * scrape_count, remoteip, remoteport); + stats_issue_event(ws, EVENT_SCRAPE, FLAG_UDP, scrape_count); + break; + } + return 1; +} + +static void udp_run(struct ot_worker *me) { + while (g_opentracker_running) + handle_udp6(me->sock, &me->ws); +} + +void udp_init(void) { + if (!g_rijndael_round_key[0]) + udp_generate_rijndael_round_key(); +} + +void udp_worker_install(int64_t sock) { +#ifdef _DEBUG + fprintf(stderr, "Installing a worker on udp socket %ld\n", (unsigned long)sock); +#endif + socket_set_blocking((int)sock, 1); + + if (worker_start(udp_run, sock) == -1) + fprintf(stderr, "Warning: could not install a udp worker, this socket stays with the workers already running.\n"); +} diff --git a/src/ot_udp.h b/src/ot_udp.h new file mode 100644 index 0000000..2710744 --- /dev/null +++ b/src/ot_udp.h @@ -0,0 +1,17 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_UDP_H__ +#define OT_UDP_H__ + +#include + +#include "trackerlogic.h" + +void udp_init(void); +/* One thread reading one socket. Called once per worker. */ +void udp_worker_install(int64_t sock); + +int handle_udp6(int64_t serversocket, struct ot_workstruct *ws); + +#endif diff --git a/src/ot_urlencoded.c b/src/ot_urlencoded.c new file mode 100644 index 0000000..4109bb5 --- /dev/null +++ b/src/ot_urlencoded.c @@ -0,0 +1,168 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* Opentracker */ +#include "ot_urlencoded.h" + +/* System */ +#include + +/* Idea is to do a in place replacement or guarantee at least + strlen( string ) bytes in deststring + watch http://www.ietf.org/rfc/rfc2396.txt + unreserved = alphanum | mark + mark = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")" + we add '%' to the matrix to not stop at encoded chars. + After losing too many requests to being too strict, add the following characters to reserved matrix + relax = "+" | "," | "/" | ";" | "<" | ">" | ":" +*/ + +/* This matrix holds for each ascii character the information, + whether it is a non-terminating character for one of the three + scan states we are in, that is 'path', 'param' and 'value' from + /path?param=value¶m=value, it is encoded in bit 0, 1 and 2 + respectively + + The top bit of lower nibble indicates, whether this character is + a hard terminator, ie. \0, \n or \s, where the whole scanning + process should terminate + */ +static const unsigned char is_unreserved[256] = { + 8,0,0,0,0,0,0,0,0,0,8,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 8,7,8,8,8,7,0,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,4,7,6, + 4,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,7, + 8,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,7,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +}; + +/* Do a fast nibble to hex representation conversion */ +static unsigned char fromhex(unsigned char x) { + x -= '0'; + if (x <= 9) + return x; + x &= ~0x20; + x -= 'A' - '0'; + if (x < 6) + return x + 10; + return 0xff; +} + +/* Skip the value of a param=value pair */ +void urlencoded_skip_value(char **string) { + const unsigned char *s = *(const unsigned char **)string; + unsigned char f; + + /* Since we are asked to skip the 'value', we assume to stop at + terminators for a 'value' string position */ + while ((f = is_unreserved[*s++]) & SCAN_SEARCHPATH_VALUE) + ; + + /* If we stopped at a hard terminator like \0 or \n, make the + next urlencoded_parse encounter it again */ + if (f & SCAN_SEARCHPATH_TERMINATOR) + --s; + + *string = (char *)s; +} + +int urlencoded_match_keyword(const ot_keywords *keywords, char **string, SCAN_SEARCHPATH_FLAG flags) { + char *deststring = *string; + ssize_t match_length = urlencoded_parse(string, deststring, flags); + + if (match_length < 0) + return match_length; + if (match_length == 0) + return -3; + + while (keywords->key) { + if (!strncmp(keywords->key, deststring, match_length) && !keywords->key[match_length]) + return keywords->value; + keywords++; + } + + return -3; +} + +ssize_t urlencoded_parse(char **string, char *deststring, SCAN_SEARCHPATH_FLAG flags) { + const unsigned char *s = *(const unsigned char **)string; + unsigned char *d = (unsigned char *)deststring; + unsigned char b, c; + + /* This is the main decoding loop. + 'flag' determines, which characters are non-terminating in current context + (ie. stop at '=' and '&' if scanning for a 'param'; stop at '?' if scanning for the path ) + */ + while (is_unreserved[c = *s++] & flags) { + + /* When encountering an url escaped character, try to decode */ + if (c == '%') { + if ((b = fromhex(*s++)) == 0xff) + return -1; + if ((c = fromhex(*s++)) == 0xff) + return -1; + c |= (b << 4); + } + + /* Write (possibly decoded) character to output */ + *d++ = c; + } + + switch (c) { + case 0: + case '\r': + case '\n': + case ' ': + /* If we started scanning on a hard terminator, indicate we've finished */ + if (d == (unsigned char *)deststring) + return -2; + + /* Else make the next call to urlencoded_parse encounter it again */ + --s; + break; + case '?': + if (flags != SCAN_PATH) + return -1; + break; + case '=': + if (flags != SCAN_SEARCHPATH_PARAM) + return -1; + break; + case '&': + if (flags == SCAN_PATH) + return -1; + if (flags == SCAN_SEARCHPATH_PARAM) + --s; + break; + default: + return -1; + } + + *string = (char *)s; + return d - (unsigned char *)deststring; +} + + +size_t urlencoded_format(char *dest, const uint8_t *src, size_t len) { + static const char safe[] = "$/.=~-_"; + static const char hexdigit[] = "0123456789abcdef"; + size_t written = 0, i; + + for (i = 0; i < len; ++i) { + uint8_t c = src[i]; + + /* strchr would find the terminator, so a zero byte never counts as safe */ + if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || (c && strchr(safe, c))) { + dest[written++] = (char)c; + } else { + dest[written] = '%'; + dest[written + 1] = hexdigit[c >> 4]; + dest[written + 2] = hexdigit[c & 15]; + written += 3; + } + } + + return written; +} diff --git a/src/ot_urlencoded.h b/src/ot_urlencoded.h new file mode 100644 index 0000000..9882468 --- /dev/null +++ b/src/ot_urlencoded.h @@ -0,0 +1,53 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef SCAN_URLENCODED_QUERY_H__ +#define SCAN_URLENCODED_QUERY_H__ + +#include +#include +#include + +typedef struct { + const char *key; + int value; +} ot_keywords; + +typedef enum { + SCAN_PATH = 1, + SCAN_SEARCHPATH_PARAM = 2, + SCAN_SEARCHPATH_VALUE = 4, + SCAN_SEARCHPATH_TERMINATOR = 8 +} SCAN_SEARCHPATH_FLAG; + +/* string in: pointer to source + out: pointer to next scan position + deststring pointer to destination + flags determines, what to parse + returns number of valid converted characters in deststring + or -1 for parse error + or -2 for terminator found +*/ +ssize_t urlencoded_parse(char **string, char *deststring, SCAN_SEARCHPATH_FLAG flags); + +/* string in: pointer to source + out: pointer to next scan position + flags determines, what to parse + returns value for matched keyword + or -1 for parse error + or -2 for terminator found + or -3 for no keyword matched + */ +int urlencoded_match_keyword(const ot_keywords *keywords, char **string, SCAN_SEARCHPATH_FLAG flags); + +/* string in: pointer to value of a param=value pair to skip + out: pointer to next scan position on return +*/ +void urlencoded_skip_value(char **string); + + +/* The other direction: percent encode len bytes from src into dest and return + how many characters were written. dest needs room for three times len. */ +size_t urlencoded_format(char *dest, const uint8_t *src, size_t len); + +#endif diff --git a/src/ot_vector.c b/src/ot_vector.c new file mode 100644 index 0000000..ab0566c --- /dev/null +++ b/src/ot_vector.c @@ -0,0 +1,287 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include + +/* Opentracker */ +#include "trackerlogic.h" + +static int vector_compare_peer6(const void *peer1, const void *peer2) { return memcmp(peer1, peer2, OT_PEER_COMPARE_SIZE6); } +static int vector_compare_peer4(const void *peer1, const void *peer2) { return memcmp(peer1, peer2, OT_PEER_COMPARE_SIZE4); } + +/* This function gives us a binary search that returns a pointer, even if + no exact match is found. In that case it sets exactmatch 0 and gives + calling functions the chance to insert data +*/ +void *vector_binary_search(const void *const key, const void *base, const size_t member_count, const size_t member_size, size_t compare_size, int *exactmatch) { + size_t interval = member_count; + + while (interval) { + uint8_t *lookat = ((uint8_t *)base) + member_size * (interval / 2); + int cmp = memcmp(lookat, key, compare_size); + if (cmp == 0) { + base = lookat; + break; + } + if (cmp < 0) { + base = lookat + member_size; + interval--; + } + interval /= 2; + } + + *exactmatch = interval; + return (void *)base; +} + +static uint8_t vector_hash_peer(ot_peer const *peer, size_t compare_size, size_t bucket_count) { + unsigned int hash = 5381; + uint8_t *p = (uint8_t *)peer; + while (compare_size--) + hash += (hash << 5) + *(p++); + return hash % bucket_count; +} + +/* This is the generic insert operation for our vector type. + It tries to locate the object at "key" with size "member_size" by comparing its first "compare_size" bytes with + those of objects in vector. Our special "vector_binary_search" function does that and either returns the match or a + pointer to where the object is to be inserted. vector_find_or_insert makes space for the object and copies it, + if it wasn't found in vector. Caller needs to check the passed "exactmatch" variable to see, whether an insert + took place. If resizing the vector failed, NULL is returned, else the pointer to the object in vector. +*/ +void *vector_find_or_insert(ot_vector *vector, void *key, size_t member_size, size_t compare_size, int *exactmatch) { + uint8_t *match = vector_binary_search(key, vector->data, vector->size, member_size, compare_size, exactmatch); + size_t match_offset, bytes_after; + + if (*exactmatch) + return match; + + /* Hold the insertion point as an offset rather than a pointer: it survives + the realloc below, and an empty vector has no data pointer to offset from + in the first place */ + match_offset = vector->size ? (size_t)(match - (uint8_t *)vector->data) : 0; + bytes_after = member_size * vector->size - match_offset; + + if (vector->size + 1 > vector->space) { + size_t new_space = vector->space ? OT_VECTOR_GROW_RATIO * vector->space : OT_VECTOR_MIN_MEMBERS; + uint8_t *new_data = realloc(vector->data, new_space * member_size); + + if (!new_data) + return NULL; + + vector->data = new_data; + vector->space = new_space; + } + + match = (uint8_t *)vector->data + match_offset; + memmove(match + member_size, match, bytes_after); + + vector->size++; + return match; +} + +ot_peer *vector_find_or_insert_peer(ot_vector *vector, ot_peer const *peer, size_t peer_size, int *exactmatch) { + ot_peer *match; + const size_t compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size); + size_t match_offset, bytes_after; + + /* If space is zero but size is set, we're dealing with a list of vector->size buckets */ + if (vector->space < vector->size) + vector = ((ot_vector *)vector->data) + vector_hash_peer(peer, compare_size, vector->size); + match = vector_binary_search(peer, vector->data, vector->size, peer_size, compare_size, exactmatch); + + if (*exactmatch) + return match; + + /* Hold the insertion point as an offset rather than a pointer: it survives + the realloc below, and an empty vector has no data pointer to offset from + in the first place. bytes_after is what has to move up by peer_size to + make room for the new peer */ + match_offset = vector->size ? (size_t)(match - (ot_peer *)vector->data) : 0; + bytes_after = vector->size * peer_size - match_offset; + + if (vector->size + 1 > vector->space) { + size_t new_space = vector->space ? OT_VECTOR_GROW_RATIO * vector->space : OT_VECTOR_MIN_MEMBERS; + ot_peer *new_data = realloc(vector->data, new_space * peer_size); + + if (!new_data) + return NULL; + + vector->data = new_data; + vector->space = new_space; + } + + match = (ot_peer *)vector->data + match_offset; + memmove(match + peer_size, match, bytes_after); + + vector->size++; + return match; +} + +/* This is the non-generic delete from vector-operation specialized for peers in pools. + It returns 0 if no peer was found (and thus not removed) + 1 if a non-seeding peer was removed + 2 if a seeding peer was removed +*/ +int vector_remove_peer(ot_vector *vector, ot_peer const *peer, size_t peer_size) { + int exactmatch, was_seeder; + ot_peer *match, *end; + size_t compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size); + + if (!vector->size) + return 0; + + /* If space is zero but size is set, we're dealing with a list of vector->size buckets */ + if (vector->space < vector->size) + vector = ((ot_vector *)vector->data) + vector_hash_peer(peer, compare_size, vector->size); + + end = ((ot_peer *)vector->data) + peer_size * vector->size; + match = (ot_peer *)vector_binary_search(peer, vector->data, vector->size, peer_size, compare_size, &exactmatch); + if (!exactmatch) + return 0; + + was_seeder = (OT_PEERFLAG_D(match, peer_size) & PEER_FLAG_SEEDING) ? 2 : 1; + memmove(match, match + peer_size, end - match - peer_size); + + vector->size--; + vector_fixup_peers(vector, peer_size); + return was_seeder; +} + +void vector_remove_torrent(ot_vector *vector, ot_torrent *match) { + ot_torrent *end; + + if (!vector->size) + return; + + end = ((ot_torrent *)vector->data) + vector->size; + + free_peerlist(match->peer_list6); + free_peerlist(match->peer_list4); + + memmove(match, match + 1, sizeof(ot_torrent) * (end - match - 1)); + if ((--vector->size * OT_VECTOR_SHRINK_THRESH < vector->space) && (vector->space >= OT_VECTOR_SHRINK_RATIO * OT_VECTOR_MIN_MEMBERS)) { + vector->space /= OT_VECTOR_SHRINK_RATIO; + vector->data = realloc(vector->data, vector->space * sizeof(ot_torrent)); + } +} + +void vector_clean_list(ot_vector *vector, size_t num_buckets) { + while (num_buckets--) + free(vector[num_buckets].data); + free(vector); + return; +} + +void vector_redistribute_buckets(ot_peerlist *peer_list, size_t peer_size) { + size_t bucket, bucket_size_new, num_buckets_new, num_buckets_old = 1; + ot_vector *bucket_list_new, *bucket_list_old = &peer_list->peers; + int (*sort_func)(const void *, const void *) = peer_size == OT_PEER_SIZE6 ? &vector_compare_peer6 : &vector_compare_peer4; + + if (OT_PEERLIST_HASBUCKETS(peer_list)) { + num_buckets_old = peer_list->peers.size; + bucket_list_old = peer_list->peers.data; + } + + if (peer_list->peer_count < 255) + num_buckets_new = 1; + else if (peer_list->peer_count > 8192) + num_buckets_new = 64; + else if (peer_list->peer_count >= 512 && peer_list->peer_count < 4096) + num_buckets_new = 16; + else if (peer_list->peer_count < 512 && num_buckets_old <= 16) + num_buckets_new = num_buckets_old; + else if (peer_list->peer_count < 512) + num_buckets_new = 1; + else if (peer_list->peer_count < 8192 && num_buckets_old > 1) + num_buckets_new = num_buckets_old; + else + num_buckets_new = 16; + + if (num_buckets_new == num_buckets_old) + return; + + /* Assume near perfect distribution */ + bucket_list_new = calloc(1, num_buckets_new * sizeof(ot_vector)); + if (!bucket_list_new) + return; + + bucket_size_new = OT_VECTOR_MIN_MEMBERS; + while (bucket_size_new * num_buckets_new < peer_list->peer_count) + bucket_size_new *= OT_VECTOR_GROW_RATIO; + + /* preallocate vectors to hold all peers */ + for (bucket = 0; bucket < num_buckets_new; ++bucket) { + bucket_list_new[bucket].space = bucket_size_new; + bucket_list_new[bucket].data = malloc(bucket_size_new * peer_size); + if (!bucket_list_new[bucket].data) { + vector_clean_list(bucket_list_new, num_buckets_new); + return; + } + } + + /* Now sort them into the correct bucket */ + for (bucket = 0; bucket < num_buckets_old; ++bucket) { + ot_peer *peers_old = bucket_list_old[bucket].data; + size_t peer_count_old = bucket_list_old[bucket].size; + while (peer_count_old--) { + ot_vector *bucket_dest = bucket_list_new; + if (num_buckets_new > 1) + bucket_dest += vector_hash_peer(peers_old, OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size), num_buckets_new); + if (bucket_dest->size + 1 > bucket_dest->space) { + void *tmp = realloc(bucket_dest->data, peer_size * OT_VECTOR_GROW_RATIO * bucket_dest->space); + if (!tmp) { + vector_clean_list(bucket_list_new, num_buckets_new); + return; + } + bucket_dest->data = tmp; + bucket_dest->space *= OT_VECTOR_GROW_RATIO; + } + memcpy((ot_peer *)bucket_dest->data + peer_size * bucket_dest->size++, peers_old, peer_size); + peers_old += peer_size; + } + } + + /* Now sort each bucket to later allow bsearch */ + for (bucket = 0; bucket < num_buckets_new; ++bucket) + qsort(bucket_list_new[bucket].data, bucket_list_new[bucket].size, peer_size, sort_func); + + /* Everything worked fine. Now link new bucket_list to peer_list */ + if (OT_PEERLIST_HASBUCKETS(peer_list)) + vector_clean_list((ot_vector *)peer_list->peers.data, peer_list->peers.size); + else + free(peer_list->peers.data); + + if (num_buckets_new > 1) { + peer_list->peers.data = bucket_list_new; + peer_list->peers.size = num_buckets_new; + peer_list->peers.space = 0; /* Magic marker for "is list of buckets" */ + } else { + peer_list->peers.data = bucket_list_new->data; + peer_list->peers.size = bucket_list_new->size; + peer_list->peers.space = bucket_list_new->space; + free(bucket_list_new); + } +} + +void vector_fixup_peers(ot_vector *vector, size_t peer_size) { + int need_fix = 0; + + if (!vector->size) { + free(vector->data); + vector->data = NULL; + vector->space = 0; + return; + } + + while ((vector->size * OT_VECTOR_SHRINK_THRESH < vector->space) && (vector->space >= OT_VECTOR_SHRINK_RATIO * OT_VECTOR_MIN_MEMBERS)) { + vector->space /= OT_VECTOR_SHRINK_RATIO; + need_fix++; + } + if (need_fix) + vector->data = realloc(vector->data, vector->space * peer_size); +} diff --git a/src/ot_vector.h b/src/ot_vector.h new file mode 100644 index 0000000..8a5ba86 --- /dev/null +++ b/src/ot_vector.h @@ -0,0 +1,39 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_VECTOR_H__ +#define OT_VECTOR_H__ + +#include + +/* These defines control vectors behaviour */ +#define OT_VECTOR_MIN_MEMBERS 2 +#define OT_VECTOR_GROW_RATIO 2 +#define OT_VECTOR_SHRINK_THRESH 4 +#define OT_VECTOR_SHRINK_RATIO 2 + +#define OT_PEER_BUCKET_MINCOUNT 512 +#define OT_PEER_BUCKET_MAXCOUNT 256 + +typedef struct { + void *data; + size_t size; + size_t space; +} ot_vector; + +/* trackerlogic.h needs ot_vector, break circular dependency here */ +#include "trackerlogic.h" + +void *vector_binary_search(const void *const key, const void *base, const size_t member_count, const size_t member_size, size_t compare_size, int *exactmatch); +void *vector_find_or_insert(ot_vector *vector, void *key, size_t member_size, size_t compare_size, int *exactmatch); +ot_peer *vector_find_or_insert_peer(ot_vector *vector, ot_peer const *peer, size_t peer_size, int *exactmatch); + +int vector_remove_peer(ot_vector *vector, ot_peer const *peer, size_t peer_size); +void vector_remove_torrent(ot_vector *vector, ot_torrent *match); + +/* For ot_clean.c */ +void vector_redistribute_buckets(ot_peerlist *peer_list, size_t peer_size); +void vector_fixup_peers(ot_vector *vector, size_t peer_size); +void vector_clean_list(ot_vector *vector, size_t num_buckets); + +#endif diff --git a/src/ot_worker.c b/src/ot_worker.c new file mode 100644 index 0000000..432d272 --- /dev/null +++ b/src/ot_worker.c @@ -0,0 +1,73 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_worker.h" +#include "trackerlogic.h" + +static struct ot_worker **g_workers; +static size_t g_worker_count; + +/* Everything a worker needs doing around its own loop, in one place, so that + a new kind of worker cannot miss a piece of it. */ +static void *worker_trampoline(void *args) { + struct ot_worker *me = args; + + if (workstruct_init(&me->ws) == -1) + fprintf(stderr, "Warning: a worker could not get its buffers and will not run.\n"); + else { + me->run(me); + workstruct_deinit(&me->ws); + } + + atomic_store(&me->left, 1); + return NULL; +} + +int worker_start(ot_worker_run run, int64_t sock) { + struct ot_worker **grown = realloc(g_workers, (g_worker_count + 1) * sizeof(*g_workers)); + struct ot_worker *worker; + + if (!grown) + return -1; + g_workers = grown; + + if (!(worker = calloc(1, sizeof(*worker)))) + return -1; + worker->sock = sock; + worker->run = run; + + if (pthread_create(&worker->thread, NULL, worker_trampoline, worker)) { + free(worker); + return -1; + } + g_workers[g_worker_count++] = worker; + return 0; +} + +void worker_stop_all(void) { + size_t i; + + for (i = 0; i < g_worker_count; ++i) { + struct ot_worker *worker = g_workers[i]; + + /* Repeated because a signal only interrupts a call already under way */ + while (!atomic_load(&worker->left)) { + pthread_kill(worker->thread, OT_SIGNAL_WAKE); + usleep(1000); + } + pthread_join(worker->thread, NULL); + free(worker); + } + + free(g_workers); + g_workers = NULL; + g_worker_count = 0; +} diff --git a/src/ot_worker.h b/src/ot_worker.h new file mode 100644 index 0000000..4954773 --- /dev/null +++ b/src/ot_worker.h @@ -0,0 +1,37 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_WORKER_H__ +#define OT_WORKER_H__ + +#include +#include +#include + +#include "trackerlogic.h" + +struct ot_worker; + +/* What a worker does between being started and being told to stop. It reads + worker->sock and works out of worker->ws; everything around it, from the + buffers to the counters to saying goodbye, is handled here. */ +typedef void (*ot_worker_run)(struct ot_worker *worker); + +/* A thread that serves requests on a socket of its own, with the buffers and + counters it needs. Each is allocated on its own, so taking on another one + never moves the record a running worker holds. */ +struct ot_worker { + pthread_t thread; + _Atomic int left; + int64_t sock; + ot_worker_run run; + struct ot_workstruct ws; +}; + +/* Start one. Returns -1 if it could not be started. */ +int worker_start(ot_worker_run run, int64_t sock); + +/* Wake every worker until it has left, then join it */ +void worker_stop_all(void); + +#endif diff --git a/src/ot_workqueue.c b/src/ot_workqueue.c new file mode 100644 index 0000000..56a7cd2 --- /dev/null +++ b/src/ot_workqueue.c @@ -0,0 +1,241 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_iovec.h" +#include "ot_workqueue.h" +#include "trackerlogic.h" + +/* Pushing a result wakes the thread that will send it */ +extern int g_self_pipe[2]; + +struct ot_task { + ot_taskid taskid; + ot_tasktype tasktype; + int64_t sock; + size_t iovec_entries; + struct iovec *iovec; + struct ot_task *next; +}; + +static ot_taskid next_free_taskid = 1; +static struct ot_task *tasklist; +static pthread_mutex_t tasklist_mutex; +static pthread_cond_t tasklist_being_filled; + +int workqueue_pushtask(int64_t sock, ot_tasktype tasktype) { + struct ot_task **tmptask, *task; + + task = malloc(sizeof(struct ot_task)); + if (!task) + return -1; + + task->taskid = 0; + task->tasktype = tasktype; + task->sock = sock; + task->iovec_entries = 0; + task->iovec = NULL; + task->next = 0; + + /* Want exclusive access to tasklist */ + pthread_mutex_lock(&tasklist_mutex); + + /* Skip to end of list */ + tmptask = &tasklist; + while (*tmptask) + tmptask = &(*tmptask)->next; + *tmptask = task; + + /* Inform waiting workers and release lock */ + pthread_cond_broadcast(&tasklist_being_filled); + pthread_mutex_unlock(&tasklist_mutex); + return 0; +} + +void workqueue_canceltask(int64_t sock) { + struct ot_task **task; + + /* Want exclusive access to tasklist */ + pthread_mutex_lock(&tasklist_mutex); + + for (task = &tasklist; *task; task = &((*task)->next)) + if ((*task)->sock == sock) { + struct iovec *iovec = (*task)->iovec; + struct ot_task *ptask = *task; + size_t i; + + /* Free task's iovec, the buffers it points at and the vector itself */ + for (i = 0; i < (*task)->iovec_entries; ++i) + free(iovec[i].iov_base); + free(iovec); + + *task = (*task)->next; + free(ptask); + break; + } + + /* Release lock */ + pthread_mutex_unlock(&tasklist_mutex); +} + +ot_taskid workqueue_poptask(ot_tasktype *tasktype) { + struct ot_task *task; + ot_taskid taskid = 0; + + /* Want exclusive access to tasklist */ + pthread_mutex_lock(&tasklist_mutex); + + while (!taskid) { + /* Skip to the first unassigned task this worker wants to do */ + for (task = tasklist; task; task = task->next) + if (!task->taskid && (TASK_CLASS_MASK & task->tasktype) == *tasktype) { + /* If we found an outstanding task, assign a taskid to it + and leave the loop */ + task->taskid = taskid = ++next_free_taskid; + *tasktype = task->tasktype; + break; + } + + /* Wait until the next task is being fed, unless the tracker is on its + way out, in which case the worker is meant to leave rather than wait */ + if (!taskid) { + if (!g_opentracker_running) + break; + pthread_cond_wait(&tasklist_being_filled, &tasklist_mutex); + } + } + + /* Release lock */ + pthread_mutex_unlock(&tasklist_mutex); + + return taskid; +} + +void workqueue_pushsuccess(ot_taskid taskid) { + struct ot_task **task; + + /* Want exclusive access to tasklist */ + pthread_mutex_lock(&tasklist_mutex); + + for (task = &tasklist; *task; task = &((*task)->next)) + if ((*task)->taskid == taskid) { + struct ot_task *ptask = *task; + *task = (*task)->next; + free(ptask); + break; + } + + /* Release lock */ + pthread_mutex_unlock(&tasklist_mutex); +} + +int workqueue_pushresult(ot_taskid taskid, size_t iovec_entries, struct iovec *iovec) { + struct ot_task *task; + const char byte = 'o'; + + /* Want exclusive access to tasklist */ + pthread_mutex_lock(&tasklist_mutex); + + for (task = tasklist; task; task = task->next) + if (task->taskid == taskid) { + task->iovec_entries = iovec_entries; + task->iovec = iovec; + task->tasktype = TASK_DONE; + break; + } + + /* Release lock */ + pthread_mutex_unlock(&tasklist_mutex); + + /* A full pipe has already done this nudge's job, so a failure is fine */ + (void)!write(g_self_pipe[1], &byte, 1); + + /* Indicate whether the worker has to throw away results */ + return task ? 0 : -1; +} + +int workqueue_pushchunked(ot_taskid taskid, struct iovec *iovec) { + struct ot_task *task; + const char byte = 'o'; + + /* Want exclusive access to tasklist */ + pthread_mutex_lock(&tasklist_mutex); + + for (task = tasklist; task; task = task->next) + if (task->taskid == taskid) { + if (iovec) { + if (iovec_append(&task->iovec_entries, &task->iovec, iovec)) + task->tasktype = TASK_DONE_PARTIAL; + else + task = NULL; + } else + task->tasktype = TASK_DONE; + break; + } + + /* Release lock */ + pthread_mutex_unlock(&tasklist_mutex); + + /* A full pipe has already done this nudge's job, so a failure is fine */ + (void)!write(g_self_pipe[1], &byte, 1); + + /* Indicate whether the worker has to throw away results */ + return task ? 0 : -1; +} + +int64_t workqueue_popresult(size_t *iovec_entries, struct iovec **iovec, int *is_partial) { + struct ot_task **task; + int64_t sock = -1; + + *is_partial = 0; + + /* Want exclusive access to tasklist */ + pthread_mutex_lock(&tasklist_mutex); + + for (task = &tasklist; *task; task = &((*task)->next)) + if (((*task)->tasktype & TASK_CLASS_MASK) == TASK_DONE) { + struct ot_task *ptask = *task; + *iovec_entries = ptask->iovec_entries; + *iovec = ptask->iovec; + sock = ptask->sock; + + if ((*task)->tasktype == TASK_DONE) { + *task = ptask->next; + free(ptask); + } else { + ptask->iovec_entries = 0; + ptask->iovec = NULL; + *is_partial = 1; + /* Prevent task from showing up immediately again unless new data was added */ + (*task)->tasktype = TASK_FULLSCRAPE; + } + break; + } + + /* Release lock */ + pthread_mutex_unlock(&tasklist_mutex); + return sock; +} + +void workqueue_wakeall(void) { + pthread_mutex_lock(&tasklist_mutex); + pthread_cond_broadcast(&tasklist_being_filled); + pthread_mutex_unlock(&tasklist_mutex); +} + +void workqueue_init(void) { + pthread_mutex_init(&tasklist_mutex, NULL); + pthread_cond_init(&tasklist_being_filled, NULL); +} + +void workqueue_deinit(void) { + pthread_mutex_destroy(&tasklist_mutex); + pthread_cond_destroy(&tasklist_being_filled); +} diff --git a/src/ot_workqueue.h b/src/ot_workqueue.h new file mode 100644 index 0000000..82ae29b --- /dev/null +++ b/src/ot_workqueue.h @@ -0,0 +1,74 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_WORKQUEUE_H__ +#define OT_WORKQUEUE_H__ + +#include +#include + +#include "trackerlogic.h" + +/* Work the tracker hands to a thread of its own: a full scrape or an + expensive stats page, which take too long to build while a client waits. + The task goes in here, the reply comes back out. */ + +typedef enum { + TASK_STATS_CONNS = 0x0001, + TASK_STATS_TCP = 0x0002, + TASK_STATS_UDP = 0x0003, + TASK_STATS_SCRAPE = 0x0004, + TASK_STATS_FULLSCRAPE = 0x0005, + TASK_STATS_TPB = 0x0006, + TASK_STATS_HTTPERRORS = 0x0007, + TASK_STATS_RENEW = 0x000a, + TASK_STATS_SYNCS = 0x000b, + TASK_STATS_COMPLETED = 0x000c, + TASK_STATS_NUMWANTS = 0x000d, + + TASK_STATS = 0x0100, /* Mask */ + TASK_STATS_TORRENTS = 0x0101, + TASK_STATS_PEERS = 0x0102, + TASK_STATS_SLASH24S = 0x0103, + TASK_STATS_TOP10 = 0x0104, + TASK_STATS_TOP100 = 0x0105, + TASK_STATS_EVERYTHING = 0x0106, + TASK_STATS_WOODPECKERS = 0x0108, + + TASK_FULLSCRAPE = 0x0200, /* Default mode */ + TASK_FULLSCRAPE_TPB_BINARY = 0x0201, + TASK_FULLSCRAPE_TPB_ASCII = 0x0202, + TASK_FULLSCRAPE_TPB_ASCII_PLUS = 0x0203, + TASK_FULLSCRAPE_TPB_URLENCODED = 0x0204, + TASK_FULLSCRAPE_TRACKERSTATE = 0x0205, + + TASK_DMEM = 0x0300, + + TASK_DONE = 0x0f00, + TASK_DONE_PARTIAL = 0x0f01, + + TASK_FLAG_GZIP = 0x1000, + TASK_FLAG_BZIP2 = 0x2000, + TASK_FLAG_ZSTD = 0x4000, + TASK_FLAG_CHUNKED = 0x8000, + + TASK_TASK_MASK = 0x0fff, + TASK_CLASS_MASK = 0x0f00, + TASK_FLAGS_MASK = 0xf000 +} ot_tasktype; + +typedef unsigned long ot_taskid; + +int workqueue_pushtask(int64_t sock, ot_tasktype tasktype); +void workqueue_canceltask(int64_t sock); +void workqueue_pushsuccess(ot_taskid taskid); +ot_taskid workqueue_poptask(ot_tasktype *tasktype); +int workqueue_pushresult(ot_taskid taskid, size_t iovec_entries, struct iovec *iovector); +int workqueue_pushchunked(ot_taskid taskid, struct iovec *iovec); +int64_t workqueue_popresult(size_t *iovec_entries, struct iovec **iovector, int *is_partial); + +void workqueue_init(void); +void workqueue_deinit(void); +void workqueue_wakeall(void); + +#endif diff --git a/src/proxy.c b/src/proxy.c new file mode 100644 index 0000000..8589db5 --- /dev/null +++ b/src/proxy.c @@ -0,0 +1,871 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* This relays sync traffic between trackers, and between multicast domains + that have no route to each other, so that no tracker has to reach out to + the outside world by itself. Defunc. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Libowfat */ +#include "io.h" + +/* Opentracker */ +#include "ot_batch.h" +#include "ot_fdstate.h" +#include "ot_ip.h" +#include "ot_mutex.h" +#include "ot_random.h" +#include "ot_scan.h" +#include "ot_socket.h" +#include "ot_stats.h" +#include "ot_vector.h" +#include "trackerlogic.h" + +#include "ot_livesync.h" + +ot_ip6 g_serverip; +uint16_t g_serverport = 9009; +uint32_t g_tracker_id; +ot_ip6 groupip_1 = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 224, 0, 23, 5}; +int g_self_pipe[2]; + +/* If you have more than 10 peers, don't use this proxy + Use 20 slots for 10 peers to have room for 10 incoming connection slots + */ +#define MAX_PEERS 20 + +#define LIVESYNC_INCOMING_BUFFSIZE (256 * 256) +#define STREAMSYNC_OUTGOING_BUFFSIZE (256 * 256) + +#define LIVESYNC_OUTGOING_BUFFSIZE_PEERS 1480 +#define LIVESYNC_OUTGOING_WATERMARK_PEERS (sizeof(ot_peer) + sizeof(ot_hash)) +#define LIVESYNC_MAXDELAY 15 /* seconds */ + +/* The amount of time a complete sync cycle should take */ +#define OT_SYNC_INTERVAL_MINUTES 2 + +/* So after each bucket wait 1 / OT_BUCKET_COUNT intervals */ +#define OT_SYNC_SLEEP (((OT_SYNC_INTERVAL_MINUTES) * 60 * 1000000) / (OT_BUCKET_COUNT)) + +enum { OT_SYNC_PEER4, OT_SYNC_PEER6 }; +enum { FLAG_SERVERSOCKET = 1 }; + +/* For incoming packets */ +static int g_socket_in = -1; +static uint8_t g_inbuffer[LIVESYNC_INCOMING_BUFFSIZE]; + +/* For outgoing packets */ +static int g_socket_out = -1; +static uint8_t g_peerbuffer_start[LIVESYNC_OUTGOING_BUFFSIZE_PEERS]; +static uint8_t *g_peerbuffer_pos; +static uint8_t *g_peerbuffer_highwater = g_peerbuffer_start + LIVESYNC_OUTGOING_BUFFSIZE_PEERS - LIVESYNC_OUTGOING_WATERMARK_PEERS; +static ot_time g_next_packet_time; + +static void *livesync_worker(void *args); +static void *streamsync_worker(void *args); +static void livesync_proxytell(uint8_t prefix, uint8_t *info_hash, uint8_t *peer); + +void exerr(const char *message) { + fprintf(stderr, "%s\n", message); + exit(111); +} + +void stats_issue_event(struct ot_workstruct *ws, ot_status_event event, PROTO_FLAG proto, uintptr_t event_data) { + (void)ws; + (void)event; + (void)proto; + (void)event_data; +} + +void livesync_bind_mcast(ot_ip6 ip, uint16_t port) { + if (!ip_is_v4mapped(ip)) + exerr("v6 mcast support not yet available."); + + if (g_socket_in != -1) + exerr("Error: Livesync listen ip specified twice."); + + if ((g_socket_in = socket_mcast_join(groupip_1, ip, port)) == -1) + exerr("Error: Cant bind live sync incoming socket."); + + if ((g_socket_out = socket_mcast_bind(ip, port, 1)) == -1) + exerr("Error: Cant bind live sync outgoing socket."); +} + +size_t add_peer_to_torrent_proxy(ot_hash hash, ot_peer *peer, size_t peer_size) { + int exactmatch; + ot_torrent *torrent; + ot_peerlist *peer_list; + ot_peer *peer_dest; + ot_vector *torrents_list = mutex_bucket_lock_by_hash(hash); + size_t compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size); + + torrent = vector_find_or_insert(torrents_list, (void *)hash, sizeof(ot_torrent), compare_size, &exactmatch); + if (!torrent) + return -1; + + if (!exactmatch) { + /* Create a new torrent entry, then */ + memcpy(torrent->hash, hash, sizeof(ot_hash)); + + if (!(torrent->peer_list6 = malloc(sizeof(ot_peerlist))) || !(torrent->peer_list4 = malloc(sizeof(ot_peerlist)))) { + vector_remove_torrent(torrents_list, torrent); + mutex_bucket_unlock_by_hash(hash, 0); + return -1; + } + + memset(torrent->peer_list6, 0, sizeof(ot_peerlist)); + memset(torrent->peer_list4, 0, sizeof(ot_peerlist)); + } + + peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; + + /* Check for peer in torrent */ + peer_dest = vector_find_or_insert_peer(&(peer_list->peers), peer, peer_size, &exactmatch); + if (!peer_dest) { + mutex_bucket_unlock_by_hash(hash, 0); + return -1; + } + /* Tell peer that it's fresh */ + OT_PEERTIME(peer, peer_size) = 0; + + /* If we hadn't had a match create peer there */ + if (!exactmatch) { + peer_list->peer_count++; + if (OT_PEERFLAG_D(peer, peer_size) & PEER_FLAG_SEEDING) + peer_list->seed_count++; + } + memcpy(peer_dest, peer, peer_size); + mutex_bucket_unlock_by_hash(hash, 0); + return 0; +} + +size_t remove_peer_from_torrent_proxy(ot_hash hash, ot_peer *peer, size_t peer_size) { + int exactmatch; + ot_vector *torrents_list = mutex_bucket_lock_by_hash(hash); + ot_torrent *torrent = vector_binary_search(hash, torrents_list->data, torrents_list->size, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); + + if (exactmatch) { + ot_peerlist *peer_list = peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; + switch (vector_remove_peer(&peer_list->peers, peer, peer_size)) { + case 2: + peer_list->seed_count--; /* Intentional fallthrough */ + case 1: + peer_list->peer_count--; /* Intentional fallthrough */ + default: + break; + } + } + + mutex_bucket_unlock_by_hash(hash, 0); + return 0; +} + +void free_peerlist(ot_peerlist *peer_list) { + if (peer_list->peers.data) { + if (OT_PEERLIST_HASBUCKETS(peer_list)) { + ot_vector *bucket_list = (ot_vector *)(peer_list->peers.data); + + while (peer_list->peers.size--) + free(bucket_list++->data); + } + free(peer_list->peers.data); + } + free(peer_list); +} + +static void livesync_handle_peersync(ssize_t datalen, size_t peer_size) { + int off = sizeof(g_tracker_id) + sizeof(uint32_t); + + fprintf(stderr, "."); + + while ((ssize_t)(off + sizeof(ot_hash) + peer_size) <= datalen) { + ot_peer *peer = (ot_peer *)(g_inbuffer + off + sizeof(ot_hash)); + ot_hash *hash = (ot_hash *)(g_inbuffer + off); + + if (OT_PEERFLAG_D(peer, peer_size) & PEER_FLAG_STOPPED) + remove_peer_from_torrent_proxy(*hash, peer, peer_size); + else + add_peer_to_torrent_proxy(*hash, peer, peer_size); + + off += sizeof(ot_hash) + peer_size; + } +} + +int usage(char *self) { + fprintf(stderr, "Usage: %s -L -l : -c :\n", self); + return 0; +} + +enum { + FLAG_OUTGOING = 0x80, + + FLAG_DISCONNECTED = 0x00, + FLAG_CONNECTING = 0x01, + FLAG_WAITTRACKERID = 0x02, + FLAG_CONNECTED = 0x03, + + FLAG_MASK = 0x07 +}; + +#define PROXYPEER_NEEDSCONNECT(flag) ((flag) == FLAG_OUTGOING) +#define PROXYPEER_ISCONNECTED(flag) (((flag) & FLAG_MASK) == FLAG_CONNECTED) +#define PROXYPEER_SETDISCONNECTED(flag) (flag) = (((flag) & FLAG_OUTGOING) | FLAG_DISCONNECTED) +#define PROXYPEER_SETCONNECTING(flag) (flag) = (((flag) & FLAG_OUTGOING) | FLAG_CONNECTING) +#define PROXYPEER_SETWAITTRACKERID(flag) (flag) = (((flag) & FLAG_OUTGOING) | FLAG_WAITTRACKERID) +#define PROXYPEER_SETCONNECTED(flag) (flag) = (((flag) & FLAG_OUTGOING) | FLAG_CONNECTED) + +typedef struct { + int state; /* Whether we want to connect, how far our handshake is, etc. */ + ot_ip6 ip; /* The peer to connect to */ + uint16_t port; /* The peers port */ + uint8_t indata[8192 * 16]; /* Any data not processed yet */ + size_t indata_length; /* Length of unprocessed data */ + uint32_t tracker_id; /* How the other end greeted */ + int64_t fd; /* A file handle, if connected, <= 0 is disconnected (0 initially, -1 else) */ + ot_batch outdata; /* The batch containing our sync data */ + + size_t packet_tcount; /* Number of unprocessed torrents in packet we currently receive */ + uint8_t packet_tprefix; /* Prefix byte for all torrents in current packet */ + uint8_t packet_type; /* Type of current packet */ + uint32_t packet_tid; /* Tracker id for current packet */ + +} proxy_peer; +static void process_indata(proxy_peer *peer); + +void reset_info_block(proxy_peer *peer) { + peer->indata_length = 0; + peer->tracker_id = 0; + peer->fd = -1; + peer->packet_tcount = 0; + batch_reset(&peer->outdata); + PROXYPEER_SETDISCONNECTED(peer->state); +} + +/* Number of connections to peers + * If a peer's IP is set, we try to reconnect, when the connection drops + * If we already have a connected tracker_id in our records for an _incoming_ connection, drop it + * Multiple connections to/from the same ip are okay, if tracker_id doesn't match + * Reconnect attempts occur only twice a minute + */ +static size_t g_connection_count; +static ot_time g_connection_reconn; +static proxy_peer g_connections[MAX_PEERS]; + +static void handle_reconnects(void) { + size_t i; + for (i = 0; i < g_connection_count; ++i) + if (PROXYPEER_NEEDSCONNECT(g_connections[i].state)) { + int64_t newfd = socket_tcp_connect(g_serverip, g_serverport, g_connections[i].ip, g_connections[i].port); + fprintf(stderr, "(Re)connecting to peer..."); + if (newfd == -1) + continue; /* No socket for you */ + io_fd(newfd); + io_wantwrite(newfd); /* So we will be informed when it is connected */ + fdstate_open(newfd, g_connections + i, 0); + + /* Prepare connection info block */ + reset_info_block(g_connections + i); + g_connections[i].fd = newfd; + PROXYPEER_SETCONNECTING(g_connections[i].state); + } + g_connection_reconn = time(NULL) + 30; +} + +/* Handle incoming connection requests, check against whitelist */ +static void handle_accept(int64_t serversocket) { + int64_t newfd; + ot_ip6 ip; + uint16_t port; + + while ((newfd = socket_accept(serversocket, ip, &port)) != -1) { + + /* XXX some access control */ + + /* Put fd into a non-blocking mode */ + io_nonblock(newfd); + + if (!io_fd(newfd)) + io_close(newfd); + else { + /* Find a new home for our incoming connection */ + int i; + for (i = 0; i < MAX_PEERS; ++i) + if (g_connections[i].state == FLAG_DISCONNECTED) + break; + if (i == MAX_PEERS) { + fprintf(stderr, "No room for incoming connection."); + close(newfd); + continue; + } + + /* Prepare connection info block */ + reset_info_block(g_connections + i); + PROXYPEER_SETCONNECTING(g_connections[i].state); + g_connections[i].port = port; + g_connections[i].fd = newfd; + + fdstate_open(newfd, g_connections + i, 0); + + /* We expect the connecting side to begin with its tracker_id */ + io_wantread(newfd); + } + } + + return; +} + +/* New sync data on the stream */ +static void handle_read(int64_t peersocket) { + int i; + int64_t datalen; + uint32_t tracker_id; + proxy_peer *peer = fdstate_cookie(peersocket); + + if (!peer) { + /* Can't happen ;) */ + io_close(peersocket); + return; + } + switch (peer->state & FLAG_MASK) { + case FLAG_DISCONNECTED: + io_close(peersocket); + break; /* Shouldnt happen */ + case FLAG_CONNECTING: + case FLAG_WAITTRACKERID: + /* We want at least the first four bytes to come at once, to avoid keeping extra states (for now) + This also catches 0 bytes reads == EOF and negative values, denoting connection errors */ + if (io_tryread(peersocket, (void *)&tracker_id, sizeof(tracker_id)) != sizeof(tracker_id)) + goto close_socket; + + /* See, if we already have a connection to that peer */ + for (i = 0; i < MAX_PEERS; ++i) + if ((g_connections[i].state & FLAG_MASK) == FLAG_CONNECTED && g_connections[i].tracker_id == tracker_id) { + fprintf(stderr, "Peer already connected. Closing connection.\n"); + goto close_socket; + } + + /* Also no need for soliloquy */ + if (tracker_id == g_tracker_id) + goto close_socket; + + /* The new connection is good, send our tracker_id on incoming connections */ + if (peer->state == FLAG_CONNECTING) + if (io_trywrite(peersocket, (void *)&g_tracker_id, sizeof(g_tracker_id)) != sizeof(g_tracker_id)) + goto close_socket; + + peer->tracker_id = tracker_id; + PROXYPEER_SETCONNECTED(peer->state); + + if (peer->state & FLAG_OUTGOING) + fprintf(stderr, "succeeded.\n"); + else + fprintf(stderr, "Incoming connection successful.\n"); + + break; + close_socket: + fprintf(stderr, "Handshake incomplete, closing socket\n"); + io_close(peersocket); + reset_info_block(peer); + break; + case FLAG_CONNECTED: + /* Here we acutally expect data from peer + indata_length should be less than 20+256*7 bytes, for incomplete torrent entries */ + datalen = io_tryread(peersocket, (void *)(peer->indata + peer->indata_length), sizeof(peer->indata) - peer->indata_length); + if (!datalen || datalen < -1) { + fprintf(stderr, "Connection closed by remote peer.\n"); + io_close(peersocket); + reset_info_block(peer); + } else if (datalen > 0) { + peer->indata_length += datalen; + process_indata(peer); + } + break; + } +} + +/* Can write new sync data to the stream */ +static void handle_write(int64_t peersocket) { + proxy_peer *peer = fdstate_cookie(peersocket); + + if (!peer) { + /* Can't happen ;) */ + io_close(peersocket); + return; + } + + switch (peer->state & FLAG_MASK) { + case FLAG_DISCONNECTED: + default: /* Should not happen */ + io_close(peersocket); + break; + case FLAG_CONNECTING: + /* Ensure that the connection is established and handle connection error */ + if (peer->state & FLAG_OUTGOING && !socket_is_connected(peersocket)) { + fprintf(stderr, "failed\n"); + reset_info_block(peer); + io_close(peersocket); + break; + } + + if (io_trywrite(peersocket, (void *)&g_tracker_id, sizeof(g_tracker_id)) == sizeof(g_tracker_id)) { + PROXYPEER_SETWAITTRACKERID(peer->state); + io_dontwantwrite(peersocket); + io_wantread(peersocket); + } else { + fprintf(stderr, "Handshake incomplete, closing socket\n"); + io_close(peersocket); + reset_info_block(peer); + } + break; + case FLAG_CONNECTED: + switch (batch_send(peersocket, &peer->outdata)) { + case BATCH_DONE: + io_dontwantwrite(peersocket); + break; + case BATCH_FAILED: + io_close(peersocket); + reset_info_block(peer); + break; + default: /* Normal operation or eagain */ + break; + } + break; + } + + return; +} + +static void server_mainloop() { + int64_t sock; + uint32_t packet_type; + + /* inlined livesync_init() */ + memset(g_peerbuffer_start, 0, sizeof(g_peerbuffer_start)); + g_peerbuffer_pos = g_peerbuffer_start; + memcpy(g_peerbuffer_pos, &g_tracker_id, sizeof(g_tracker_id)); + packet_type = htonl(OT_SYNC_PEER); + memcpy(g_peerbuffer_pos + sizeof(g_tracker_id), &packet_type, sizeof(packet_type)); + g_peerbuffer_pos += sizeof(g_tracker_id) + sizeof(uint32_t); + g_next_packet_time = time(NULL) + LIVESYNC_MAXDELAY; + + while (1) { + /* See if we need to connect to anyone */ + if (time(NULL) > g_connection_reconn) + handle_reconnects(); + + /* Wait for io events until next approx reconn check time */ + io_waituntil2(30 * 1000); + + /* Loop over readable sockets */ + while ((sock = io_canread()) != -1) { + const void *cookie = fdstate_cookie(sock); + if ((uintptr_t)cookie == FLAG_SERVERSOCKET) + handle_accept(sock); + else + handle_read(sock); + } + + /* Loop over writable sockets */ + while ((sock = io_canwrite()) != -1) + handle_write(sock); + + livesync_ticker(); + } +} + +static void panic(const char *routine) { + fprintf(stderr, "%s: %s\n", routine, strerror(errno)); + exit(111); +} + +static int64_t ot_try_bind(ot_ip6 ip, uint16_t port) { + int64_t sock = socket_tcp_listen(ip, port, SOCKET_EXCLUSIVE); + + if (sock == -1) + panic("socket_tcp_listen"); + + if (!io_fd(sock)) + panic("io_fd"); + + fdstate_open(sock, (void *)FLAG_SERVERSOCKET, 0); + io_wantread(sock); + return sock; +} + +static int scan_ip6_port(const char *src, ot_ip6 ip, uint16_t *port) { + const char *s = src; + uint64_t value; + int off, bracket = 0; + while (isspace(*s)) + ++s; + if (*s == '[') + ++s, ++bracket; /* for v6 style notation */ + if (!(off = ip_parse(s, ip))) + return 0; + s += off; + if (*s == 0 || isspace(*s)) + return s - src; + if (*s == ']' && bracket) + ++s; + if (!ip_is_v4mapped(ip)) { + if ((bracket && *(s) != ':') || (*(s) != '.')) + return 0; + s++; + } else { + if (*(s++) != ':') + return 0; + } + if (!(off = scan_uint(s, &value, 65535))) + return 0; + *port = (uint16_t)value; + return off + s - src; +} + +int main(int argc, char **argv) { + static pthread_t sync_in_thread_id; + static pthread_t sync_out_thread_id; + ot_ip6 serverip; + uint16_t tmpport; + int scanon = 1, lbound = 0, sbound = 0; + + random_init(); + if (fdstate_init() == -1) + panic("fdstate_init"); + g_tracker_id = random_uint32(); + + while (scanon) { + switch (getopt(argc, argv, ":l:c:L:h")) { + case -1: + scanon = 0; + break; + case 'l': + tmpport = 0; + if (!scan_ip6_port(optarg, serverip, &tmpport) || !tmpport) { + usage(argv[0]); + exit(1); + } + ot_try_bind(serverip, tmpport); + ++sbound; + break; + case 'c': + if (g_connection_count > MAX_PEERS / 2) + exerr("Connection limit exceeded.\n"); + tmpport = 0; + if (!scan_ip6_port(optarg, g_connections[g_connection_count].ip, &g_connections[g_connection_count].port) || !g_connections[g_connection_count].port) { + usage(argv[0]); + exit(1); + } + g_connections[g_connection_count++].state = FLAG_OUTGOING; + break; + case 'L': + tmpport = 9696; + if (!scan_ip6_port(optarg, serverip, &tmpport) || !tmpport) { + usage(argv[0]); + exit(1); + } + livesync_bind_mcast(serverip, tmpport); + ++lbound; + break; + default: + case '?': + usage(argv[0]); + exit(1); + } + } + + if (!lbound) + exerr("No livesync port bound."); + if (!g_connection_count && !sbound) + exerr("No streamsync port bound."); + pthread_create(&sync_in_thread_id, NULL, livesync_worker, NULL); + pthread_create(&sync_out_thread_id, NULL, streamsync_worker, NULL); + + server_mainloop(); + return 0; +} + +static void *streamsync_worker(void *args) { + (void)args; + while (1) { + size_t bucket; + /* For each bucket... */ + for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { + /* Get exclusive access to that bucket */ + ot_vector *torrents_list = mutex_bucket_lock(bucket); + size_t tor_offset, count_def = 0, count_one = 0, count_two = 0, count_peers = 0; + size_t mem, mem_a = 0, mem_b = 0; + uint8_t *ptr = 0, *ptr_a, *ptr_b, *ptr_c; + + if (!torrents_list->size) + goto unlock_continue; + + /* For each torrent in this bucket.. */ + for (tor_offset = 0; tor_offset < torrents_list->size; ++tor_offset) { + /* Address torrents members */ + ot_peerlist *peer_list = (((ot_torrent *)(torrents_list->data))[tor_offset]).peer_list; + switch (peer_list->peer_count) { + case 2: + count_two++; + break; + case 1: + count_one++; + break; + case 0: + break; + default: + count_def++; + count_peers += peer_list->peer_count; + } + } + + /* Maximal memory requirement: max 3 blocks, max torrents * 20 + max peers * 7 */ + mem = 3 * (1 + 1 + 2) + (count_one + count_two) * (19 + 1) + count_def * (19 + 8) + (count_one + 2 * count_two + count_peers) * 7; + + fprintf(stderr, "Mem: %zd\n", mem); + + ptr = ptr_a = ptr_b = ptr_c = malloc(mem); + if (!ptr) + goto unlock_continue; + + if (count_one > 4 || !count_def) { + mem_a = 1 + 1 + 2 + count_one * (19 + 7); + ptr_b += mem_a; + ptr_c += mem_a; + ptr_a[0] = 1; /* Offset 0: packet type 1 */ + ptr_a[1] = (bucket << 8) >> OT_BUCKET_COUNT_BITS; /* Offset 1: the shared prefix */ + ptr_a[2] = count_one >> 8; + ptr_a[3] = count_one & 255; + ptr_a += 4; + } else + count_def += count_one; + + if (count_two > 4 || !count_def) { + mem_b = 1 + 1 + 2 + count_two * (19 + 14); + ptr_c += mem_b; + ptr_b[0] = 2; /* Offset 0: packet type 2 */ + ptr_b[1] = (bucket << 8) >> OT_BUCKET_COUNT_BITS; /* Offset 1: the shared prefix */ + ptr_b[2] = count_two >> 8; + ptr_b[3] = count_two & 255; + ptr_b += 4; + } else + count_def += count_two; + + if (count_def) { + ptr_c[0] = 0; /* Offset 0: packet type 0 */ + ptr_c[1] = (bucket << 8) >> OT_BUCKET_COUNT_BITS; /* Offset 1: the shared prefix */ + ptr_c[2] = count_def >> 8; + ptr_c[3] = count_def & 255; + ptr_c += 4; + } + + /* For each torrent in this bucket.. */ + for (tor_offset = 0; tor_offset < torrents_list->size; ++tor_offset) { + /* Address torrents members */ + ot_torrent *torrent = ((ot_torrent *)(torrents_list->data)) + tor_offset; + ot_peerlist *peer_list = torrent->peer_list; + ot_peer *peers = (ot_peer *)(peer_list->peers.data); + uint8_t **dst; + + /* Determine destination slot */ + count_peers = peer_list->peer_count; + switch (count_peers) { + case 0: + continue; + case 1: + dst = mem_a ? &ptr_a : &ptr_c; + break; + case 2: + dst = mem_b ? &ptr_b : &ptr_c; + break; + default: + dst = &ptr_c; + break; + } + + /* Copy tail of info_hash, advance pointer */ + memcpy(*dst, ((uint8_t *)torrent->hash) + 1, sizeof(ot_hash) - 1); + *dst += sizeof(ot_hash) - 1; + + /* Encode peer count */ + if (dst == &ptr_c) + while (count_peers) { + if (count_peers <= 0x7f) + *(*dst)++ = count_peers; + else + *(*dst)++ = 0x80 | (count_peers & 0x7f); + count_peers >>= 7; + } + + /* Copy peers */ + count_peers = peer_list->peer_count; + while (count_peers--) { + memcpy(*dst, peers++, OT_IP_SIZE + 3); + *dst += OT_IP_SIZE + 3; + } + free_peerlist(peer_list); + } + + free(torrents_list->data); + memset(torrents_list, 0, sizeof(*torrents_list)); + unlock_continue: + mutex_bucket_unlock(bucket, 0); + + if (ptr) { + int i; + + if (ptr_b > ptr_c) + ptr_c = ptr_b; + if (ptr_a > ptr_c) + ptr_c = ptr_a; + mem = ptr_c - ptr; + + for (i = 0; i < MAX_PEERS; ++i) { + if (PROXYPEER_ISCONNECTED(g_connections[i].state)) { + void *tmp = malloc(mem); + if (tmp) { + memcpy(tmp, ptr, mem); + batch_add(&g_connections[i].outdata, tmp, mem); + io_wantwrite(g_connections[i].fd); + } + } + } + + free(ptr); + } + usleep(OT_SYNC_SLEEP); + } + } + return 0; +} + +static void livesync_issue_peersync() { + socket_mcast_send(g_socket_out, g_peerbuffer_start, (size_t)(g_peerbuffer_pos - g_peerbuffer_start), groupip_1, LIVESYNC_PORT); + g_peerbuffer_pos = g_peerbuffer_start + sizeof(g_tracker_id) + sizeof(uint32_t); + g_next_packet_time = time(NULL) + LIVESYNC_MAXDELAY; +} + +void livesync_ticker() { + /* livesync_issue_peersync sets g_next_packet_time */ + if (time(NULL) > g_next_packet_time && g_peerbuffer_pos > g_peerbuffer_start + sizeof(g_tracker_id)) + livesync_issue_peersync(); +} + +static void livesync_proxytell(uint8_t prefix, uint8_t *info_hash, uint8_t *peer) { + // unsigned int i; + + *g_peerbuffer_pos = prefix; + memcpy(g_peerbuffer_pos + 1, info_hash, sizeof(ot_hash) - 1); + memcpy(g_peerbuffer_pos + sizeof(ot_hash), peer, sizeof(ot_peer) - 1); + +#if 0 + /* Dump info_hash */ + for( i=0; i= g_peerbuffer_highwater) + livesync_issue_peersync(); +} + +static void process_indata(proxy_peer *peer) { + size_t consumed, peers; + uint8_t *data = peer->indata, *hash; + uint8_t *dataend = data + peer->indata_length; + + while (1) { + /* If we're not inside of a packet, make a new one */ + if (!peer->packet_tcount) { + /* Ensure the header is complete or postpone processing */ + if (data + 4 > dataend) + break; + peer->packet_type = data[0]; + peer->packet_tprefix = data[1]; + peer->packet_tcount = data[2] * 256 + data[3]; + data += 4; + printf("type: %hhu, prefix: %02X, torrentcount: %zd\n", peer->packet_type, peer->packet_tprefix, peer->packet_tcount); + } + + /* Ensure size for a minimal torrent block */ + if (data + sizeof(ot_hash) + OT_IP_SIZE + 3 > dataend) + break; + + /* Advance pointer to peer count or peers */ + hash = data; + data += sizeof(ot_hash) - 1; + + /* Type 0 has peer count encoded before each peers */ + peers = peer->packet_type; + if (!peers) { + int shift = 0; + do + peers |= (0x7f & *data) << (7 * shift); + while (*(data++) & 0x80 && shift++ < 6); + } +#if 0 +printf( "peers: %zd\n", peers ); +#endif + /* Ensure enough data being read to hold all peers */ + if (data + (OT_IP_SIZE + 3) * peers > dataend) { + data = hash; + break; + } + while (peers--) { + livesync_proxytell(peer->packet_tprefix, hash, data); + data += OT_IP_SIZE + 3; + } + --peer->packet_tcount; + } + + consumed = data - peer->indata; + memmove(peer->indata, data, peer->indata_length - consumed); + peer->indata_length -= consumed; +} + +static void *livesync_worker(void *args) { + (void)args; + while (1) { + ot_ip6 in_ip; + uint16_t in_port; + uint32_t packet_type; + ssize_t datalen = socket_recv(g_socket_in, g_inbuffer, LIVESYNC_INCOMING_BUFFSIZE, in_ip, &in_port); + + /* Expect at least tracker id and packet type */ + if (datalen <= (ssize_t)(sizeof(g_tracker_id) + sizeof(uint32_t))) + continue; + if (!memcmp(g_inbuffer, &g_tracker_id, sizeof(g_tracker_id))) { + /* drop packet coming from ourselves */ + continue; + } + memcpy(&packet_type, g_inbuffer + sizeof(g_tracker_id), sizeof(packet_type)); + + switch (ntohl(packet_type)) { + case OT_SYNC_PEER4: + livesync_handle_peersync(datalen, OT_PEER_SIZE4); + break; + case OT_SYNC_PEER6: + livesync_handle_peersync(datalen, OT_PEER_SIZE6); + break; + default: + // fprintf( stderr, "Received an unknown live sync packet type %u.\n", ntohl( packet_type ) ); + break; + } + } + return 0; +} diff --git a/src/trackerlogic.c b/src/trackerlogic.c new file mode 100644 index 0000000..9565090 --- /dev/null +++ b/src/trackerlogic.c @@ -0,0 +1,628 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include + + +/* Opentracker */ +#include "ot_accesslist.h" +#include "ot_clean.h" +#include "ot_fmt.h" +#include "ot_fullscrape.h" +#include "ot_http.h" +#include "ot_ip.h" +#include "ot_livesync.h" +#include "ot_mutex.h" +#include "ot_workqueue.h" +#include "ot_random.h" +#include "ot_stats.h" +#include "ot_udp.h" +#include "ot_vector.h" +#include "trackerlogic.h" + +/* Forward declaration */ +size_t return_peers_for_torrent(struct ot_workstruct *ws, ot_torrent *torrent, size_t amount, char *reply, PROTO_FLAG proto); + +void free_peerlist(ot_peerlist *peer_list) { + if (peer_list && peer_list->peers.data) { + if (OT_PEERLIST_HASBUCKETS(peer_list)) + vector_clean_list((ot_vector *)peer_list->peers.data, peer_list->peers.size); + else + free(peer_list->peers.data); + } + free(peer_list); +} + +void add_torrent_from_saved_state(ot_hash const hash, ot_time base, size_t down_count) { + int exactmatch; + ot_torrent *torrent; + ot_vector *torrents_list = mutex_bucket_lock_by_hash(hash); + + if (!accesslist_hashisvalid(hash)) { + mutex_bucket_unlock_by_hash(hash, 0); + return; + } + + torrent = vector_find_or_insert(torrents_list, (void *)hash, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); + if (!torrent || exactmatch) { + mutex_bucket_unlock_by_hash(hash, 0); + return; + } + + /* Create a new torrent entry, then */ + memset(torrent, 0, sizeof(ot_torrent)); + memcpy(torrent->hash, hash, sizeof(ot_hash)); + + if (!(torrent->peer_list6 = malloc(sizeof(ot_peerlist))) || !(torrent->peer_list4 = malloc(sizeof(ot_peerlist)))) { + vector_remove_torrent(torrents_list, torrent); + mutex_bucket_unlock_by_hash(hash, 0); + return; + } + + memset(torrent->peer_list6, 0, sizeof(ot_peerlist)); + memset(torrent->peer_list4, 0, sizeof(ot_peerlist)); + torrent->peer_list6->base = base; + torrent->peer_list4->base = base; + torrent->peer_list6->down_count = down_count; + torrent->peer_list4->down_count = down_count; + + mutex_bucket_unlock_by_hash(hash, 1); +} + +size_t add_peer_to_torrent_and_return_peers(PROTO_FLAG proto, struct ot_workstruct *ws, size_t amount) { + int exactmatch, delta_torrentcount = 0; + ot_torrent *torrent; + ot_peer *peer_dest; + ot_vector *torrents_list = mutex_bucket_lock_by_hash(*ws->hash); + ot_peerlist *peer_list; + size_t peer_size; /* initialized in next line */ + ot_peer const *peer_src = peer_from_peer6(&ws->peer, &peer_size); + + if (!accesslist_hashisvalid(*ws->hash)) { + mutex_bucket_unlock_by_hash(*ws->hash, 0); + if (proto == FLAG_TCP) { + const char invalid_hash[] = "d14:failure reason63:Requested download is not authorized for use with this tracker.e"; + memcpy(ws->reply, invalid_hash, strlen(invalid_hash)); + return strlen(invalid_hash); + } + return 0; + } + + torrent = vector_find_or_insert(torrents_list, (void *)ws->hash, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); + if (!torrent) { + mutex_bucket_unlock_by_hash(*ws->hash, 0); + return 0; + } + + if (!exactmatch) { + /* Create a new torrent entry, then */ + memset(torrent, 0, sizeof(ot_torrent)); + memcpy(torrent->hash, *ws->hash, sizeof(ot_hash)); + + if (!(torrent->peer_list6 = malloc(sizeof(ot_peerlist))) || !(torrent->peer_list4 = malloc(sizeof(ot_peerlist)))) { + vector_remove_torrent(torrents_list, torrent); + mutex_bucket_unlock_by_hash(*ws->hash, 0); + return 0; + } + + memset(torrent->peer_list6, 0, sizeof(ot_peerlist)); + memset(torrent->peer_list4, 0, sizeof(ot_peerlist)); + delta_torrentcount = 1; + } else + clean_single_torrent(torrent); + + torrent->peer_list6->base = time_now_minutes(); + torrent->peer_list4->base = time_now_minutes(); + + peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; + + /* Check for peer in torrent */ + peer_dest = vector_find_or_insert_peer(&(peer_list->peers), peer_src, peer_size, &exactmatch); + if (!peer_dest) { + mutex_bucket_unlock_by_hash(*ws->hash, delta_torrentcount); + return 0; + } + + /* Tell peer that it's fresh */ + OT_PEERTIME(ws->peer, OT_PEER_SIZE6) = 0; + + /* Sanitize flags: Whoever claims to have completed download, must be a seeder */ + if ((OT_PEERFLAG(ws->peer) & (PEER_FLAG_COMPLETED | PEER_FLAG_SEEDING)) == PEER_FLAG_COMPLETED) + OT_PEERFLAG(ws->peer) ^= PEER_FLAG_COMPLETED; + + /* If we hadn't had a match create peer there */ + if (!exactmatch) { + +#ifdef WANT_SYNC_LIVE + if (proto == FLAG_MCA) + OT_PEERFLAG(ws->peer) |= PEER_FLAG_FROM_SYNC; + else + livesync_tell(ws); +#endif + + peer_list->peer_count++; + if (OT_PEERFLAG(ws->peer) & PEER_FLAG_COMPLETED) { + peer_list->down_count++; + stats_issue_event(ws, EVENT_COMPLETED, 0, 0); + } + if (OT_PEERFLAG(ws->peer) & PEER_FLAG_SEEDING) + peer_list->seed_count++; + + } else { + stats_issue_event(ws, EVENT_RENEW, 0, OT_PEERTIME(peer_dest, peer_size)); +#ifdef WANT_SPOT_WOODPECKER + if ((OT_PEERTIME(peer_dest, peer_size) > 0) && (OT_PEERTIME(peer_dest, peer_size) < 20)) + stats_issue_event(ws, EVENT_WOODPECKER, 0, (uintptr_t)&ws->peer); +#endif +#ifdef WANT_SYNC_LIVE + /* Won't live sync peers that come back too fast. Only exception: + fresh "completed" reports */ + if (proto != FLAG_MCA) { + if (OT_PEERTIME(peer_dest, peer_size) > OT_CLIENT_SYNC_RENEW_BOUNDARY || + (!(OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_COMPLETED) && (OT_PEERFLAG(ws->peer) & PEER_FLAG_COMPLETED))) + livesync_tell(ws); + } +#endif + + if ((OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_SEEDING) && !(OT_PEERFLAG(ws->peer) & PEER_FLAG_SEEDING)) + peer_list->seed_count--; + if (!(OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_SEEDING) && (OT_PEERFLAG(ws->peer) & PEER_FLAG_SEEDING)) + peer_list->seed_count++; + if (!(OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_COMPLETED) && (OT_PEERFLAG(ws->peer) & PEER_FLAG_COMPLETED)) { + peer_list->down_count++; + stats_issue_event(ws, EVENT_COMPLETED, 0, 0); + } + if (OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_COMPLETED) + OT_PEERFLAG(ws->peer) |= PEER_FLAG_COMPLETED; + } + + memcpy(peer_dest, peer_src, peer_size); +#ifdef WANT_SYNC + if (proto == FLAG_MCA) { + mutex_bucket_unlock_by_hash(*ws->hash, delta_torrentcount); + return 0; + } +#endif + + ws->reply_size = return_peers_for_torrent(ws, torrent, amount, ws->reply, proto); + mutex_bucket_unlock_by_hash(*ws->hash, delta_torrentcount); + return ws->reply_size; +} + +static size_t return_peers_all(ot_peerlist *peer_list, size_t peer_size, char *reply) { + size_t bucket, num_buckets = 1; + ot_vector *bucket_list = &peer_list->peers; + size_t compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size); + size_t result = compare_size * peer_list->peer_count; + char *r_end = reply + result; + + if (OT_PEERLIST_HASBUCKETS(peer_list)) { + num_buckets = bucket_list->size; + bucket_list = (ot_vector *)bucket_list->data; + } + + for (bucket = 0; bucket < num_buckets; ++bucket) { + ot_peer *peers = bucket_list[bucket].data; + size_t peer_count = bucket_list[bucket].size; + while (peer_count--) { + if (OT_PEERFLAG_D(peers, peer_size) & PEER_FLAG_SEEDING) { + r_end -= compare_size; + memcpy(r_end, peers, compare_size); + } else { + memcpy(reply, peers, compare_size); + reply += compare_size; + } + peers += peer_size; + } + } + return result; +} + +static size_t return_peers_selection(struct ot_workstruct *ws, ot_peerlist *peer_list, size_t peer_size, size_t amount, char *reply) { + size_t bucket_offset, bucket_index = 0, num_buckets = 1; + ot_vector *bucket_list = &peer_list->peers; + unsigned int shifted_pc = peer_list->peer_count; + unsigned int shifted_step = 0; + unsigned int shift = 0; + size_t compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size); + size_t result = compare_size * amount; + char *r_end = reply + result; + + if (OT_PEERLIST_HASBUCKETS(peer_list)) { + num_buckets = bucket_list->size; + bucket_list = (ot_vector *)bucket_list->data; + } + + /* Make fixpoint arithmetic as exact as possible */ +#define MAXPRECBIT (1 << (8 * sizeof(int) - 3)) + while (!(shifted_pc & MAXPRECBIT)) { + shifted_pc <<= 1; + shift++; + } + shifted_step = shifted_pc / amount; +#undef MAXPRECBIT + + /* Initialize somewhere in the middle of peers so that + fixpoint's aliasing doesn't alway miss the same peers */ + bucket_offset = nrand48(ws->rand48_state) % peer_list->peer_count; + + while (amount--) { + ot_peer *peer; + + /* This is the aliased, non shifted range, next value may fall into */ + unsigned int diff = (((amount + 1) * shifted_step) >> shift) - ((amount * shifted_step) >> shift); + bucket_offset += 1 + nrand48(ws->rand48_state) % diff; + + while (bucket_offset >= bucket_list[bucket_index].size) { + bucket_offset -= bucket_list[bucket_index].size; + bucket_index = (bucket_index + 1) % num_buckets; + } + peer = (ot_peer*)bucket_list[bucket_index].data + peer_size * bucket_offset; + if (OT_PEERFLAG_D(peer, peer_size) & PEER_FLAG_SEEDING) { + r_end -= compare_size; + memcpy(r_end, peer, compare_size); + } else { + memcpy(reply, peer, compare_size); + reply += compare_size; + } + } + return result; +} + +static size_t return_peers_for_torrent_udp(struct ot_workstruct *ws, ot_torrent *torrent, size_t amount, char *reply) { + char *r = reply; + size_t peer_size = peer_size_from_peer6(&ws->peer); + ot_peerlist *peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; + size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; + size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; + + if (amount > peer_list->peer_count) + amount = peer_list->peer_count; + + *(uint32_t *)(r + 0) = htonl(OT_CLIENT_REQUEST_INTERVAL_RANDOM); + *(uint32_t *)(r + 4) = htonl(peer_count - seed_count); + *(uint32_t *)(r + 8) = htonl(seed_count); + r += 12; + + if (amount) { + if (amount == peer_list->peer_count) + r += return_peers_all(peer_list, peer_size, r); + else + r += return_peers_selection(ws, peer_list, peer_size, amount, r); + } + return r - reply; +} + +static size_t return_peers_for_torrent_tcp(struct ot_workstruct *ws, ot_torrent *torrent, size_t amount, char *reply) { + char *r = reply; + const char *r_end = ws->outbuf + G_OUTBUF_SIZE; + int erval = OT_CLIENT_REQUEST_INTERVAL_RANDOM; + size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; + size_t down_count = torrent->peer_list6->down_count + torrent->peer_list4->down_count; + size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - seed_count; + + /* Simple case: amount of peers in both lists is less than requested, here we return all results */ + size_t amount_v4 = torrent->peer_list4->peer_count; + size_t amount_v6 = torrent->peer_list6->peer_count; + + /* Complex case: both lists have more than enough entries and we need to split between v4 and v6 clients */ + if (amount_v4 + amount_v6 > amount) { + size_t amount_left, percent_v6 = 0, percent_v4 = 0, left_v6, left_v4; + const size_t SCALE = 1024; + + /* If possible, fill at least a quarter of peer from each family */ + if (amount / 4 <= amount_v4) + amount_v4 = amount / 4; + if (amount / 4 <= amount_v6) + amount_v6 = amount / 4; + + /* Fill the rest according to which family's pool provides more peers */ + amount_left = amount - (amount_v4 + amount_v6); + + left_v4 = torrent->peer_list4->peer_count - amount_v4; + left_v6 = torrent->peer_list6->peer_count - amount_v6; + + if (left_v4 + left_v6) { + percent_v4 = (SCALE * left_v4) / (left_v4 + left_v6); + percent_v6 = (SCALE * left_v6) / (left_v4 + left_v6); + } + + amount_v4 += (amount_left * percent_v4) / SCALE; + amount_v6 += (amount_left * percent_v6) / SCALE; + + /* Integer division rounding can leave out a peer */ + if (amount_v4 + amount_v6 < amount && amount_v6 < torrent->peer_list6->peer_count) + ++amount_v6; + if (amount_v4 + amount_v6 < amount && amount_v4 < torrent->peer_list4->peer_count) + ++amount_v4; + } + + fmt_append(&r, r_end, "d8:completei%zde10:downloadedi%zde10:incompletei%zde8:intervali%ie12:min intervali%ie", seed_count, down_count, peer_count, erval, + erval / 2); + + if (amount_v4) { + fmt_append(&r, r_end, PEERS_BENCODED4 "%zd:", OT_PEER_COMPARE_SIZE4 * amount_v4); + if (amount_v4 == torrent->peer_list4->peer_count) + r += return_peers_all(torrent->peer_list4, OT_PEER_SIZE4, r); + else + r += return_peers_selection(ws, torrent->peer_list4, OT_PEER_SIZE4, amount_v4, r); + } + + if (amount_v6) { + fmt_append(&r, r_end, PEERS_BENCODED6 "%zd:", OT_PEER_COMPARE_SIZE6 * amount_v6); + if (amount_v6 == torrent->peer_list6->peer_count) + r += return_peers_all(torrent->peer_list6, OT_PEER_SIZE6, r); + else + r += return_peers_selection(ws, torrent->peer_list6, OT_PEER_SIZE6, amount_v6, r); + } + + fmt_append(&r, r_end, "e"); + + return r - reply; +} + +/* Compiles a list of random peers for a torrent + * Reply must have enough space to hold: + * 92 + 6 * amount bytes for TCP/IPv4 + * 92 + 18 * amount bytes for TCP/IPv6 + * 12 + 6 * amount bytes for UDP/IPv4 + * 12 + 18 * amount bytes for UDP/IPv6 + * Does not yet check not to return self + */ +size_t return_peers_for_torrent(struct ot_workstruct *ws, ot_torrent *torrent, size_t amount, char *reply, PROTO_FLAG proto) { + return proto == FLAG_TCP ? return_peers_for_torrent_tcp(ws, torrent, amount, reply) : return_peers_for_torrent_udp(ws, torrent, amount, reply); +} + +/* Fetches scrape info for a specific torrent */ +size_t return_udp_scrape_for_torrent(ot_hash const hash, char *reply) { + int exactmatch, delta_torrentcount = 0; + ot_vector *torrents_list = mutex_bucket_lock_by_hash(hash); + ot_torrent *torrent = vector_binary_search(hash, torrents_list->data, torrents_list->size, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); + + if (!exactmatch) { + memset(reply, 0, 12); + } else { + uint32_t *r = (uint32_t *)reply; + + if (clean_single_torrent(torrent)) { + vector_remove_torrent(torrents_list, torrent); + memset(reply, 0, 12); + delta_torrentcount = -1; + } else { + r[0] = htonl(torrent->peer_list6->seed_count + torrent->peer_list4->seed_count); + r[1] = htonl(torrent->peer_list6->down_count + torrent->peer_list4->down_count); + r[2] = htonl(torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - torrent->peer_list6->seed_count - torrent->peer_list4->seed_count); + } + } + mutex_bucket_unlock_by_hash(hash, delta_torrentcount); + return 12; +} + +/* Fetches scrape info for a specific torrent */ +void return_tcp_scrape_for_torrent(ot_hash const *hash_list, size_t amount, char **reply, const char *reply_end) { + int exactmatch; + size_t i; + + fmt_append(reply, reply_end, "d5:filesd"); + + for (i = 0; i < amount; ++i) { + int delta_torrentcount = 0; + ot_hash const *hash = hash_list + i; + ot_vector *torrents_list = mutex_bucket_lock_by_hash(*hash); + ot_torrent *torrent = vector_binary_search(hash, torrents_list->data, torrents_list->size, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); + + if (exactmatch) { + if (clean_single_torrent(torrent)) { + vector_remove_torrent(torrents_list, torrent); + delta_torrentcount = -1; + } else { + fmt_append(reply, reply_end, "20:"); + fmt_bytes(reply, reply_end, hash, sizeof(ot_hash)); + fmt_append(reply, reply_end, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", + torrent->peer_list6->seed_count + torrent->peer_list4->seed_count, torrent->peer_list6->down_count + torrent->peer_list4->down_count, + torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - torrent->peer_list6->seed_count - torrent->peer_list4->seed_count); + } + } + mutex_bucket_unlock_by_hash(*hash, delta_torrentcount); + } + + fmt_append(reply, reply_end, "ee"); +} + +static ot_peerlist dummy_list; +size_t remove_peer_from_torrent(PROTO_FLAG proto, struct ot_workstruct *ws) { + int exactmatch; + ot_vector *torrents_list = mutex_bucket_lock_by_hash(*ws->hash); + ot_torrent *torrent = vector_binary_search(ws->hash, torrents_list->data, torrents_list->size, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); + ot_peerlist *peer_list = &dummy_list; + size_t peer_size; /* initialized in next line */ + ot_peer const *peer_src = peer_from_peer6(&ws->peer, &peer_size); + size_t peer_count = 0, seed_count = 0; + +#ifdef WANT_SYNC_LIVE + if (proto != FLAG_MCA) { + OT_PEERFLAG(ws->peer) |= PEER_FLAG_STOPPED; + livesync_tell(ws); + } +#endif + + if (exactmatch) { + peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; + switch (vector_remove_peer(&peer_list->peers, peer_src, peer_size)) { + case 2: + peer_list->seed_count--; /* Intentional fallthrough */ + case 1: + peer_list->peer_count--; /* Intentional fallthrough */ + default: + break; + } + + peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; + seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; + } + + if (proto == FLAG_TCP) { + int erval = OT_CLIENT_REQUEST_INTERVAL_RANDOM; + char *r = ws->reply; + + fmt_append(&r, ws->outbuf + G_OUTBUF_SIZE, "d8:completei%zde10:incompletei%zde8:intervali%ie12:min intervali%ie%s0:e", seed_count, peer_count - seed_count, + erval, erval / 2, peer_size == OT_PEER_SIZE6 ? PEERS_BENCODED6 : PEERS_BENCODED4); + ws->reply_size = r - ws->reply; + } + + /* Handle UDP reply */ + if (proto == FLAG_UDP) { + ((uint32_t *)ws->reply)[2] = htonl(OT_CLIENT_REQUEST_INTERVAL_RANDOM); + ((uint32_t *)ws->reply)[3] = htonl(peer_count - seed_count); + ((uint32_t *)ws->reply)[4] = htonl(seed_count); + ws->reply_size = 20; + } + + mutex_bucket_unlock_by_hash(*ws->hash, 0); + return ws->reply_size; +} + +void iterate_all_torrents(int (*for_each)(ot_torrent *torrent, uintptr_t data), uintptr_t data) { + size_t bucket, j; + + for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { + ot_vector *torrents_list = mutex_bucket_lock(bucket); + ot_torrent *torrents = (ot_torrent *)(torrents_list->data); + + for (j = 0; j < torrents_list->size; ++j) + if (for_each(torrents + j, data)) + break; + + mutex_bucket_unlock(bucket, 0); + if (!g_opentracker_running) + return; + } +} + +ot_peer *peer_from_peer6(ot_peer6 *peer, size_t *peer_size) { + ot_ip6 *ip = (ot_ip6 *)peer; + if (!ip_is_v4mapped(ip)) { + *peer_size = OT_PEER_SIZE6; + return (ot_peer *)peer; + } + *peer_size = OT_PEER_SIZE4; + return (ot_peer *)(((uint8_t *)peer) + 12); +} + +size_t peer_size_from_peer6(ot_peer6 *peer) { + ot_ip6 *ip = (ot_ip6 *)peer; + if (!ip_is_v4mapped(ip)) + return OT_PEER_SIZE6; + return OT_PEER_SIZE4; +} + +#ifdef _DEBUG_RANDOMTORRENTS +void trackerlogic_add_random_torrents(size_t amount) { + struct ot_workstruct ws; + memset(&ws, 0, sizeof(ws)); + + ws.inbuf = malloc(G_INBUF_SIZE); + ws.outbuf = malloc(G_OUTBUF_SIZE); + ws.reply = ws.outbuf; + ws.hash = (ot_hash *)ws.inbuf; + + while (amount--) { + random_bytes(ws.hash, sizeof(ot_hash)); + random_bytes(&ws.peer, sizeof(ws.peer)); + + OT_PEERFLAG(ws.peer) &= PEER_FLAG_SEEDING | PEER_FLAG_COMPLETED | PEER_FLAG_STOPPED; + + add_peer_to_torrent_and_return_peers(FLAG_TCP, &ws, 1); + } + + free(ws.inbuf); + free(ws.outbuf); +} +#endif + +void exerr(const char *message) { + fprintf(stderr, "%s\n", message); + exit(111); +} + +void trackerlogic_init(void) { + g_tracker_id = random_uint32(); + + if (!g_stats_path) + g_stats_path = "stats"; + g_stats_path_len = strlen(g_stats_path); + + /* Initialise background worker threads */ + mutex_init(); + workqueue_init(); + clean_init(); + fullscrape_init(); + accesslist_init(); + livesync_init(); + stats_init(); +} + +int workstruct_init(struct ot_workstruct *ws) { + memset(ws, 0, sizeof(*ws)); + ws->inbuf = malloc(G_INBUF_SIZE); + ws->outbuf = malloc(G_OUTBUF_SIZE); +#ifdef _DEBUG_HTTPERROR + ws->debugbuf = malloc(G_DEBUGBUF_SIZE); +#endif + if (!ws->inbuf || !ws->outbuf) + return -1; + + /* Picking which peers to hand out starts here, so two workers seeded alike + would hand out alike */ + random_bytes(ws->rand48_state, sizeof(ws->rand48_state)); + + stats_counters_register(ws); + return 0; +} + +void workstruct_deinit(struct ot_workstruct *ws) { + /* The tally goes to the global before this struct goes out of scope */ + stats_counters_retire(ws); + + free(ws->inbuf); + free(ws->outbuf); +#ifdef _DEBUG_HTTPERROR + free(ws->debugbuf); +#endif +} + +void trackerlogic_deinit(void) { + int delta_torrentcount = 0; + size_t bucket, j; + + /* Stop the background workers first */ + stats_deinit(); + livesync_deinit(); + accesslist_deinit(); + fullscrape_deinit(); + clean_deinit(); + + /* Free all torrents... */ + for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { + ot_vector *torrents_list = mutex_bucket_lock(bucket); + if (torrents_list->size) { + for (j = 0; j < torrents_list->size; ++j) { + ot_torrent *torrent = ((ot_torrent *)(torrents_list->data)) + j; + free_peerlist(torrent->peer_list6); + free_peerlist(torrent->peer_list4); + delta_torrentcount -= 1; + } + free(torrents_list->data); + } + mutex_bucket_unlock(bucket, delta_torrentcount); + } + + /* Release mutexes */ + workqueue_deinit(); + mutex_deinit(); +} diff --git a/src/trackerlogic.h b/src/trackerlogic.h new file mode 100644 index 0000000..7292d36 --- /dev/null +++ b/src/trackerlogic.h @@ -0,0 +1,283 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_TRACKERLOGIC_H__ +#define OT_TRACKERLOGIC_H__ + +#include +#include +#include +#include +#include +#include +#include + +typedef uint8_t ot_hash[20]; +typedef time_t ot_time; +typedef char ot_ip6[16]; +typedef struct { + ot_ip6 address; + int bits; +} ot_net; +/* List of peers should fit in a single UDP packet (around 1200 bytes) */ +#define OT_MAX_PEERS_UDP6 66 +#define OT_MAX_PEERS_UDP4 200 + +#define OT_IP_SIZE6 16 +#define OT_IP_SIZE4 4 +#define OT_PORT_SIZE 2 +#define OT_FLAG_SIZE 1 +#define OT_TIME_SIZE 1 + +/* Some tracker behaviour tunable */ +#ifndef OT_CLIENT_TIMEOUT +#define OT_CLIENT_TIMEOUT 30 +#endif +#ifndef OT_CLIENT_TIMEOUT_CHECKINTERVAL +#define OT_CLIENT_TIMEOUT_CHECKINTERVAL 10 +#endif +#define OT_CLIENT_TIMEOUT_SEND (60 * 15) +#define OT_CLIENT_REQUEST_INTERVAL (60 * 30) +#define OT_CLIENT_REQUEST_VARIATION (60 * 6) + +#define OT_TORRENT_TIMEOUT_HOURS 24 +#define OT_TORRENT_TIMEOUT (60 * OT_TORRENT_TIMEOUT_HOURS) + +#define OT_CLIENT_REQUEST_INTERVAL_RANDOM \ + (OT_CLIENT_REQUEST_INTERVAL - OT_CLIENT_REQUEST_VARIATION / 2 + (int)(nrand48(ws->rand48_state) % OT_CLIENT_REQUEST_VARIATION)) + +/* If WANT_MODEST_FULLSCRAPES is on, ip addresses may not + fullscrape more frequently than this amount in seconds */ +#define OT_MODEST_PEER_TIMEOUT (60 * 5) + +/* If peers come back before 10 minutes, don't live sync them */ +#define OT_CLIENT_SYNC_RENEW_BOUNDARY 10 + +/* Number of tracker admin ip addresses allowed */ +#define OT_ADMINIP_MAX 64 +#define OT_MAX_THREADS 64 + +/* Number of minutes after announce before peer is removed */ +#define OT_PEER_TIMEOUT 45 + +/* We maintain a list of 1024 pointers to sorted list of ot_torrent structs + Sort key is, of course, its hash */ +/* Set from the Makefile, 8 to 16. More buckets means shorter vectors and a + cheaper insert, which only pays for a tracker carrying millions of + torrents; see DESIGN. */ +#ifndef OT_BUCKET_COUNT_BITS +#define OT_BUCKET_COUNT_BITS 10 +#endif + +#define OT_BUCKET_COUNT (1 << OT_BUCKET_COUNT_BITS) +#define OT_BUCKET_COUNT_SHIFT (32 - OT_BUCKET_COUNT_BITS) + +/* if _DEBUG_RANDOMTORRENTS is set, this is the amount of torrents to create + on startup */ +#define RANDOMTORRENTS (1024 * 1024 * 1) + +#if defined(CLOCK_REALTIME_COARSE) +#define OT_CLOCK_CHEAP CLOCK_REALTIME_COARSE /* Linux */ +#elif defined(CLOCK_REALTIME_FAST) +#define OT_CLOCK_CHEAP CLOCK_REALTIME_FAST /* the BSDs */ +#else +#define OT_CLOCK_CHEAP CLOCK_REALTIME /* macOS and whatever else */ +#endif + +static inline ot_time time_now_seconds(void) { + struct timespec now; + clock_gettime(OT_CLOCK_CHEAP, &now); + return (ot_time)now.tv_sec; +} + +static inline ot_time time_now_minutes(void) { return time_now_seconds() / 60; } + +/* For the deadlines the tracker sets on itself, which read better as the + span they allow than as an addition to the current second */ +static inline ot_time time_future_seconds(int span) { return time_now_seconds() + span; } +/* Cleared from the interrupt handler and read by every worker */ +extern _Atomic int g_opentracker_running; + +/* Sent to a thread parked in a blocking read */ +#define OT_SIGNAL_WAKE SIGUSR1 + + +extern uint32_t g_tracker_id; +typedef enum { FLAG_TCP, FLAG_UDP, FLAG_MCA, FLAG_SELFPIPE } PROTO_FLAG; + +#define OT_PEER_COMPARE_SIZE6 ((OT_IP_SIZE6) + (OT_PORT_SIZE)) +#define OT_PEER_COMPARE_SIZE4 ((OT_IP_SIZE4) + (OT_PORT_SIZE)) +#define OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(PEER_SIZE) ((PEER_SIZE) - (OT_TIME_SIZE) - (OT_FLAG_SIZE)) + +#define OT_PEER_SIZE6 ((OT_TIME_SIZE) + (OT_FLAG_SIZE) + (OT_PEER_COMPARE_SIZE6)) +#define OT_PEER_SIZE4 ((OT_TIME_SIZE) + (OT_FLAG_SIZE) + (OT_PEER_COMPARE_SIZE4)) + +typedef uint8_t ot_peer; /* Generic pointer to a v6 or v4 peer */ +typedef uint8_t ot_peer6[OT_PEER_SIZE6]; +typedef uint8_t ot_peer4[OT_PEER_SIZE4]; +static const uint8_t PEER_FLAG_SEEDING = 0x80; +static const uint8_t PEER_FLAG_COMPLETED = 0x40; +static const uint8_t PEER_FLAG_STOPPED = 0x20; +static const uint8_t PEER_FLAG_FROM_SYNC = 0x10; +static const uint8_t PEER_FLAG_LEECHING = 0x00; + +/* Takes an ot_peer6 and returns the proper pointer to the peer and sets peer_size */ +ot_peer *peer_from_peer6(ot_peer6 *peer, size_t *peer_size); +size_t peer_size_from_peer6(ot_peer6 *peer); + +/* New style */ +#define OT_SETIP(peer, ip) memcpy((uint8_t *)(peer), (ip), OT_IP_SIZE6) +#define OT_SETPORT(peer, port) memcpy(((uint8_t *)(peer)) + (OT_IP_SIZE6), (port), 2) +#define OT_PEERFLAG(peer) (((uint8_t *)(peer))[(OT_IP_SIZE6) + 2]) +#define OT_PEERFLAG_D(peer, peersize) (((uint8_t *)(peer))[(peersize) - 2]) +#define OT_PEERTIME(peer, peersize) (((uint8_t *)(peer))[(peersize) - 1]) + +#define PEERS_BENCODED6 "6:peers6" +#define PEERS_BENCODED4 "5:peers" + +#define OT_HASH_COMPARE_SIZE (sizeof(ot_hash)) + +struct ot_peerlist; +typedef struct ot_peerlist ot_peerlist; +typedef struct { + ot_hash hash; + ot_peerlist *peer_list6; + ot_peerlist *peer_list4; +} ot_torrent; + +#include "ot_vector.h" + +struct ot_peerlist { + ot_time base; + size_t seed_count; + size_t peer_count; + size_t down_count; + /* normal peers vector or + pointer to ot_vector[32] buckets if data != NULL and space == 0 + */ + ot_vector peers; +}; +#define OT_PEERLIST_HASBUCKETS(peer_list) ((peer_list)->peers.size > (peer_list)->peers.space) + +/* The http replies the tracker counts when it has to refuse a request. */ +enum { + CODE_HTTPERROR_302, + CODE_HTTPERROR_400, + CODE_HTTPERROR_400_PARAM, + CODE_HTTPERROR_400_COMPACT, + CODE_HTTPERROR_402_NOTMODEST, + CODE_HTTPERROR_402_PAYMENT_REQUIRED, + CODE_HTTPERROR_403_IP, + CODE_HTTPERROR_404, + CODE_HTTPERROR_500, + + CODE_HTTPERROR_COUNT +}; + +/* One statistic. Used when workers report stats to their workstruct */ +typedef _Atomic unsigned long long ot_counter; + +/* What each thread counts for itself. The two runs at the end are indexed by + the http error code and by the age in minutes of a renewed peer */ +typedef enum { + OT_COUNTER_TCP_CONNECTIONS, + OT_COUNTER_UDP_CONNECTIONS, + OT_COUNTER_TCP_ANNOUNCES, + OT_COUNTER_UDP_ANNOUNCES, + OT_COUNTER_TCP_SCRAPES, + OT_COUNTER_UDP_SCRAPES, + OT_COUNTER_TCP_CONNECTS, + OT_COUNTER_UDP_CONNECTS, + OT_COUNTER_UDP_CONNID_MISSMATCH, + OT_COUNTER_COMPLETED, + OT_COUNTER_FULLSCRAPES, + OT_COUNTER_FULLSCRAPE_REQUESTS, + OT_COUNTER_FULLSCRAPE_BYTES, + OT_COUNTER_SYNCS, + OT_COUNTER_STALLS, + + OT_COUNTER_FAILED, /* CODE_HTTPERROR_COUNT of them */ + OT_COUNTER_RENEWED = OT_COUNTER_FAILED + CODE_HTTPERROR_COUNT, /* OT_PEER_TIMEOUT of them */ + + OT_COUNTER_COUNT = OT_COUNTER_RENEWED + OT_PEER_TIMEOUT +} ot_counter_index; + +struct ot_workstruct { + /* Thread specific, static */ + + /* This thread's own statistics, and the link ot_stats keeps them on */ + ot_counter counters[OT_COUNTER_COUNT]; + struct ot_workstruct *counters_next; + + char *inbuf; +#define G_INBUF_SIZE 8192 + char *outbuf; +#define G_OUTBUF_SIZE 8192 +#ifdef _DEBUG_HTTPERROR + char *debugbuf; +#define G_DEBUGBUF_SIZE 8192 +#endif + + /* The peer currently in the working */ + ot_peer6 peer; /* Can fit v6 and v4 peers */ + + /* Pointers into the request buffer */ + ot_hash *hash; + char *peer_id; + + /* HTTP specific, non static */ + char *request; + ssize_t request_size; + ssize_t header_size; + char *reply; + ssize_t reply_size; + + /* Entropy state for rand48 function so that threads don't need to acquire mutexes for + global random() or arc4random() state, which causes heavy load on linuxes */ + uint16_t rand48_state[3]; + + int keep_alive; +}; + +/* Everything a thread that serves requests needs for itself. Set up and torn + down in one place so that a new kind of worker cannot miss a piece. + Returns -1 if the buffers could not be had. */ +int workstruct_init(struct ot_workstruct *ws); +void workstruct_deinit(struct ot_workstruct *ws); + +/* + Exported functions +*/ + +#ifdef WANT_SYNC_LIVE +#define WANT_SYNC +#endif + +#ifdef WANT_SYNC +#define WANT_SYNC_PARAM(param) , param +#else +#define WANT_SYNC_PARAM(param) +#endif + +void trackerlogic_init(void); +void trackerlogic_deinit(void); +void exerr(const char *message); + +/* add_peer_to_torrent does only release the torrent bucket if from_sync is set, + otherwise it is released in return_peers_for_torrent */ +size_t add_peer_to_torrent_and_return_peers(PROTO_FLAG proto, struct ot_workstruct *ws, size_t amount); +size_t remove_peer_from_torrent(PROTO_FLAG proto, struct ot_workstruct *ws); +void return_tcp_scrape_for_torrent(ot_hash const *hash_list, size_t amount, char **reply, const char *reply_end); +size_t return_udp_scrape_for_torrent(ot_hash const hash, char *reply); +void add_torrent_from_saved_state(ot_hash const hash, ot_time base, size_t down_count); +#ifdef _DEBUG_RANDOMTORRENTS +void trackerlogic_add_random_torrents(size_t amount); +#endif + +/* torrent iterator */ +void iterate_all_torrents(int (*for_each)(ot_torrent *torrent, uintptr_t data), uintptr_t data); + +/* Helper, before it moves to its own object */ +void free_peerlist(ot_peerlist *peer_list); + +#endif diff --git a/trackerlogic.c b/trackerlogic.c deleted file mode 100644 index 9565090..0000000 --- a/trackerlogic.c +++ /dev/null @@ -1,628 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -/* System */ -#include -#include -#include -#include -#include - - -/* Opentracker */ -#include "ot_accesslist.h" -#include "ot_clean.h" -#include "ot_fmt.h" -#include "ot_fullscrape.h" -#include "ot_http.h" -#include "ot_ip.h" -#include "ot_livesync.h" -#include "ot_mutex.h" -#include "ot_workqueue.h" -#include "ot_random.h" -#include "ot_stats.h" -#include "ot_udp.h" -#include "ot_vector.h" -#include "trackerlogic.h" - -/* Forward declaration */ -size_t return_peers_for_torrent(struct ot_workstruct *ws, ot_torrent *torrent, size_t amount, char *reply, PROTO_FLAG proto); - -void free_peerlist(ot_peerlist *peer_list) { - if (peer_list && peer_list->peers.data) { - if (OT_PEERLIST_HASBUCKETS(peer_list)) - vector_clean_list((ot_vector *)peer_list->peers.data, peer_list->peers.size); - else - free(peer_list->peers.data); - } - free(peer_list); -} - -void add_torrent_from_saved_state(ot_hash const hash, ot_time base, size_t down_count) { - int exactmatch; - ot_torrent *torrent; - ot_vector *torrents_list = mutex_bucket_lock_by_hash(hash); - - if (!accesslist_hashisvalid(hash)) { - mutex_bucket_unlock_by_hash(hash, 0); - return; - } - - torrent = vector_find_or_insert(torrents_list, (void *)hash, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); - if (!torrent || exactmatch) { - mutex_bucket_unlock_by_hash(hash, 0); - return; - } - - /* Create a new torrent entry, then */ - memset(torrent, 0, sizeof(ot_torrent)); - memcpy(torrent->hash, hash, sizeof(ot_hash)); - - if (!(torrent->peer_list6 = malloc(sizeof(ot_peerlist))) || !(torrent->peer_list4 = malloc(sizeof(ot_peerlist)))) { - vector_remove_torrent(torrents_list, torrent); - mutex_bucket_unlock_by_hash(hash, 0); - return; - } - - memset(torrent->peer_list6, 0, sizeof(ot_peerlist)); - memset(torrent->peer_list4, 0, sizeof(ot_peerlist)); - torrent->peer_list6->base = base; - torrent->peer_list4->base = base; - torrent->peer_list6->down_count = down_count; - torrent->peer_list4->down_count = down_count; - - mutex_bucket_unlock_by_hash(hash, 1); -} - -size_t add_peer_to_torrent_and_return_peers(PROTO_FLAG proto, struct ot_workstruct *ws, size_t amount) { - int exactmatch, delta_torrentcount = 0; - ot_torrent *torrent; - ot_peer *peer_dest; - ot_vector *torrents_list = mutex_bucket_lock_by_hash(*ws->hash); - ot_peerlist *peer_list; - size_t peer_size; /* initialized in next line */ - ot_peer const *peer_src = peer_from_peer6(&ws->peer, &peer_size); - - if (!accesslist_hashisvalid(*ws->hash)) { - mutex_bucket_unlock_by_hash(*ws->hash, 0); - if (proto == FLAG_TCP) { - const char invalid_hash[] = "d14:failure reason63:Requested download is not authorized for use with this tracker.e"; - memcpy(ws->reply, invalid_hash, strlen(invalid_hash)); - return strlen(invalid_hash); - } - return 0; - } - - torrent = vector_find_or_insert(torrents_list, (void *)ws->hash, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); - if (!torrent) { - mutex_bucket_unlock_by_hash(*ws->hash, 0); - return 0; - } - - if (!exactmatch) { - /* Create a new torrent entry, then */ - memset(torrent, 0, sizeof(ot_torrent)); - memcpy(torrent->hash, *ws->hash, sizeof(ot_hash)); - - if (!(torrent->peer_list6 = malloc(sizeof(ot_peerlist))) || !(torrent->peer_list4 = malloc(sizeof(ot_peerlist)))) { - vector_remove_torrent(torrents_list, torrent); - mutex_bucket_unlock_by_hash(*ws->hash, 0); - return 0; - } - - memset(torrent->peer_list6, 0, sizeof(ot_peerlist)); - memset(torrent->peer_list4, 0, sizeof(ot_peerlist)); - delta_torrentcount = 1; - } else - clean_single_torrent(torrent); - - torrent->peer_list6->base = time_now_minutes(); - torrent->peer_list4->base = time_now_minutes(); - - peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; - - /* Check for peer in torrent */ - peer_dest = vector_find_or_insert_peer(&(peer_list->peers), peer_src, peer_size, &exactmatch); - if (!peer_dest) { - mutex_bucket_unlock_by_hash(*ws->hash, delta_torrentcount); - return 0; - } - - /* Tell peer that it's fresh */ - OT_PEERTIME(ws->peer, OT_PEER_SIZE6) = 0; - - /* Sanitize flags: Whoever claims to have completed download, must be a seeder */ - if ((OT_PEERFLAG(ws->peer) & (PEER_FLAG_COMPLETED | PEER_FLAG_SEEDING)) == PEER_FLAG_COMPLETED) - OT_PEERFLAG(ws->peer) ^= PEER_FLAG_COMPLETED; - - /* If we hadn't had a match create peer there */ - if (!exactmatch) { - -#ifdef WANT_SYNC_LIVE - if (proto == FLAG_MCA) - OT_PEERFLAG(ws->peer) |= PEER_FLAG_FROM_SYNC; - else - livesync_tell(ws); -#endif - - peer_list->peer_count++; - if (OT_PEERFLAG(ws->peer) & PEER_FLAG_COMPLETED) { - peer_list->down_count++; - stats_issue_event(ws, EVENT_COMPLETED, 0, 0); - } - if (OT_PEERFLAG(ws->peer) & PEER_FLAG_SEEDING) - peer_list->seed_count++; - - } else { - stats_issue_event(ws, EVENT_RENEW, 0, OT_PEERTIME(peer_dest, peer_size)); -#ifdef WANT_SPOT_WOODPECKER - if ((OT_PEERTIME(peer_dest, peer_size) > 0) && (OT_PEERTIME(peer_dest, peer_size) < 20)) - stats_issue_event(ws, EVENT_WOODPECKER, 0, (uintptr_t)&ws->peer); -#endif -#ifdef WANT_SYNC_LIVE - /* Won't live sync peers that come back too fast. Only exception: - fresh "completed" reports */ - if (proto != FLAG_MCA) { - if (OT_PEERTIME(peer_dest, peer_size) > OT_CLIENT_SYNC_RENEW_BOUNDARY || - (!(OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_COMPLETED) && (OT_PEERFLAG(ws->peer) & PEER_FLAG_COMPLETED))) - livesync_tell(ws); - } -#endif - - if ((OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_SEEDING) && !(OT_PEERFLAG(ws->peer) & PEER_FLAG_SEEDING)) - peer_list->seed_count--; - if (!(OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_SEEDING) && (OT_PEERFLAG(ws->peer) & PEER_FLAG_SEEDING)) - peer_list->seed_count++; - if (!(OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_COMPLETED) && (OT_PEERFLAG(ws->peer) & PEER_FLAG_COMPLETED)) { - peer_list->down_count++; - stats_issue_event(ws, EVENT_COMPLETED, 0, 0); - } - if (OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_COMPLETED) - OT_PEERFLAG(ws->peer) |= PEER_FLAG_COMPLETED; - } - - memcpy(peer_dest, peer_src, peer_size); -#ifdef WANT_SYNC - if (proto == FLAG_MCA) { - mutex_bucket_unlock_by_hash(*ws->hash, delta_torrentcount); - return 0; - } -#endif - - ws->reply_size = return_peers_for_torrent(ws, torrent, amount, ws->reply, proto); - mutex_bucket_unlock_by_hash(*ws->hash, delta_torrentcount); - return ws->reply_size; -} - -static size_t return_peers_all(ot_peerlist *peer_list, size_t peer_size, char *reply) { - size_t bucket, num_buckets = 1; - ot_vector *bucket_list = &peer_list->peers; - size_t compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size); - size_t result = compare_size * peer_list->peer_count; - char *r_end = reply + result; - - if (OT_PEERLIST_HASBUCKETS(peer_list)) { - num_buckets = bucket_list->size; - bucket_list = (ot_vector *)bucket_list->data; - } - - for (bucket = 0; bucket < num_buckets; ++bucket) { - ot_peer *peers = bucket_list[bucket].data; - size_t peer_count = bucket_list[bucket].size; - while (peer_count--) { - if (OT_PEERFLAG_D(peers, peer_size) & PEER_FLAG_SEEDING) { - r_end -= compare_size; - memcpy(r_end, peers, compare_size); - } else { - memcpy(reply, peers, compare_size); - reply += compare_size; - } - peers += peer_size; - } - } - return result; -} - -static size_t return_peers_selection(struct ot_workstruct *ws, ot_peerlist *peer_list, size_t peer_size, size_t amount, char *reply) { - size_t bucket_offset, bucket_index = 0, num_buckets = 1; - ot_vector *bucket_list = &peer_list->peers; - unsigned int shifted_pc = peer_list->peer_count; - unsigned int shifted_step = 0; - unsigned int shift = 0; - size_t compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size); - size_t result = compare_size * amount; - char *r_end = reply + result; - - if (OT_PEERLIST_HASBUCKETS(peer_list)) { - num_buckets = bucket_list->size; - bucket_list = (ot_vector *)bucket_list->data; - } - - /* Make fixpoint arithmetic as exact as possible */ -#define MAXPRECBIT (1 << (8 * sizeof(int) - 3)) - while (!(shifted_pc & MAXPRECBIT)) { - shifted_pc <<= 1; - shift++; - } - shifted_step = shifted_pc / amount; -#undef MAXPRECBIT - - /* Initialize somewhere in the middle of peers so that - fixpoint's aliasing doesn't alway miss the same peers */ - bucket_offset = nrand48(ws->rand48_state) % peer_list->peer_count; - - while (amount--) { - ot_peer *peer; - - /* This is the aliased, non shifted range, next value may fall into */ - unsigned int diff = (((amount + 1) * shifted_step) >> shift) - ((amount * shifted_step) >> shift); - bucket_offset += 1 + nrand48(ws->rand48_state) % diff; - - while (bucket_offset >= bucket_list[bucket_index].size) { - bucket_offset -= bucket_list[bucket_index].size; - bucket_index = (bucket_index + 1) % num_buckets; - } - peer = (ot_peer*)bucket_list[bucket_index].data + peer_size * bucket_offset; - if (OT_PEERFLAG_D(peer, peer_size) & PEER_FLAG_SEEDING) { - r_end -= compare_size; - memcpy(r_end, peer, compare_size); - } else { - memcpy(reply, peer, compare_size); - reply += compare_size; - } - } - return result; -} - -static size_t return_peers_for_torrent_udp(struct ot_workstruct *ws, ot_torrent *torrent, size_t amount, char *reply) { - char *r = reply; - size_t peer_size = peer_size_from_peer6(&ws->peer); - ot_peerlist *peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; - size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; - size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; - - if (amount > peer_list->peer_count) - amount = peer_list->peer_count; - - *(uint32_t *)(r + 0) = htonl(OT_CLIENT_REQUEST_INTERVAL_RANDOM); - *(uint32_t *)(r + 4) = htonl(peer_count - seed_count); - *(uint32_t *)(r + 8) = htonl(seed_count); - r += 12; - - if (amount) { - if (amount == peer_list->peer_count) - r += return_peers_all(peer_list, peer_size, r); - else - r += return_peers_selection(ws, peer_list, peer_size, amount, r); - } - return r - reply; -} - -static size_t return_peers_for_torrent_tcp(struct ot_workstruct *ws, ot_torrent *torrent, size_t amount, char *reply) { - char *r = reply; - const char *r_end = ws->outbuf + G_OUTBUF_SIZE; - int erval = OT_CLIENT_REQUEST_INTERVAL_RANDOM; - size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; - size_t down_count = torrent->peer_list6->down_count + torrent->peer_list4->down_count; - size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - seed_count; - - /* Simple case: amount of peers in both lists is less than requested, here we return all results */ - size_t amount_v4 = torrent->peer_list4->peer_count; - size_t amount_v6 = torrent->peer_list6->peer_count; - - /* Complex case: both lists have more than enough entries and we need to split between v4 and v6 clients */ - if (amount_v4 + amount_v6 > amount) { - size_t amount_left, percent_v6 = 0, percent_v4 = 0, left_v6, left_v4; - const size_t SCALE = 1024; - - /* If possible, fill at least a quarter of peer from each family */ - if (amount / 4 <= amount_v4) - amount_v4 = amount / 4; - if (amount / 4 <= amount_v6) - amount_v6 = amount / 4; - - /* Fill the rest according to which family's pool provides more peers */ - amount_left = amount - (amount_v4 + amount_v6); - - left_v4 = torrent->peer_list4->peer_count - amount_v4; - left_v6 = torrent->peer_list6->peer_count - amount_v6; - - if (left_v4 + left_v6) { - percent_v4 = (SCALE * left_v4) / (left_v4 + left_v6); - percent_v6 = (SCALE * left_v6) / (left_v4 + left_v6); - } - - amount_v4 += (amount_left * percent_v4) / SCALE; - amount_v6 += (amount_left * percent_v6) / SCALE; - - /* Integer division rounding can leave out a peer */ - if (amount_v4 + amount_v6 < amount && amount_v6 < torrent->peer_list6->peer_count) - ++amount_v6; - if (amount_v4 + amount_v6 < amount && amount_v4 < torrent->peer_list4->peer_count) - ++amount_v4; - } - - fmt_append(&r, r_end, "d8:completei%zde10:downloadedi%zde10:incompletei%zde8:intervali%ie12:min intervali%ie", seed_count, down_count, peer_count, erval, - erval / 2); - - if (amount_v4) { - fmt_append(&r, r_end, PEERS_BENCODED4 "%zd:", OT_PEER_COMPARE_SIZE4 * amount_v4); - if (amount_v4 == torrent->peer_list4->peer_count) - r += return_peers_all(torrent->peer_list4, OT_PEER_SIZE4, r); - else - r += return_peers_selection(ws, torrent->peer_list4, OT_PEER_SIZE4, amount_v4, r); - } - - if (amount_v6) { - fmt_append(&r, r_end, PEERS_BENCODED6 "%zd:", OT_PEER_COMPARE_SIZE6 * amount_v6); - if (amount_v6 == torrent->peer_list6->peer_count) - r += return_peers_all(torrent->peer_list6, OT_PEER_SIZE6, r); - else - r += return_peers_selection(ws, torrent->peer_list6, OT_PEER_SIZE6, amount_v6, r); - } - - fmt_append(&r, r_end, "e"); - - return r - reply; -} - -/* Compiles a list of random peers for a torrent - * Reply must have enough space to hold: - * 92 + 6 * amount bytes for TCP/IPv4 - * 92 + 18 * amount bytes for TCP/IPv6 - * 12 + 6 * amount bytes for UDP/IPv4 - * 12 + 18 * amount bytes for UDP/IPv6 - * Does not yet check not to return self - */ -size_t return_peers_for_torrent(struct ot_workstruct *ws, ot_torrent *torrent, size_t amount, char *reply, PROTO_FLAG proto) { - return proto == FLAG_TCP ? return_peers_for_torrent_tcp(ws, torrent, amount, reply) : return_peers_for_torrent_udp(ws, torrent, amount, reply); -} - -/* Fetches scrape info for a specific torrent */ -size_t return_udp_scrape_for_torrent(ot_hash const hash, char *reply) { - int exactmatch, delta_torrentcount = 0; - ot_vector *torrents_list = mutex_bucket_lock_by_hash(hash); - ot_torrent *torrent = vector_binary_search(hash, torrents_list->data, torrents_list->size, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); - - if (!exactmatch) { - memset(reply, 0, 12); - } else { - uint32_t *r = (uint32_t *)reply; - - if (clean_single_torrent(torrent)) { - vector_remove_torrent(torrents_list, torrent); - memset(reply, 0, 12); - delta_torrentcount = -1; - } else { - r[0] = htonl(torrent->peer_list6->seed_count + torrent->peer_list4->seed_count); - r[1] = htonl(torrent->peer_list6->down_count + torrent->peer_list4->down_count); - r[2] = htonl(torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - torrent->peer_list6->seed_count - torrent->peer_list4->seed_count); - } - } - mutex_bucket_unlock_by_hash(hash, delta_torrentcount); - return 12; -} - -/* Fetches scrape info for a specific torrent */ -void return_tcp_scrape_for_torrent(ot_hash const *hash_list, size_t amount, char **reply, const char *reply_end) { - int exactmatch; - size_t i; - - fmt_append(reply, reply_end, "d5:filesd"); - - for (i = 0; i < amount; ++i) { - int delta_torrentcount = 0; - ot_hash const *hash = hash_list + i; - ot_vector *torrents_list = mutex_bucket_lock_by_hash(*hash); - ot_torrent *torrent = vector_binary_search(hash, torrents_list->data, torrents_list->size, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); - - if (exactmatch) { - if (clean_single_torrent(torrent)) { - vector_remove_torrent(torrents_list, torrent); - delta_torrentcount = -1; - } else { - fmt_append(reply, reply_end, "20:"); - fmt_bytes(reply, reply_end, hash, sizeof(ot_hash)); - fmt_append(reply, reply_end, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", - torrent->peer_list6->seed_count + torrent->peer_list4->seed_count, torrent->peer_list6->down_count + torrent->peer_list4->down_count, - torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - torrent->peer_list6->seed_count - torrent->peer_list4->seed_count); - } - } - mutex_bucket_unlock_by_hash(*hash, delta_torrentcount); - } - - fmt_append(reply, reply_end, "ee"); -} - -static ot_peerlist dummy_list; -size_t remove_peer_from_torrent(PROTO_FLAG proto, struct ot_workstruct *ws) { - int exactmatch; - ot_vector *torrents_list = mutex_bucket_lock_by_hash(*ws->hash); - ot_torrent *torrent = vector_binary_search(ws->hash, torrents_list->data, torrents_list->size, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); - ot_peerlist *peer_list = &dummy_list; - size_t peer_size; /* initialized in next line */ - ot_peer const *peer_src = peer_from_peer6(&ws->peer, &peer_size); - size_t peer_count = 0, seed_count = 0; - -#ifdef WANT_SYNC_LIVE - if (proto != FLAG_MCA) { - OT_PEERFLAG(ws->peer) |= PEER_FLAG_STOPPED; - livesync_tell(ws); - } -#endif - - if (exactmatch) { - peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; - switch (vector_remove_peer(&peer_list->peers, peer_src, peer_size)) { - case 2: - peer_list->seed_count--; /* Intentional fallthrough */ - case 1: - peer_list->peer_count--; /* Intentional fallthrough */ - default: - break; - } - - peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; - seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; - } - - if (proto == FLAG_TCP) { - int erval = OT_CLIENT_REQUEST_INTERVAL_RANDOM; - char *r = ws->reply; - - fmt_append(&r, ws->outbuf + G_OUTBUF_SIZE, "d8:completei%zde10:incompletei%zde8:intervali%ie12:min intervali%ie%s0:e", seed_count, peer_count - seed_count, - erval, erval / 2, peer_size == OT_PEER_SIZE6 ? PEERS_BENCODED6 : PEERS_BENCODED4); - ws->reply_size = r - ws->reply; - } - - /* Handle UDP reply */ - if (proto == FLAG_UDP) { - ((uint32_t *)ws->reply)[2] = htonl(OT_CLIENT_REQUEST_INTERVAL_RANDOM); - ((uint32_t *)ws->reply)[3] = htonl(peer_count - seed_count); - ((uint32_t *)ws->reply)[4] = htonl(seed_count); - ws->reply_size = 20; - } - - mutex_bucket_unlock_by_hash(*ws->hash, 0); - return ws->reply_size; -} - -void iterate_all_torrents(int (*for_each)(ot_torrent *torrent, uintptr_t data), uintptr_t data) { - size_t bucket, j; - - for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { - ot_vector *torrents_list = mutex_bucket_lock(bucket); - ot_torrent *torrents = (ot_torrent *)(torrents_list->data); - - for (j = 0; j < torrents_list->size; ++j) - if (for_each(torrents + j, data)) - break; - - mutex_bucket_unlock(bucket, 0); - if (!g_opentracker_running) - return; - } -} - -ot_peer *peer_from_peer6(ot_peer6 *peer, size_t *peer_size) { - ot_ip6 *ip = (ot_ip6 *)peer; - if (!ip_is_v4mapped(ip)) { - *peer_size = OT_PEER_SIZE6; - return (ot_peer *)peer; - } - *peer_size = OT_PEER_SIZE4; - return (ot_peer *)(((uint8_t *)peer) + 12); -} - -size_t peer_size_from_peer6(ot_peer6 *peer) { - ot_ip6 *ip = (ot_ip6 *)peer; - if (!ip_is_v4mapped(ip)) - return OT_PEER_SIZE6; - return OT_PEER_SIZE4; -} - -#ifdef _DEBUG_RANDOMTORRENTS -void trackerlogic_add_random_torrents(size_t amount) { - struct ot_workstruct ws; - memset(&ws, 0, sizeof(ws)); - - ws.inbuf = malloc(G_INBUF_SIZE); - ws.outbuf = malloc(G_OUTBUF_SIZE); - ws.reply = ws.outbuf; - ws.hash = (ot_hash *)ws.inbuf; - - while (amount--) { - random_bytes(ws.hash, sizeof(ot_hash)); - random_bytes(&ws.peer, sizeof(ws.peer)); - - OT_PEERFLAG(ws.peer) &= PEER_FLAG_SEEDING | PEER_FLAG_COMPLETED | PEER_FLAG_STOPPED; - - add_peer_to_torrent_and_return_peers(FLAG_TCP, &ws, 1); - } - - free(ws.inbuf); - free(ws.outbuf); -} -#endif - -void exerr(const char *message) { - fprintf(stderr, "%s\n", message); - exit(111); -} - -void trackerlogic_init(void) { - g_tracker_id = random_uint32(); - - if (!g_stats_path) - g_stats_path = "stats"; - g_stats_path_len = strlen(g_stats_path); - - /* Initialise background worker threads */ - mutex_init(); - workqueue_init(); - clean_init(); - fullscrape_init(); - accesslist_init(); - livesync_init(); - stats_init(); -} - -int workstruct_init(struct ot_workstruct *ws) { - memset(ws, 0, sizeof(*ws)); - ws->inbuf = malloc(G_INBUF_SIZE); - ws->outbuf = malloc(G_OUTBUF_SIZE); -#ifdef _DEBUG_HTTPERROR - ws->debugbuf = malloc(G_DEBUGBUF_SIZE); -#endif - if (!ws->inbuf || !ws->outbuf) - return -1; - - /* Picking which peers to hand out starts here, so two workers seeded alike - would hand out alike */ - random_bytes(ws->rand48_state, sizeof(ws->rand48_state)); - - stats_counters_register(ws); - return 0; -} - -void workstruct_deinit(struct ot_workstruct *ws) { - /* The tally goes to the global before this struct goes out of scope */ - stats_counters_retire(ws); - - free(ws->inbuf); - free(ws->outbuf); -#ifdef _DEBUG_HTTPERROR - free(ws->debugbuf); -#endif -} - -void trackerlogic_deinit(void) { - int delta_torrentcount = 0; - size_t bucket, j; - - /* Stop the background workers first */ - stats_deinit(); - livesync_deinit(); - accesslist_deinit(); - fullscrape_deinit(); - clean_deinit(); - - /* Free all torrents... */ - for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { - ot_vector *torrents_list = mutex_bucket_lock(bucket); - if (torrents_list->size) { - for (j = 0; j < torrents_list->size; ++j) { - ot_torrent *torrent = ((ot_torrent *)(torrents_list->data)) + j; - free_peerlist(torrent->peer_list6); - free_peerlist(torrent->peer_list4); - delta_torrentcount -= 1; - } - free(torrents_list->data); - } - mutex_bucket_unlock(bucket, delta_torrentcount); - } - - /* Release mutexes */ - workqueue_deinit(); - mutex_deinit(); -} diff --git a/trackerlogic.h b/trackerlogic.h deleted file mode 100644 index 7292d36..0000000 --- a/trackerlogic.h +++ /dev/null @@ -1,283 +0,0 @@ -/* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. */ - -#ifndef OT_TRACKERLOGIC_H__ -#define OT_TRACKERLOGIC_H__ - -#include -#include -#include -#include -#include -#include -#include - -typedef uint8_t ot_hash[20]; -typedef time_t ot_time; -typedef char ot_ip6[16]; -typedef struct { - ot_ip6 address; - int bits; -} ot_net; -/* List of peers should fit in a single UDP packet (around 1200 bytes) */ -#define OT_MAX_PEERS_UDP6 66 -#define OT_MAX_PEERS_UDP4 200 - -#define OT_IP_SIZE6 16 -#define OT_IP_SIZE4 4 -#define OT_PORT_SIZE 2 -#define OT_FLAG_SIZE 1 -#define OT_TIME_SIZE 1 - -/* Some tracker behaviour tunable */ -#ifndef OT_CLIENT_TIMEOUT -#define OT_CLIENT_TIMEOUT 30 -#endif -#ifndef OT_CLIENT_TIMEOUT_CHECKINTERVAL -#define OT_CLIENT_TIMEOUT_CHECKINTERVAL 10 -#endif -#define OT_CLIENT_TIMEOUT_SEND (60 * 15) -#define OT_CLIENT_REQUEST_INTERVAL (60 * 30) -#define OT_CLIENT_REQUEST_VARIATION (60 * 6) - -#define OT_TORRENT_TIMEOUT_HOURS 24 -#define OT_TORRENT_TIMEOUT (60 * OT_TORRENT_TIMEOUT_HOURS) - -#define OT_CLIENT_REQUEST_INTERVAL_RANDOM \ - (OT_CLIENT_REQUEST_INTERVAL - OT_CLIENT_REQUEST_VARIATION / 2 + (int)(nrand48(ws->rand48_state) % OT_CLIENT_REQUEST_VARIATION)) - -/* If WANT_MODEST_FULLSCRAPES is on, ip addresses may not - fullscrape more frequently than this amount in seconds */ -#define OT_MODEST_PEER_TIMEOUT (60 * 5) - -/* If peers come back before 10 minutes, don't live sync them */ -#define OT_CLIENT_SYNC_RENEW_BOUNDARY 10 - -/* Number of tracker admin ip addresses allowed */ -#define OT_ADMINIP_MAX 64 -#define OT_MAX_THREADS 64 - -/* Number of minutes after announce before peer is removed */ -#define OT_PEER_TIMEOUT 45 - -/* We maintain a list of 1024 pointers to sorted list of ot_torrent structs - Sort key is, of course, its hash */ -/* Set from the Makefile, 8 to 16. More buckets means shorter vectors and a - cheaper insert, which only pays for a tracker carrying millions of - torrents; see DESIGN. */ -#ifndef OT_BUCKET_COUNT_BITS -#define OT_BUCKET_COUNT_BITS 10 -#endif - -#define OT_BUCKET_COUNT (1 << OT_BUCKET_COUNT_BITS) -#define OT_BUCKET_COUNT_SHIFT (32 - OT_BUCKET_COUNT_BITS) - -/* if _DEBUG_RANDOMTORRENTS is set, this is the amount of torrents to create - on startup */ -#define RANDOMTORRENTS (1024 * 1024 * 1) - -#if defined(CLOCK_REALTIME_COARSE) -#define OT_CLOCK_CHEAP CLOCK_REALTIME_COARSE /* Linux */ -#elif defined(CLOCK_REALTIME_FAST) -#define OT_CLOCK_CHEAP CLOCK_REALTIME_FAST /* the BSDs */ -#else -#define OT_CLOCK_CHEAP CLOCK_REALTIME /* macOS and whatever else */ -#endif - -static inline ot_time time_now_seconds(void) { - struct timespec now; - clock_gettime(OT_CLOCK_CHEAP, &now); - return (ot_time)now.tv_sec; -} - -static inline ot_time time_now_minutes(void) { return time_now_seconds() / 60; } - -/* For the deadlines the tracker sets on itself, which read better as the - span they allow than as an addition to the current second */ -static inline ot_time time_future_seconds(int span) { return time_now_seconds() + span; } -/* Cleared from the interrupt handler and read by every worker */ -extern _Atomic int g_opentracker_running; - -/* Sent to a thread parked in a blocking read */ -#define OT_SIGNAL_WAKE SIGUSR1 - - -extern uint32_t g_tracker_id; -typedef enum { FLAG_TCP, FLAG_UDP, FLAG_MCA, FLAG_SELFPIPE } PROTO_FLAG; - -#define OT_PEER_COMPARE_SIZE6 ((OT_IP_SIZE6) + (OT_PORT_SIZE)) -#define OT_PEER_COMPARE_SIZE4 ((OT_IP_SIZE4) + (OT_PORT_SIZE)) -#define OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(PEER_SIZE) ((PEER_SIZE) - (OT_TIME_SIZE) - (OT_FLAG_SIZE)) - -#define OT_PEER_SIZE6 ((OT_TIME_SIZE) + (OT_FLAG_SIZE) + (OT_PEER_COMPARE_SIZE6)) -#define OT_PEER_SIZE4 ((OT_TIME_SIZE) + (OT_FLAG_SIZE) + (OT_PEER_COMPARE_SIZE4)) - -typedef uint8_t ot_peer; /* Generic pointer to a v6 or v4 peer */ -typedef uint8_t ot_peer6[OT_PEER_SIZE6]; -typedef uint8_t ot_peer4[OT_PEER_SIZE4]; -static const uint8_t PEER_FLAG_SEEDING = 0x80; -static const uint8_t PEER_FLAG_COMPLETED = 0x40; -static const uint8_t PEER_FLAG_STOPPED = 0x20; -static const uint8_t PEER_FLAG_FROM_SYNC = 0x10; -static const uint8_t PEER_FLAG_LEECHING = 0x00; - -/* Takes an ot_peer6 and returns the proper pointer to the peer and sets peer_size */ -ot_peer *peer_from_peer6(ot_peer6 *peer, size_t *peer_size); -size_t peer_size_from_peer6(ot_peer6 *peer); - -/* New style */ -#define OT_SETIP(peer, ip) memcpy((uint8_t *)(peer), (ip), OT_IP_SIZE6) -#define OT_SETPORT(peer, port) memcpy(((uint8_t *)(peer)) + (OT_IP_SIZE6), (port), 2) -#define OT_PEERFLAG(peer) (((uint8_t *)(peer))[(OT_IP_SIZE6) + 2]) -#define OT_PEERFLAG_D(peer, peersize) (((uint8_t *)(peer))[(peersize) - 2]) -#define OT_PEERTIME(peer, peersize) (((uint8_t *)(peer))[(peersize) - 1]) - -#define PEERS_BENCODED6 "6:peers6" -#define PEERS_BENCODED4 "5:peers" - -#define OT_HASH_COMPARE_SIZE (sizeof(ot_hash)) - -struct ot_peerlist; -typedef struct ot_peerlist ot_peerlist; -typedef struct { - ot_hash hash; - ot_peerlist *peer_list6; - ot_peerlist *peer_list4; -} ot_torrent; - -#include "ot_vector.h" - -struct ot_peerlist { - ot_time base; - size_t seed_count; - size_t peer_count; - size_t down_count; - /* normal peers vector or - pointer to ot_vector[32] buckets if data != NULL and space == 0 - */ - ot_vector peers; -}; -#define OT_PEERLIST_HASBUCKETS(peer_list) ((peer_list)->peers.size > (peer_list)->peers.space) - -/* The http replies the tracker counts when it has to refuse a request. */ -enum { - CODE_HTTPERROR_302, - CODE_HTTPERROR_400, - CODE_HTTPERROR_400_PARAM, - CODE_HTTPERROR_400_COMPACT, - CODE_HTTPERROR_402_NOTMODEST, - CODE_HTTPERROR_402_PAYMENT_REQUIRED, - CODE_HTTPERROR_403_IP, - CODE_HTTPERROR_404, - CODE_HTTPERROR_500, - - CODE_HTTPERROR_COUNT -}; - -/* One statistic. Used when workers report stats to their workstruct */ -typedef _Atomic unsigned long long ot_counter; - -/* What each thread counts for itself. The two runs at the end are indexed by - the http error code and by the age in minutes of a renewed peer */ -typedef enum { - OT_COUNTER_TCP_CONNECTIONS, - OT_COUNTER_UDP_CONNECTIONS, - OT_COUNTER_TCP_ANNOUNCES, - OT_COUNTER_UDP_ANNOUNCES, - OT_COUNTER_TCP_SCRAPES, - OT_COUNTER_UDP_SCRAPES, - OT_COUNTER_TCP_CONNECTS, - OT_COUNTER_UDP_CONNECTS, - OT_COUNTER_UDP_CONNID_MISSMATCH, - OT_COUNTER_COMPLETED, - OT_COUNTER_FULLSCRAPES, - OT_COUNTER_FULLSCRAPE_REQUESTS, - OT_COUNTER_FULLSCRAPE_BYTES, - OT_COUNTER_SYNCS, - OT_COUNTER_STALLS, - - OT_COUNTER_FAILED, /* CODE_HTTPERROR_COUNT of them */ - OT_COUNTER_RENEWED = OT_COUNTER_FAILED + CODE_HTTPERROR_COUNT, /* OT_PEER_TIMEOUT of them */ - - OT_COUNTER_COUNT = OT_COUNTER_RENEWED + OT_PEER_TIMEOUT -} ot_counter_index; - -struct ot_workstruct { - /* Thread specific, static */ - - /* This thread's own statistics, and the link ot_stats keeps them on */ - ot_counter counters[OT_COUNTER_COUNT]; - struct ot_workstruct *counters_next; - - char *inbuf; -#define G_INBUF_SIZE 8192 - char *outbuf; -#define G_OUTBUF_SIZE 8192 -#ifdef _DEBUG_HTTPERROR - char *debugbuf; -#define G_DEBUGBUF_SIZE 8192 -#endif - - /* The peer currently in the working */ - ot_peer6 peer; /* Can fit v6 and v4 peers */ - - /* Pointers into the request buffer */ - ot_hash *hash; - char *peer_id; - - /* HTTP specific, non static */ - char *request; - ssize_t request_size; - ssize_t header_size; - char *reply; - ssize_t reply_size; - - /* Entropy state for rand48 function so that threads don't need to acquire mutexes for - global random() or arc4random() state, which causes heavy load on linuxes */ - uint16_t rand48_state[3]; - - int keep_alive; -}; - -/* Everything a thread that serves requests needs for itself. Set up and torn - down in one place so that a new kind of worker cannot miss a piece. - Returns -1 if the buffers could not be had. */ -int workstruct_init(struct ot_workstruct *ws); -void workstruct_deinit(struct ot_workstruct *ws); - -/* - Exported functions -*/ - -#ifdef WANT_SYNC_LIVE -#define WANT_SYNC -#endif - -#ifdef WANT_SYNC -#define WANT_SYNC_PARAM(param) , param -#else -#define WANT_SYNC_PARAM(param) -#endif - -void trackerlogic_init(void); -void trackerlogic_deinit(void); -void exerr(const char *message); - -/* add_peer_to_torrent does only release the torrent bucket if from_sync is set, - otherwise it is released in return_peers_for_torrent */ -size_t add_peer_to_torrent_and_return_peers(PROTO_FLAG proto, struct ot_workstruct *ws, size_t amount); -size_t remove_peer_from_torrent(PROTO_FLAG proto, struct ot_workstruct *ws); -void return_tcp_scrape_for_torrent(ot_hash const *hash_list, size_t amount, char **reply, const char *reply_end); -size_t return_udp_scrape_for_torrent(ot_hash const hash, char *reply); -void add_torrent_from_saved_state(ot_hash const hash, ot_time base, size_t down_count); -#ifdef _DEBUG_RANDOMTORRENTS -void trackerlogic_add_random_torrents(size_t amount); -#endif - -/* torrent iterator */ -void iterate_all_torrents(int (*for_each)(ot_torrent *torrent, uintptr_t data), uintptr_t data); - -/* Helper, before it moves to its own object */ -void free_peerlist(ot_peerlist *peer_list); - -#endif -- cgit v1.3