From be825f57597b0e9dcf07d257e93f03e30935f7db Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Thu, 24 Nov 2022 04:20:06 +0100 Subject: Add support for dynamic accesslists --- Makefile | 1 + opentracker.c | 12 ++- opentracker.conf.sample | 37 +++++++ ot_accesslist.c | 273 +++++++++++++++++++++++++++++++++++++++++------- ot_accesslist.h | 8 +- ot_mutex.h | 6 +- ot_stats.h | 6 +- trackerlogic.h | 5 +- 8 files changed, 298 insertions(+), 50 deletions(-) diff --git a/Makefile b/Makefile index 949e63e..e3301a5 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,7 @@ STRIP?=strip #FEATURES+=-DWANT_ACCESSLIST_BLACK #FEATURES+=-DWANT_ACCESSLIST_WHITE +#FEATURES+=-DWANT_DYNAMIC_ACCESSLIST #FEATURES+=-DWANT_SYNC_LIVE #FEATURES+=-DWANT_IP_FROM_QUERY_STRING diff --git a/opentracker.c b/opentracker.c index a5ba7d3..b6d2bb9 100644 --- a/opentracker.c +++ b/opentracker.c @@ -135,8 +135,8 @@ static void help( char *name ) { } #undef HELPLINE -static size_t header_complete( char * request, ssize_t byte_count ) { - int i = 0, state = 0; +static ssize_t header_complete( char * request, ssize_t byte_count ) { + ssize_t i = 0, state = 0; for( i=1; i < byte_count; i+=2 ) if( request[i] <= 13 ) { @@ -184,7 +184,7 @@ static void handle_read( const int64 sock, struct ot_workstruct *ws ) { ws->request_size = byte_count; http_handle_request( sock, ws ); } else - array_catb( &cookie->request, ws->inbuf, byte_count ); + array_catb( &cookie->request, ws->inbuf, (size_t)byte_count ); return; } @@ -463,6 +463,12 @@ int parse_configfile( char * config_filename ) { } else if(!byte_diff(p, 16, "access.blacklist" ) && isspace(p[16])) { set_config_option( &g_accesslist_filename, p+17 ); #endif +#ifdef WANT_DYNAMIC_ACCESSLIST + } else if(!byte_diff(p, 15, "access.fifo_add" ) && isspace(p[15])) { + set_config_option( &g_accesslist_pipe_add, p+16 ); + } else if(!byte_diff(p, 18, "access.fifo_delete" ) && isspace(p[18])) { + set_config_option( &g_accesslist_pipe_delete, p+19 ); +#endif #ifdef WANT_RESTRICT_STATS } else if(!byte_diff(p, 12, "access.stats" ) && isspace(p[12])) { if( !scan_ip6( p+13, tmpip )) goto parse_error; diff --git a/opentracker.conf.sample b/opentracker.conf.sample index db45122..55c2828 100644 --- a/opentracker.conf.sample +++ b/opentracker.conf.sample @@ -44,6 +44,43 @@ # listing, so choose one of those options at compile time. File format # is straight forward: "\n\n..." # +# IIa) You can enable dynamic changesets to accesslists by enabling +# WANT_DYNAMIC_ACCESSLIST. +# +# The suggested way to work with dynamic changeset lists is to keep a +# main accesslist file that is loaded when opentracker (re)starts and +# reloaded infrequently (hourly or daily). +# +# All changes to the accesslist (e.g. from a web frontend) should be +# both appended to or removed from that file and sent to opentracker. By +# keeping dynamic changeset lists, you can avoid reloading huge +# accesslists whenever just a single entry is added or removed. +# +# Any info_hash (format see above) written to the fifo_add file will be +# kept on a dynamic add-changeset, removed from the dynamic +# delete-changeset and treated as if it was in the main accesslist file. +# The semantic of the respective dynamic changeset depends on whether +# WANT_ACCESSLIST_WHITE or WANT_ACCESSLIST_BLACK is enabled. +# +# access.fifo_add /var/run/opentracker/adder.fifo +# +# Any info_hash (format see above) written to the fifo_delete file will +# be kept on a dynamic delete-changeset, removed from the dynamic +# add-changeset and treated as if it was not in the main accesslist +# file. +# +# access.fifo_delete /var/run/opentracker/deleter.fifo +# +# If you reload the accesslist by sending SIGHUP to the tracker process, +# the dynamic lists are flushed, as opentracker assumes thoses lists are +# merged into the main accesslist. +# +# NOTE: While you can have multiple writers sending lines to the fifos, +# any writes larger than PIPE_BUF (see your limits.h, minimally 512 +# bytes but usually 4096) may be interleaved with data sent by other +# writers. This can lead to unparsable lines of info_hashes. +# +# IIb) # If you do not want to grant anyone access to your stats, enable the # WANT_RESTRICT_STATS option in Makefile and bless the ip addresses # allowed to fetch stats here. diff --git a/ot_accesslist.c b/ot_accesslist.c index a3a2049..078cebd 100644 --- a/ot_accesslist.c +++ b/ot_accesslist.c @@ -10,6 +10,11 @@ #include #include #include +#ifdef WANT_DYNAMIC_ACCESSLIST +#include +#include +#include +#endif /* Libowfat */ #include "byte.h" @@ -24,23 +29,80 @@ /* GLOBAL VARIABLES */ #ifdef WANT_ACCESSLIST - char *g_accesslist_filename; +char *g_accesslist_filename = NULL; +#ifdef WANT_DYNAMIC_ACCESSLIST +char *g_accesslist_pipe_add = NULL; +char *g_accesslist_pipe_delete = NULL; +#endif static pthread_mutex_t g_accesslist_mutex; -typedef struct { - ot_hash *list; - size_t size; -} ot_accesslist; -ot_accesslist * g_accesslist = NULL; -ot_accesslist * g_accesslist_old = NULL; +/* Accesslists are lock free linked lists. We can not make them locking, because every announce + would try to acquire the mutex, making it the most contested mutex in the whole of opentracker, + basically creating a central performance choke point. + + The idea is that updating the list heads happens under the g_accesslist_mutex guard and is + done atomically, while consumers might potentially still hold pointers deeper inside the list. + + Consumers (for now only via accesslist_hashisvalid) will always fetch the list head pointer + that is guaranteed to live for at least five minutes. This should be many orders of magnitudes + more than how long it will be needed by the bsearch done on the list. */ +struct ot_accesslist; +typedef struct ot_accesslist ot_accesslist; +struct ot_accesslist { + ot_hash *list; + size_t size; + ot_time base; + ot_accesslist *next; +}; +static ot_accesslist * g_accesslist = NULL; +#ifdef WANT_DYNAMIC_ACCESSLIST +static ot_accesslist * g_accesslist_add = NULL; +static ot_accesslist * g_accesslist_delete = NULL; +#endif +/* Helpers to work on access lists */ static int vector_compare_hash(const void *hash1, const void *hash2 ) { return memcmp( hash1, hash2, OT_HASH_COMPARE_SIZE ); } +static ot_accesslist * accesslist_free(ot_accesslist *accesslist) { + while (accesslist) { + ot_accesslist * this_accesslist = accesslist; + accesslist = this_accesslist->next; + free(this_accesslist->list); + free(this_accesslist); + } + return NULL; +} + +static ot_accesslist * accesslist_make(ot_accesslist *next, size_t size) { + ot_accesslist * accesslist_new = malloc(sizeof(ot_accesslist)); + if (accesslist_new) { + accesslist_new->list = size ? malloc(sizeof(ot_hash) * size) : NULL; + accesslist_new->size = size; + accesslist_new->base = g_now_minutes; + accesslist_new->next = next; + if (size && !accesslist_new->list) { + free(accesslist_new); + accesslist_new = NULL; + } + } + return accesslist_new; +} + +/* This must be called with g_accesslist_mutex held. + This will never delete head, because that might still be in use. */ +static void accesslist_clean(ot_accesslist *accesslist) { + while (accesslist && accesslist->next) { + if (accesslist->next->base + 5 < g_now_minutes) + accesslist->next = accesslist_free(accesslist->next); + accesslist = accesslist->next; + } +} + /* Read initial access list */ static void accesslist_readfile( void ) { - ot_accesslist * accesslist_new = malloc(sizeof(ot_accesslist)); + ot_accesslist * accesslist_new; ot_hash *info_hash; const char *map, *map_end, *read_offs; size_t maplen; @@ -54,14 +116,13 @@ static void accesslist_readfile( void ) { /* You need at least 41 bytes to pass an info_hash, make enough room for the maximum amount of them */ - accesslist_new->size = 0; - info_hash = accesslist_new->list = malloc( ( maplen / 41 ) * 20 ); - if( !accesslist_new->list ) { + accesslist_new = accesslist_make(g_accesslist, maplen / 41); + if( !accesslist_new ) { fprintf( stderr, "Warning: Not enough memory to allocate %zd bytes for accesslist buffer. May succeed later.\n", ( maplen / 41 ) * 20 ); mmap_unmap( map, maplen); - free(accesslist_new); return; } + info_hash = accesslist_new->list; /* No use to scan if there's not enough room for another full info_hash */ map_end = map + maplen - 40; @@ -71,18 +132,18 @@ static void accesslist_readfile( void ) { while( read_offs <= map_end ) { int i; for( i=0; i<(int)sizeof(ot_hash); ++i ) { - int eger1 = scan_fromhex( read_offs[ 2*i ] ); - int eger2 = scan_fromhex( read_offs[ 1 + 2*i ] ); + int eger1 = scan_fromhex( (unsigned char)read_offs[ 2*i ] ); + int eger2 = scan_fromhex( (unsigned char)read_offs[ 1 + 2*i ] ); if( eger1 < 0 || eger2 < 0 ) break; - (*info_hash)[i] = eger1 * 16 + eger2; + (*info_hash)[i] = (uint8_t)(eger1 * 16 + eger2); } if( i == sizeof(ot_hash) ) { read_offs += 40; /* Append accesslist to accesslist vector */ - if( read_offs == map_end || scan_fromhex( *read_offs ) < 0 ) + if( read_offs == map_end || scan_fromhex( (unsigned char)*read_offs ) < 0 ) ++info_hash; } @@ -100,14 +161,19 @@ static void accesslist_readfile( void ) { /* Now exchange the accesslist vector in the least race condition prone way */ pthread_mutex_lock(&g_accesslist_mutex); + accesslist_new->next = g_accesslist; + g_accesslist = accesslist_new; /* Only now set a new list */ + +#ifdef WANT_DYNAMIC_ACCESSLIST + /* If we have dynamic accesslists, reloading a new one will always void the add/delete lists. + Insert empty ones at the list head */ + if (g_accesslist_add && (accesslist_new = accesslist_make(g_accesslist_add, 0)) != NULL) + g_accesslist_add = accesslist_new; + if (g_accesslist_delete && (accesslist_new = accesslist_make(g_accesslist_delete, 0)) != NULL) + g_accesslist_delete = accesslist_new; +#endif - if (g_accesslist_old) { - free(g_accesslist_old->list); - free(g_accesslist_old); - } - - g_accesslist_old = g_accesslist; /* Keep a copy for later free */ - g_accesslist = accesslist_new; /* Only now set a new list */ + accesslist_clean(g_accesslist); pthread_mutex_unlock(&g_accesslist_mutex); } @@ -115,11 +181,25 @@ static void accesslist_readfile( void ) { int accesslist_hashisvalid( ot_hash hash ) { /* Get working copy of current access list */ ot_accesslist * accesslist = g_accesslist; - +#ifdef WANT_DYNAMIC_ACCESSLIST + ot_accesslist * accesslist_add, * accesslist_delete; +#endif void * exactmatch = NULL; if (accesslist) - exactmatch = bsearch( hash, accesslist->list, accesslist->size, OT_HASH_COMPARE_SIZE, vector_compare_hash ); + exactmatch = bsearch( hash, accesslist->list, accesslist->size, OT_HASH_COMPARE_SIZE, vector_compare_hash ); + +#ifdef WANT_DYNAMIC_ACCESSLIST + /* If we had no match on the main list, scan the list of dynamically added hashes */ + accesslist_add = g_accesslist_add; + if ((exactmatch == NULL) && accesslist_add) + exactmatch = bsearch( hash, accesslist_add->list, accesslist_add->size, OT_HASH_COMPARE_SIZE, vector_compare_hash ); + + /* If we found a matching hash on the main list, scan the list of dynamically deleted hashes */ + accesslist_delete = g_accesslist_delete; + if ((exactmatch != NULL) && accesslist_delete && bsearch( hash, accesslist_add->list, accesslist_add->size, OT_HASH_COMPARE_SIZE, vector_compare_hash )) + exactmatch = NULL; +#endif #ifdef WANT_ACCESSLIST_BLACK return exactmatch == NULL; @@ -138,6 +218,8 @@ static void * accesslist_worker( void * args ) { (void)args; while( 1 ) { + if (!g_opentracker_running) + return NULL; /* Initial attempt to read accesslist */ accesslist_readfile( ); @@ -148,27 +230,144 @@ static void * accesslist_worker( void * args ) { return NULL; } +#ifdef WANT_DYNAMIC_ACCESSLIST +static pthread_t thread_adder_id, thread_deleter_id; +static void * accesslist_adddel_worker(char * fifoname, ot_accesslist ** adding_to, ot_accesslist ** removing_from) { + struct stat st; + + if (!stat(fifoname, &st)) { + if (!S_ISFIFO(st.st_mode)) { + fprintf(stderr, "Error when starting dynamic accesslists: Found Non-FIFO file at %s.\nPlease remove it and restart opentracker.\n", fifoname); + return NULL; + } + } else { + int error = mkfifo(fifoname, 0755); + if (error && error != EEXIST) { + fprintf(stderr, "Error when starting dynamic accesslists: Couldn't create FIFO at %s, error: %s\n", fifoname, strerror(errno)); + return NULL; + } + } + + while (g_opentracker_running) { + FILE * fifo = fopen(fifoname, "r"); + char *line = NULL; + size_t linecap = 0; + ssize_t linelen; + + if (!fifo) { + fprintf(stderr, "Error when reading dynamic accesslists: Couldn't open FIFO at %s, error: %s\n", fifoname, strerror(errno)); + return NULL; + } + + while ((linelen = getline(&line, &linecap, fifo)) > 0) { + ot_hash info_hash; + int i; + + printf("Got line %*s", (int)linelen, line); + /* We do ignore anything that is not of the form "^[:xdigit:]{40}[^:xdigit:].*" + If there's not enough characters for an info_hash in the line, skip it. */ + if (linelen < 41) + continue; + + for( i=0; i<(int)sizeof(ot_hash); ++i ) { + int eger1 = scan_fromhex( (unsigned char)line[ 2*i ] ); + int eger2 = scan_fromhex( (unsigned char)line[ 1 + 2*i ] ); + if( eger1 < 0 || eger2 < 0 ) + break; + ((uint8_t*)info_hash)[i] = (uint8_t)(eger1 * 16 + eger2); + } +printf("parsed info_hash %20s\n", info_hash); + if( i != sizeof(ot_hash) ) + continue; + + /* From now on we modify g_accesslist_add and g_accesslist_delete, so prevent the + other worker threads from doing the same */ + pthread_mutex_lock(&g_accesslist_mutex); + + /* If the info hash is in the removing_from list, create a new head without that entry */ + if (*removing_from && (*removing_from)->list) { + ot_hash * exactmatch = bsearch( info_hash, (*removing_from)->list, (*removing_from)->size, OT_HASH_COMPARE_SIZE, vector_compare_hash ); + if (exactmatch) { + ptrdiff_t off = exactmatch - (*removing_from)->list; + ot_accesslist * accesslist_new = accesslist_make(*removing_from, (*removing_from)->size - 1); + if (accesslist_new) { + memcpy(accesslist_new->list, (*removing_from)->list, sizeof(ot_hash) * off); + memcpy(accesslist_new->list + off, (*removing_from)->list + off + 1, (*removing_from)->size - off - 1); + *removing_from = accesslist_new; + } + } + } + accesslist_clean(*removing_from); + + /* Simple case: there's no adding_to list yet, create one with one member */ + if (!*adding_to) { + *adding_to = accesslist_make(NULL, 1); + if (*adding_to) + memcpy((*adding_to)->list, info_hash, sizeof(ot_hash)); + } else { + int exactmatch = 0; + ot_hash * insert_point = binary_search( info_hash, (*adding_to)->list, (*adding_to)->size, OT_HASH_COMPARE_SIZE, sizeof(ot_hash), &exactmatch ); + + /* Only if the info hash is not in the adding_to list, create a new head with that entry */ + if (!exactmatch) { + ot_accesslist * accesslist_new = accesslist_make(*adding_to, (*adding_to)->size + 1); + ptrdiff_t off = insert_point - (*adding_to)->list; + if (accesslist_new) { + memcpy(accesslist_new->list, (*adding_to)->list, sizeof(ot_hash) * off); + memcpy(accesslist_new->list + off, info_hash, sizeof(info_hash)); + memcpy(accesslist_new->list + off + 1, (*adding_to)->list + off, (*adding_to)->size - off); + *adding_to = accesslist_new; + } + } + } + accesslist_clean(*adding_to); + + pthread_mutex_unlock(&g_accesslist_mutex); + } + + fclose(fifo); + } + return NULL; +} + +static void * accesslist_adder_worker( void * args ) { + (void)args; + return accesslist_adddel_worker(g_accesslist_pipe_add, &g_accesslist_add, &g_accesslist_delete); +} +static void * accesslist_deleter_worker( void * args ) { + (void)args; + return accesslist_adddel_worker(g_accesslist_pipe_delete, &g_accesslist_delete, &g_accesslist_add); +} +#endif + static pthread_t thread_id; void accesslist_init( ) { pthread_mutex_init(&g_accesslist_mutex, NULL); pthread_create( &thread_id, NULL, accesslist_worker, NULL ); +#ifdef WANT_DYNAMIC_ACCESSLIST + if (g_accesslist_pipe_add) + pthread_create( &thread_adder_id, NULL, accesslist_adder_worker, NULL ); + if (g_accesslist_pipe_delete) + pthread_create( &thread_deleter_id, NULL, accesslist_deleter_worker, NULL ); +#endif } void accesslist_deinit( void ) { - pthread_cancel( thread_id ); - pthread_mutex_destroy(&g_accesslist_mutex); + /* Wake up sleeping worker */ + pthread_kill(thread_id, SIGHUP); - if (g_accesslist_old) { - free(g_accesslist_old->list); - free(g_accesslist_old); - g_accesslist_old = 0; - } + pthread_mutex_lock(&g_accesslist_mutex); - if (g_accesslist) { - free(g_accesslist->list); - free(g_accesslist); - g_accesslist = 0; - } + g_accesslist = accesslist_free(g_accesslist); + +#ifdef WANT_DYNAMIC_ACCESSLIST + g_accesslist_add = accesslist_free(g_accesslist_add); + g_accesslist_delete = accesslist_free(g_accesslist_delete); +#endif + + pthread_mutex_unlock(&g_accesslist_mutex); + pthread_cancel( thread_id ); + pthread_mutex_destroy(&g_accesslist_mutex); } #endif diff --git a/ot_accesslist.h b/ot_accesslist.h index b38b91a..e86969b 100644 --- a/ot_accesslist.h +++ b/ot_accesslist.h @@ -12,11 +12,15 @@ #if defined ( WANT_ACCESSLIST_BLACK ) || defined (WANT_ACCESSLIST_WHITE ) #define WANT_ACCESSLIST -void accesslist_init( ); -void accesslist_deinit( ); +void accesslist_init( void ); +void accesslist_deinit( void ); int accesslist_hashisvalid( ot_hash hash ); extern char *g_accesslist_filename; +#ifdef WANT_DYNAMIC_ACCESSLIST +extern char *g_accesslist_pipe_add; +extern char *g_accesslist_pipe_delete; +#endif #else #define accesslist_init( accesslist_filename ) diff --git a/ot_mutex.h b/ot_mutex.h index bd07009..93c1ecf 100644 --- a/ot_mutex.h +++ b/ot_mutex.h @@ -8,8 +8,8 @@ #include -void mutex_init( ); -void mutex_deinit( ); +void mutex_init( void ); +void mutex_deinit( void ); ot_vector *mutex_bucket_lock( int bucket ); ot_vector *mutex_bucket_lock_by_hash( ot_hash hash ); @@ -17,7 +17,7 @@ ot_vector *mutex_bucket_lock_by_hash( ot_hash hash ); void mutex_bucket_unlock( int bucket, int delta_torrentcount ); void mutex_bucket_unlock_by_hash( ot_hash hash, int delta_torrentcount ); -size_t mutex_get_torrent_count(); +size_t mutex_get_torrent_count(void); typedef enum { TASK_STATS_CONNS = 0x0001, diff --git a/ot_stats.h b/ot_stats.h index 6a2515b..ed60f68 100644 --- a/ot_stats.h +++ b/ot_stats.h @@ -40,11 +40,11 @@ enum { void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uintptr_t event_data ); void stats_deliver( int64 sock, int tasktype ); -void stats_cleanup(); +void stats_cleanup( void ); size_t return_stats_for_tracker( char *reply, int mode, int format ); size_t stats_return_tracker_version( char *reply ); -void stats_init( ); -void stats_deinit( ); +void stats_init( void ); +void stats_deinit( void ); extern const char *g_version_rijndael_c; extern const char *g_version_livesync_c; diff --git a/trackerlogic.h b/trackerlogic.h index 87b9138..ef59179 100644 --- a/trackerlogic.h +++ b/trackerlogic.h @@ -133,7 +133,6 @@ struct ot_workstruct { char *peer_id; /* HTTP specific, non static */ - int keep_alive; char *request; ssize_t request_size; ssize_t header_size; @@ -143,6 +142,8 @@ struct ot_workstruct { /* Entropy state for rand48 function so that threads don't need to acquire mutexes for global random() or arc4random() state, which causes heavy load on linuxes */ uint16_t rand48_state[3]; + + int keep_alive; }; /* @@ -163,7 +164,7 @@ struct ot_workstruct { #error Live logging networks disabled at the moment. #endif -void trackerlogic_init( ); +void trackerlogic_init( void ); void trackerlogic_deinit( void ); void exerr( char * message ); -- cgit v1.2.3