Add a stat option the count bucket stalls

Add an iterator for all torrents. It's slower but for stats it's okay.
Move some stats collection stuff to the new iterator. More to come.
Start a "report all stats" page.
Start fixing the code to identify "busy" networks.
Add the concept of hosts allowed to proxy.
Add a parser for the X-Forwarded-For: HTTP header.
Clean up HTTP Header handling code. (Remove some left overs of now vanished sync code).

9 files changed, 252 insertions, 177 deletions

diff --git a/opentracker.c b/opentracker.c
index b839efc..c67e331 100644
--- a/opentracker.c
+++ b/opentracker.c
@@ -135,12 +135,10 @@ static ssize_t handle_read( const int64 sock, struct ot_workstruct *ws ) {
 
   array_catb( &cookie->data.request, ws->inbuf, byte_count );
 
-  if( array_failed( &cookie->data.request ) )
+  if( array_failed( &cookie->data.request ) ||
+      array_bytes( &cookie->data.request ) > 8192 )
     return http_issue_error( sock, ws, CODE_HTTPERROR_500 );
 
-  if( array_bytes( &cookie->data.request ) > 8192 )
-     return http_issue_error( sock, ws, CODE_HTTPERROR_500 );
-
   if( !memchr( array_start( &cookie->data.request ), '\n', array_bytes( &cookie->data.request ) ) )
     return 0;
 
@@ -370,6 +368,11 @@ int parse_configfile( char * config_filename ) {
       if( !scan_ip6( p+13, tmpip )) goto parse_error;
       accesslist_blessip( tmpip, OT_PERMISSION_MAY_STAT );
 #endif
+#ifdef WANT_IP_FROM_PROXY
+    } else if(!byte_diff(p, 12, "access.proxy" ) && isspace(p[12])) {
+      if( !scan_ip6( p+13, tmpip )) goto parse_error;
+      accesslist_blessip( tmpip, OT_PERMISSION_MAY_PROXY );
+#endif
     } else if(!byte_diff(p, 20, "tracker.redirect_url" ) && isspace(p[20])) {
       set_config_option( &g_redirecturl, p+21 );
 #ifdef WANT_SYNC_LIVE
diff --git a/ot_accesslist.c b/ot_accesslist.c
index 304b3f1..631ab88 100644
--- a/ot_accesslist.c
+++ b/ot_accesslist.c
@@ -124,6 +124,7 @@ int accesslist_blessip( ot_ip6 ip, ot_permissions permissions ) {
     if( permissions & OT_PERMISSION_MAY_STAT       ) off += snprintf( _debug+off, 512-off, " may_fetch_stats" );
     if( permissions & OT_PERMISSION_MAY_LIVESYNC   ) off += snprintf( _debug+off, 512-off, " may_sync_live" );
     if( permissions & OT_PERMISSION_MAY_FULLSCRAPE ) off += snprintf( _debug+off, 512-off, " may_fetch_fullscrapes" );
+    if( permissions & OT_PERMISSION_MAY_PROXY      ) off += snprintf( _debug+off, 512-off, " may_proxy" );
     if( !permissions ) off += snprintf( _debug+off, sizeof(_debug)-off, " nothing\n" );
     _debug[off++] = '.';
     write( 2, _debug, off );
diff --git a/ot_accesslist.h b/ot_accesslist.h
index 836a233..5954a4a 100644
--- a/ot_accesslist.h
+++ b/ot_accesslist.h
@@ -27,7 +27,8 @@ extern char *g_accesslist_filename;
 typedef enum {
   OT_PERMISSION_MAY_FULLSCRAPE = 0x1,
   OT_PERMISSION_MAY_STAT       = 0x2,
-  OT_PERMISSION_MAY_LIVESYNC   = 0x4
+  OT_PERMISSION_MAY_LIVESYNC   = 0x4,
+  OT_PERMISSION_MAY_PROXY      = 0x8
 } ot_permissions;
 
 int  accesslist_blessip( ot_ip6 ip, ot_permissions permissions );
diff --git a/ot_http.c b/ot_http.c
index 971d81d..de57dad 100644
--- a/ot_http.c
+++ b/ot_http.c
@@ -306,13 +306,14 @@ static ot_keywords keywords_announce[] = { { "port", 1 }, { "left", 2 }, { "even
 { NULL, -3 } };
 static ot_keywords keywords_announce_event[] = { { "completed", 1 }, { "stopped", 2 }, { NULL, -3 } };
 static ssize_t http_handle_announce( const int64 sock, struct ot_workstruct *ws, char *read_ptr ) {
-  int            numwant, tmp, scanon;
-  ot_peer        peer;
-  ot_hash       *hash = NULL;
-  unsigned short port = htons(6881);
-  char          *write_ptr;
-  ssize_t        len;
-  
+  int               numwant, tmp, scanon;
+  ot_peer           peer;
+  ot_hash          *hash = NULL;
+  unsigned short    port = htons(6881);
+  char             *write_ptr;
+  ssize_t           len;
+  struct http_data *cookie = io_getcookie( sock );    
+
   /* This is to hack around stupid clients that send "announce ?info_hash" */
   if( read_ptr[-1] != '?' ) {
     while( ( *read_ptr != '?' ) && ( *read_ptr != '\n' ) ) ++read_ptr;
@@ -320,7 +321,33 @@ static ssize_t http_handle_announce( const int64 sock, struct ot_workstruct *ws,
     ++read_ptr;
   }
 
-  OT_SETIP( &peer, ((struct http_data*)io_getcookie( sock ) )->ip );
+#ifdef WANT_IP_FROM_PROXY
+  if( accesslist_isblessed( cookie->ip, OT_PERMISSION_MAY_PROXY ) ) {
+    ot_ip6 proxied_ip;
+    char *fwd, *fwd_new = ws->request;
+
+    /* Zero terminate for string routines. Normally we'd only overwrite bollocks */
+    ws->request[ws->request_size-1] = 0;
+
+    /* Find last occurence of the forwarded header */
+    do {
+      fwd = fwd_new;
+      fwd_new = strcasestr( fwd_new, "\nX-Forwarded-For:" );
+    } while( fwd_new );
+
+    /* Skip spaces between : and the ip address */
+    if( fwd ) {
+      fwd += 18; /* sizeof( "\nX-Forwarded-For:" ) */
+      while( *fwd == ' ' ) ++fwd;
+    }
+
+    if( fwd && scan_ip6( fwd, proxied_ip ) )
+      OT_SETIP( &peer, proxied_ip );
+    else
+      OT_SETIP( &peer, cookie->ip );
+  }
+#endif
+  OT_SETIP( &peer, cookie->ip );
   OT_SETPORT( &peer, &port );
   OT_PEERFLAG( &peer ) = 0;
   numwant = 50;
diff --git a/ot_mutex.c b/ot_mutex.c
index 925eb27..2052ffa 100644
--- a/ot_mutex.c
+++ b/ot_mutex.c
@@ -18,6 +18,7 @@
 /* Opentracker */
 #include "trackerlogic.h"
 #include "ot_mutex.h"
+#include "ot_stats.h"
 
 /* #define MTX_DBG( STRING ) fprintf( stderr, STRING ) */
 #define MTX_DBG( STRING )
@@ -47,8 +48,10 @@ static int bucket_check( int bucket ) {
 
   /* See, if bucket is already locked */
   for( i=0; i<bucket_locklist_count; ++i )
-    if( bucket_locklist[ i ] == bucket )
+    if( bucket_locklist[ i ] == bucket ) {
+      stats_issue_event( EVENT_BUCKET_LOCKED, 0, 0 );
       return -1;
+    }
 
   return 0;
 }
diff --git a/ot_stats.c b/ot_stats.c
index a53ac3b..518dbb5 100644
--- a/ot_stats.c
+++ b/ot_stats.c
@@ -1,7 +1,7 @@
 /* This software was written by Dirk Engling <erdgeist@erdgeist.org>
-   It is considered beerware. Prost. Skol. Cheers or whatever.
-
-   $id$ */
+ It is considered beerware. Prost. Skol. Cheers or whatever.
+ 
+ $id$ */
 
 /* System */
 #include <stdlib.h>
@@ -50,12 +50,13 @@ static unsigned long long ot_full_scrape_size = 0;
 static unsigned long long ot_failed_request_counts[CODE_HTTPERROR_COUNT];
 static unsigned long long ot_renewed[OT_PEER_TIMEOUT];
 static unsigned long long ot_overall_sync_count;
+static unsigned long long ot_overall_stall_count;
 
 static time_t ot_start_time;
 
 #ifdef WANT_LOG_NETWORKS
 #define STATS_NETWORK_NODE_BITWIDTH  8
-#define STATS_NETWORK_NODE_MAXDEPTH  3
+#define STATS_NETWORK_NODE_MAXDEPTH  16
 
 #define STATS_NETWORK_NODE_BITMASK ((1<<STATS_NETWORK_NODE_BITWIDTH)-1)
 #define STATS_NETWORK_NODE_COUNT    (1<<STATS_NETWORK_NODE_BITWIDTH)
@@ -68,19 +69,20 @@ union stats_network_node {
 
 static stats_network_node *stats_network_counters_root = NULL;
 
-static int stat_increase_network_count( stats_network_node **node, int depth, uint32_t ip ) {
-  int foo = ( ip >> ( 32 - STATS_NETWORK_NODE_BITWIDTH * ( ++depth ) ) ) & STATS_NETWORK_NODE_BITMASK;
-
+static int stat_increase_network_count( stats_network_node **node, int depth, uintptr_t ip ) {
+  ot_ip6 *_ip = (ot_ip6*)ip;
+  int foo = (*_ip)[depth];
+  
   if( !*node ) {
     *node = malloc( sizeof( stats_network_node ) );
     if( !*node )
       return -1;
     memset( *node, 0, sizeof( stats_network_node ) );
   }
-
+  
   if( depth < STATS_NETWORK_NODE_MAXDEPTH )
     return stat_increase_network_count( &(*node)->children[ foo ], depth, ip );
-
+  
   (*node)->counters[ foo ]++;
   return 0;
 }
@@ -88,40 +90,40 @@ static int stat_increase_network_count( stats_network_node **node, int depth, ui
 static int stats_shift_down_network_count( stats_network_node **node, int depth, int shift ) {
   int i, rest = 0;
   if( !*node ) return 0;
-
+  
   if( ++depth == STATS_NETWORK_NODE_MAXDEPTH )
     for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i ) {
       rest += ((*node)->counters[i]>>=shift);
-    return rest;
-  }
-
+      return rest;
+    }
+  
   for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i ) {
     stats_network_node **childnode = &(*node)->children[i];
     int rest_val;
-
+    
     if( !*childnode ) continue;
-
+    
     rest += rest_val = stats_shift_down_network_count( childnode, depth, shift );
-
+    
     if( rest_val ) continue;
-
+    
     free( (*node)->children[i] );
     (*node)->children[i] = NULL;
   }
-
+  
   return rest;
 }
 
 static void stats_get_highscore_networks( stats_network_node *node, int depth, uint32_t node_value, int *scores, uint32_t *networks, int network_count ) {
   int i;
-
+  
   if( !node ) return;
-
+  
   if( !depth++ ) {
     memset( scores, 0, sizeof( *scores ) * network_count );
     memset( networks, 0, sizeof( *networks ) * network_count );
   }
-
+  
   if( depth < STATS_NETWORK_NODE_MAXDEPTH ) {
     for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i )
       if( node->children[i] )
@@ -130,15 +132,15 @@ static void stats_get_highscore_networks( stats_network_node *node, int depth, u
     for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i ) {
       int j=1;
       if( node->counters[i] <= scores[0] ) continue;
-
+      
       while( (j<network_count) && (node->counters[i]>scores[j] ) ) ++j;
       --j;
-
+      
       memcpy( scores, scores + 1, j * sizeof( *scores ) );
       memcpy( networks, networks + 1, j * sizeof( *networks ) );
       scores[ j ] = node->counters[ i ];
       networks[ j ] = node_value | ( i << ( 32 - depth * STATS_NETWORK_NODE_BITWIDTH ) );
-  }
+    }
 }
 
 static size_t stats_return_busy_networks( char * reply ) {
@@ -146,17 +148,31 @@ static size_t stats_return_busy_networks( char * reply ) {
   int      scores[16];
   int      i;
   char   * r = reply;
-
+  
   stats_get_highscore_networks( stats_network_counters_root, 0, 0, scores, networks, 16 );
-
+  
   for( i=15; i>=0; --i)
     r += sprintf( r, "%08i: %d.%d.%d.0/24\n", scores[i], (networks[i]>>24)&0xff, (networks[i]>>16)&0xff, (networks[i]>>8)&0xff );
-
+  
   return r - reply;
 }
 
 #endif
 
+typedef struct {
+  unsigned long long torrent_count;
+  unsigned long long peer_count;
+  unsigned long long seed_count;
+} torrent_stats;
+
+static int torrent_statter( ot_torrent *torrent, uintptr_t data ) {
+  torrent_stats *stats = (torrent_stats*)data;
+  stats->torrent_count++;
+  stats->peer_count += torrent->peer_list->peer_count;
+  stats->seed_count += torrent->peer_list->seed_count;
+  return 0;
+}
+
 /* Converter function from memory to human readable hex strings */
 static char*to_hex(char*d,uint8_t*s){char*m="0123456789ABCDEF";char *t=d;char*e=d+40;while(d<e){*d++=m[*s>>4];*d++=m[*s++&15];}*d=0;return t;}
 
@@ -168,10 +184,10 @@ size_t stats_top10_txt( char * reply ) {
   ot_record top10s[10], top10c[10];
   char     *r  = reply, hex_out[42];
   int       idx, bucket;
-
+  
   byte_zero( top10s, sizeof( top10s ) );
   byte_zero( top10c, sizeof( top10c ) );
-
+  
   for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
     ot_vector *torrents_list = mutex_bucket_lock( bucket );
     for( j=0; j<torrents_list->size; ++j ) {
@@ -193,7 +209,7 @@ size_t stats_top10_txt( char * reply ) {
     if( !g_opentracker_running )
       return 0;
   }
-
+  
   r += sprintf( r, "Top 10 torrents by peers:\n" );
   for( idx=0; idx<10; ++idx )
     if( top10c[idx].torrent )
@@ -202,31 +218,31 @@ size_t stats_top10_txt( char * reply ) {
   for( idx=0; idx<10; ++idx )
     if( top10s[idx].torrent )
       r += sprintf( r, "\t%zd\t%s\n", top10s[idx].val, to_hex( hex_out, top10s[idx].torrent->hash) );
-
+  
   return r - reply;
 }
 
 /* This function collects 4096 /24s in 4096 possible
-   malloc blocks
-*/
+ malloc blocks
+ */
 static size_t stats_slash24s_txt( char * reply, size_t amount, uint32_t thresh ) {
-
+  
 #define NUM_TOPBITS 12
 #define NUM_LOWBITS (24-NUM_TOPBITS)
 #define NUM_BUFS    (1<<NUM_TOPBITS)
 #define NUM_S24S    (1<<NUM_LOWBITS)
 #define MSK_S24S    (NUM_S24S-1)
-
+  
   uint32_t *counts[ NUM_BUFS ];
   uint32_t  slash24s[amount*2];  /* first dword amount, second dword subnet */
   size_t    i, j, k, l;
   char     *r  = reply;
-
+  
   byte_zero( counts, sizeof( counts ) );
   byte_zero( slash24s, amount * 2 * sizeof(uint32_t) );
-
+  
   r += sprintf( r, "Stats for all /24s with more than %u announced torrents:\n\n", thresh );
-
+  
 #if 0
   /* XXX: TOOD: Doesn't work yet with new peer storage model */
   for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
@@ -257,7 +273,7 @@ static size_t stats_slash24s_txt( char * reply, size_t amount, uint32_t thresh )
       goto bailout_cleanup;
   }
 #endif
-
+  
   k = l = 0; /* Debug: count allocated bufs */
   for( i=0; i < NUM_BUFS; ++i ) {
     uint32_t *count = counts[i];
@@ -281,28 +297,28 @@ static size_t stats_slash24s_txt( char * reply, size_t amount, uint32_t thresh )
     }
     free( count );
   }
-
+  
   r += sprintf( r, "Allocated bufs: %zd, used s24s: %zd\n", k, l );
-
+  
   for( i=0; i < amount; ++i )
     if( slash24s[ 2*i ] >= thresh ) {
       uint32_t ip = slash24s[ 2*i +1 ];
       r += sprintf( r, "% 10ld %d.%d.%d.0/24\n", (long)slash24s[ 2*i ], (int)(ip >> 16), (int)(255 & ( ip >> 8 )), (int)(ip & 255) );
     }
-
+  
   return r - reply;
-
+  
   for( i=0; i < NUM_BUFS; ++i )
     free( counts[i] );
-
+  
   return 0;
 }
 
 /*
  struct {
-   size_t size
-   size_t space
-   size_t count
+ size_t size
+ size_t space
+ size_t count
  }
  */
 
@@ -313,105 +329,95 @@ static unsigned long events_per_time( unsigned long long events, time_t t ) {
 static size_t stats_connections_mrtg( char * reply ) {
   ot_time t = time( NULL ) - ot_start_time;
   return sprintf( reply,
-    "%llu\n%llu\n%i seconds (%i hours)\nopentracker connections, %lu conns/s :: %lu success/s.",
-    ot_overall_tcp_connections+ot_overall_udp_connections,
-    ot_overall_tcp_successfulannounces+ot_overall_udp_successfulannounces+ot_overall_udp_connects,
-    (int)t,
-    (int)(t / 3600),
-    events_per_time( ot_overall_tcp_connections+ot_overall_udp_connections, t ),
-    events_per_time( ot_overall_tcp_successfulannounces+ot_overall_udp_successfulannounces+ot_overall_udp_connects, t )
-  );
+                 "%llu\n%llu\n%i seconds (%i hours)\nopentracker connections, %lu conns/s :: %lu success/s.",
+                 ot_overall_tcp_connections+ot_overall_udp_connections,
+                 ot_overall_tcp_successfulannounces+ot_overall_udp_successfulannounces+ot_overall_udp_connects,
+                 (int)t,
+                 (int)(t / 3600),
+                 events_per_time( ot_overall_tcp_connections+ot_overall_udp_connections, t ),
+                 events_per_time( ot_overall_tcp_successfulannounces+ot_overall_udp_successfulannounces+ot_overall_udp_connects, t )
+                 );
 }
 
 static size_t stats_udpconnections_mrtg( char * reply ) {
   ot_time t = time( NULL ) - ot_start_time;
   return sprintf( reply,
-    "%llu\n%llu\n%i seconds (%i hours)\nopentracker udp4 stats, %lu conns/s :: %lu success/s.",
-    ot_overall_udp_connections,
-    ot_overall_udp_successfulannounces+ot_overall_udp_connects,
-    (int)t,
-    (int)(t / 3600),
-    events_per_time( ot_overall_udp_connections, t ),
-    events_per_time( ot_overall_udp_successfulannounces+ot_overall_udp_connects, t )
-  );
+                 "%llu\n%llu\n%i seconds (%i hours)\nopentracker udp4 stats, %lu conns/s :: %lu success/s.",
+                 ot_overall_udp_connections,
+                 ot_overall_udp_successfulannounces+ot_overall_udp_connects,
+                 (int)t,
+                 (int)(t / 3600),
+                 events_per_time( ot_overall_udp_connections, t ),
+                 events_per_time( ot_overall_udp_successfulannounces+ot_overall_udp_connects, t )
+                 );
 }
 
 static size_t stats_tcpconnections_mrtg( char * reply ) {
   time_t t = time( NULL ) - ot_start_time;
   return sprintf( reply,
-    "%llu\n%llu\n%i seconds (%i hours)\nopentracker tcp4 stats, %lu conns/s :: %lu success/s.",
-    ot_overall_tcp_connections,
-    ot_overall_tcp_successfulannounces,
-    (int)t,
-    (int)(t / 3600),
-    events_per_time( ot_overall_tcp_connections, t ),
-    events_per_time( ot_overall_tcp_successfulannounces, t )
-  );
+                 "%llu\n%llu\n%i seconds (%i hours)\nopentracker tcp4 stats, %lu conns/s :: %lu success/s.",
+                 ot_overall_tcp_connections,
+                 ot_overall_tcp_successfulannounces,
+                 (int)t,
+                 (int)(t / 3600),
+                 events_per_time( ot_overall_tcp_connections, t ),
+                 events_per_time( ot_overall_tcp_successfulannounces, t )
+                 );
 }
 
 static size_t stats_scrape_mrtg( char * reply ) {
   time_t t = time( NULL ) - ot_start_time;
   return sprintf( reply,
-    "%llu\n%llu\n%i seconds (%i hours)\nopentracker scrape stats, %lu scrape/s (tcp and udp)",
-    ot_overall_tcp_successfulscrapes,
-    ot_overall_udp_successfulscrapes,
-    (int)t,
-    (int)(t / 3600),
-    events_per_time( (ot_overall_tcp_successfulscrapes+ot_overall_udp_successfulscrapes), t )
-  );
+                 "%llu\n%llu\n%i seconds (%i hours)\nopentracker scrape stats, %lu scrape/s (tcp and udp)",
+                 ot_overall_tcp_successfulscrapes,
+                 ot_overall_udp_successfulscrapes,
+                 (int)t,
+                 (int)(t / 3600),
+                 events_per_time( (ot_overall_tcp_successfulscrapes+ot_overall_udp_successfulscrapes), t )
+                 );
 }
 
 static size_t stats_fullscrapes_mrtg( char * reply ) {
   ot_time t = time( NULL ) - ot_start_time;
   return sprintf( reply,
-    "%llu\n%llu\n%i seconds (%i hours)\nopentracker full scrape stats, %lu conns/s :: %lu bytes/s.",
-    ot_full_scrape_count * 1000,
-    ot_full_scrape_size,
-    (int)t,
-    (int)(t / 3600),
-    events_per_time( ot_full_scrape_count, t ),
-    events_per_time( ot_full_scrape_size, t )
-  );
+                 "%llu\n%llu\n%i seconds (%i hours)\nopentracker full scrape stats, %lu conns/s :: %lu bytes/s.",
+                 ot_full_scrape_count * 1000,
+                 ot_full_scrape_size,
+                 (int)t,
+                 (int)(t / 3600),
+                 events_per_time( ot_full_scrape_count, t ),
+                 events_per_time( ot_full_scrape_size, t )
+                 );
 }
 
 static size_t stats_peers_mrtg( char * reply ) {
-  size_t    torrent_count = 0, peer_count = 0, seed_count = 0, j;
-  int bucket;
-
-  for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
-    ot_vector *torrents_list = mutex_bucket_lock( bucket );
-    torrent_count += torrents_list->size;
-    for( j=0; j<torrents_list->size; ++j ) {
-      ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[j] ).peer_list;
-      peer_count += peer_list->peer_count; seed_count += peer_list->seed_count;
-    }
-    mutex_bucket_unlock( bucket, 0 );
-    if( !g_opentracker_running )
-      return 0;
-  }
-  return sprintf( reply, "%zd\n%zd\nopentracker serving %zd torrents\nopentracker",
-    peer_count,
-    seed_count,
-    torrent_count
-  );
+  torrent_stats stats = {0,0,0};
+  
+  iterate_all_torrents( torrent_statter, (uintptr_t)&stats );
+  
+  return sprintf( reply, "%llu\n%llu\nopentracker serving %llu torrents\nopentracker",
+                 stats.peer_count,
+                 stats.seed_count,
+                 stats.torrent_count
+                 );
 }
 
 static size_t stats_startstop_mrtg( char * reply )
 {
   size_t    torrent_count = mutex_get_torrent_count();
-
+  
   return sprintf( reply, "%zd\n%zd\nopentracker handling %zd torrents\nopentracker",
-    (size_t)0,
-    (size_t)0,
-    torrent_count
-  );
+                 (size_t)0,
+                 (size_t)0,
+                 torrent_count
+                 );
 }
 
 static size_t stats_toraddrem_mrtg( char * reply )
 {
   size_t    peer_count = 0, j;
   int bucket;
-
+  
   for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket )
   {
     ot_vector *torrents_list = mutex_bucket_lock( bucket );
@@ -424,53 +430,63 @@ static size_t stats_toraddrem_mrtg( char * reply )
     if( !g_opentracker_running )
       return 0;
   }
-
+  
   return sprintf( reply, "%zd\n%zd\nopentracker handling %zd peers\nopentracker",
-    (size_t)0,
-    (size_t)0,
-    peer_count
-  );
+                 (size_t)0,
+                 (size_t)0,
+                 peer_count
+                 );
 }
 
 static size_t stats_torrents_mrtg( char * reply )
 {
   size_t torrent_count = mutex_get_torrent_count();
-
+  
   return sprintf( reply, "%zd\n%zd\nopentracker serving %zd torrents\nopentracker",
-    torrent_count,
-    (size_t)0,
-    torrent_count
-  );
+                 torrent_count,
+                 (size_t)0,
+                 torrent_count
+                 );
 }
 
 static size_t stats_httperrors_txt ( char * reply ) {
   return sprintf( reply, "302 RED %llu\n400 ... %llu\n400 PAR %llu\n400 COM %llu\n403 IP  %llu\n404 INV %llu\n500 SRV %llu\n",
-  ot_failed_request_counts[0], ot_failed_request_counts[1], ot_failed_request_counts[2],
-  ot_failed_request_counts[3], ot_failed_request_counts[4], ot_failed_request_counts[5],
-  ot_failed_request_counts[6] );
+                 ot_failed_request_counts[0], ot_failed_request_counts[1], ot_failed_request_counts[2],
+                 ot_failed_request_counts[3], ot_failed_request_counts[4], ot_failed_request_counts[5],
+                 ot_failed_request_counts[6] );
 }
 
 static size_t stats_return_renew_bucket( char * reply ) {
   char *r = reply;
   int i;
-
+  
   for( i=0; i<OT_PEER_TIMEOUT; ++i )
     r+=sprintf(r,"%02i %llu\n", i, ot_renewed[i] );
   return r - reply;
 }
 
-static size_t stats_return_sync_mrtg( char * reply )
-{
+static size_t stats_return_sync_mrtg( char * reply ) {
         ot_time t = time( NULL ) - ot_start_time;
         return sprintf( reply,
-                                   "%llu\n%llu\n%i seconds (%i hours)\nopentracker connections, %lu conns/s :: %lu success/s.",
-                                   ot_overall_sync_count,
-                                   0LL,
-                                   (int)t,
-                                   (int)(t / 3600),
-                                   events_per_time( ot_overall_tcp_connections+ot_overall_udp_connections, t ),
-                                   events_per_time( ot_overall_tcp_successfulannounces+ot_overall_udp_successfulannounces+ot_overall_udp_connects, t )
-                                   );
+                 "%llu\n%llu\n%i seconds (%i hours)\nopentracker connections, %lu conns/s :: %lu success/s.",
+                 ot_overall_sync_count,
+                 0LL,
+                 (int)t,
+                 (int)(t / 3600),
+                 events_per_time( ot_overall_tcp_connections+ot_overall_udp_connections, t ),
+                 events_per_time( ot_overall_tcp_successfulannounces+ot_overall_udp_successfulannounces+ot_overall_udp_connects, t )
+                 );
+}
+
+static size_t stats_return_everything( char * reply ) {
+  char * r = reply;
+  r += sprintf( r, "<stats>\n" );
+  r += sprintf( r, "  <uptime>%llu</uptime>\n", (unsigned long long)(time( NULL ) - ot_start_time) );
+  r += sprintf( r, "  <torrents>%zd</torrents>\n", mutex_get_torrent_count() );
+  /*  r += sprintf( r, "  <peers>%llu</peers>\n",  ); */
+  
+  r += sprintf( reply, "</stats>" );
+  return r - reply;
 }
 
 extern const char
@@ -480,9 +496,9 @@ extern const char
 
 size_t stats_return_tracker_version( char *reply ) {
   return sprintf( reply, "%s%s%s%s%s%s%s%s%s%s%s%s%s",
-  g_version_opentracker_c, g_version_accesslist_c, g_version_clean_c, g_version_fullscrape_c, g_version_http_c,
-  g_version_iovec_c, g_version_mutex_c, g_version_stats_c, g_version_udp_c, g_version_vector_c,
-  g_version_scan_urlencoded_query_c, g_version_trackerlogic_c, g_version_livesync_c );
+                 g_version_opentracker_c, g_version_accesslist_c, g_version_clean_c, g_version_fullscrape_c, g_version_http_c,
+                 g_version_iovec_c, g_version_mutex_c, g_version_stats_c, g_version_udp_c, g_version_vector_c,
+                 g_version_scan_urlencoded_query_c, g_version_trackerlogic_c, g_version_livesync_c );
 }
 
 size_t return_stats_for_tracker( char *reply, int mode, int format ) {
@@ -509,7 +525,7 @@ size_t return_stats_for_tracker( char *reply, int mode, int format ) {
     case TASK_STATS_RENEW:
       return stats_return_renew_bucket( reply );
     case TASK_STATS_SYNCS:
-          return stats_return_sync_mrtg( reply );
+      return stats_return_sync_mrtg( reply );
 #ifdef WANT_LOG_NETWORKS
     case TASK_STATS_BUSY_NETWORKS:
       return stats_return_busy_networks( reply );
@@ -521,12 +537,12 @@ size_t return_stats_for_tracker( char *reply, int mode, int format ) {
 
 static void stats_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ) {
   char *r;
-
+  
   *iovec_entries = 0;
   *iovector      = NULL;
   if( !( r = iovec_increase( iovec_entries, iovector, OT_STATS_TMPSIZE ) ) )
     return;
-
+  
   switch( mode & TASK_TASK_MASK ) {
     case TASK_STATS_TORRENTS:    r += stats_torrents_mrtg( r );             break;
     case TASK_STATS_PEERS:       r += stats_peers_mrtg( r );                break;
@@ -560,26 +576,26 @@ void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uintptr_t event
       ot_full_scrape_size += event_data;
       break;
     case EVENT_FULLSCRAPE_REQUEST:
-      {
-        ot_ip6 *ip = (ot_ip6*)event_data; /* ugly hack to transfer ip to stats */
-        char _debug[512];
-        int off = snprintf( _debug, sizeof(_debug), "[%08d] scrp:  ", (unsigned int)(g_now_seconds - ot_start_time)/60 );
-        off += fmt_ip6( _debug+off, *ip );
-        off += snprintf( _debug+off, sizeof(_debug)-off, " - FULL SCRAPE\n" );
-        write( 2, _debug, off );
-        ot_full_scrape_request_count++;
-      }
+    {
+      ot_ip6 *ip = (ot_ip6*)event_data; /* ugly hack to transfer ip to stats */
+      char _debug[512];
+      int off = snprintf( _debug, sizeof(_debug), "[%08d] scrp:  ", (unsigned int)(g_now_seconds - ot_start_time)/60 );
+      off += fmt_ip6( _debug+off, *ip );
+      off += snprintf( _debug+off, sizeof(_debug)-off, " - FULL SCRAPE\n" );
+      write( 2, _debug, off );
+      ot_full_scrape_request_count++;
+    }
       break;
     case EVENT_FULLSCRAPE_REQUEST_GZIP:
-      {
-        ot_ip6 *ip = (ot_ip6*)event_data; /* ugly hack to transfer ip to stats */
-        char _debug[512];
-        int off = snprintf( _debug, sizeof(_debug), "[%08d] scrp:  ", (unsigned int)(g_now_seconds - ot_start_time)/60 );
-        off += fmt_ip6(_debug+off, *ip );
-        off += snprintf( _debug+off, sizeof(_debug)-off, " - FULL SCRAPE\n" );
-        write( 2, _debug, off );
-        ot_full_scrape_request_count++;
-      }
+    {
+      ot_ip6 *ip = (ot_ip6*)event_data; /* ugly hack to transfer ip to stats */
+      char _debug[512];
+      int off = snprintf( _debug, sizeof(_debug), "[%08d] scrp:  ", (unsigned int)(g_now_seconds - ot_start_time)/60 );
+      off += fmt_ip6(_debug+off, *ip );
+      off += snprintf( _debug+off, sizeof(_debug)-off, " - FULL SCRAPE\n" );
+      write( 2, _debug, off );
+      ot_full_scrape_request_count++;
+    }
       break;
     case EVENT_FAILED:
       ot_failed_request_counts[event_data]++;
@@ -590,6 +606,9 @@ void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uintptr_t event
     case EVENT_SYNC:
       ot_overall_sync_count+=event_data;
             break;
+    case EVENT_BUCKET_LOCKED:
+      ot_overall_stall_count++;
+      break;
     default:
       break;
   }
@@ -598,9 +617,9 @@ void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uintptr_t event
 static void * stats_worker( void * args ) {
   int iovec_entries;
   struct iovec *iovector;
-
+  
   args = args;
-
+  
   while( 1 ) {
     ot_tasktype tasktype = TASK_STATS;
     ot_taskid   taskid   = mutex_workqueue_poptask( &tasktype );
diff --git a/ot_stats.h b/ot_stats.h
index b8a64a1..7753947 100644
--- a/ot_stats.h
+++ b/ot_stats.h
@@ -17,7 +17,8 @@ typedef enum {
   EVENT_FULLSCRAPE_REQUEST,
   EVENT_FULLSCRAPE_REQUEST_GZIP,
   EVENT_FULLSCRAPE,   /* TCP only */
-  EVENT_FAILED
+  EVENT_FAILED,
+  EVENT_BUCKET_LOCKED
 } ot_status_event;
 
 enum {
diff --git a/trackerlogic.c b/trackerlogic.c
index 249a2a0..8ebaa46 100644
--- a/trackerlogic.c
+++ b/trackerlogic.c
@@ -337,6 +337,23 @@ size_t remove_peer_from_torrent( ot_hash hash, ot_peer *peer, char *reply, PROTO
   return reply_size;
 }
 
+void iterate_all_torrents( int (*for_each)( ot_torrent* torrent, uintptr_t data ), uintptr_t data ) {
+  int bucket;
+  size_t j;
+
+  for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
+    ot_vector  *torrents_list = mutex_bucket_lock( bucket );
+    ot_torrent *torrents = (ot_torrent*)(torrents_list->data);
+
+    for( j=0; j<torrents_list->size; ++j )
+      if( for_each( torrents + j, data ) )
+        break;
+
+    mutex_bucket_unlock( bucket, 0 );
+    if( !g_opentracker_running ) return;
+  }
+}
+
 void exerr( char * message ) {
   fprintf( stderr, "%s\n", message );
   exit( 111 );
@@ -358,7 +375,7 @@ void trackerlogic_init( ) {
 void trackerlogic_deinit( void ) {
   int bucket, delta_torrentcount = 0;
   size_t j;
-
+  
   /* Free all torrents... */
   for(bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
     ot_vector *torrents_list = mutex_bucket_lock( bucket );
diff --git a/trackerlogic.h b/trackerlogic.h
index 69deddd..ce3feb1 100644
--- a/trackerlogic.h
+++ b/trackerlogic.h
@@ -148,6 +148,9 @@ size_t  remove_peer_from_torrent( ot_hash hash, ot_peer *peer, char *reply, PROT
 size_t  return_tcp_scrape_for_torrent( ot_hash *hash, int amount, char *reply );
 size_t  return_udp_scrape_for_torrent( ot_hash hash, char *reply );
 
+/* torrent iterator */
+void iterate_all_torrents( int (*for_each)( ot_torrent* torrent, uintptr_t data ), uintptr_t data );
+
 /* Helper, before it moves to its own object */
 void free_peerlist( ot_peerlist *peer_list );