1 files changed, 282 insertions, 210 deletions

diff --git a/ot_stats.c b/ot_stats.c
index 6589aef..5f97a2f 100644
--- a/ot_stats.c
+++ b/ot_stats.c
@@ -6,6 +6,7 @@
 /* System */
 #include <inttypes.h>
 #include <pthread.h>
+#include <stdatomic.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -16,15 +17,13 @@
 #include <syslog.h>
 #endif
 
-/* Libowfat */
-#include "io.h"
-
 /* Opentracker */
 #include "ot_accesslist.h"
 #include "ot_hex.h"
 #include "ot_iovec.h"
 #include "ot_ip.h"
 #include "ot_mutex.h"
+#include "ot_fmt.h"
 #include "ot_stats.h"
 #include "trackerlogic.h"
 
@@ -38,27 +37,58 @@
 static void stats_make(size_t *iovec_entries, struct iovec **iovector, ot_tasktype mode);
 #define OT_STATS_TMPSIZE 8192
 
-/* Clumsy counters... to be rethought */
-static unsigned long long ot_overall_tcp_connections;
-static unsigned long long ot_overall_udp_connections;
-static unsigned long long ot_overall_tcp_successfulannounces;
-static unsigned long long ot_overall_udp_successfulannounces;
-static unsigned long long ot_overall_tcp_successfulscrapes;
-static unsigned long long ot_overall_udp_successfulscrapes;
-static unsigned long long ot_overall_udp_connectionidmissmatches;
-static unsigned long long ot_overall_tcp_connects;
-static unsigned long long ot_overall_udp_connects;
-static unsigned long long ot_overall_completed;
-static unsigned long long ot_full_scrape_count;
-static unsigned long long ot_full_scrape_request_count;
-static unsigned long long ot_full_scrape_size;
-static unsigned long long ot_failed_request_counts[CODE_HTTPERROR_COUNT];
+/* Each serving thread keeps its own statistics tally. The list is linked
+   through intrusive next pointers within the workstruct.
+
+   Registration happens once per thread and the adding up once per statistics
+   request, so this lock is never on anybody's hot path. */
+static struct ot_workstruct *g_counter_contexts;
+static ot_counter            g_retired[OT_COUNTER_COUNT];
+static pthread_mutex_t       g_counter_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+void stats_counters_register(struct ot_workstruct *ws) {
+  pthread_mutex_lock(&g_counter_mutex);
+  ws->counters_next = g_counter_contexts;
+  g_counter_contexts = ws;
+  pthread_mutex_unlock(&g_counter_mutex);
+}
+
+void stats_counters_retire(struct ot_workstruct *ws) {
+  struct ot_workstruct **link;
+  size_t                 i;
+
+  pthread_mutex_lock(&g_counter_mutex);
+  for (link = &g_counter_contexts; *link; link = &(*link)->counters_next)
+    if (*link == ws) {
+      *link = ws->counters_next;
+      break;
+    }
+
+  /* When retiring a worker thread, collect the counter tally in the global */
+  for (i = 0; i < OT_COUNTER_COUNT; ++i)
+    atomic_fetch_add_explicit(g_retired + i, atomic_load_explicit(ws->counters + i, memory_order_relaxed), memory_order_relaxed);
+  pthread_mutex_unlock(&g_counter_mutex);
+}
+
+/* What every thread has counted so far, added together */
+static void counters_total(unsigned long long *total) {
+  struct ot_workstruct *ws;
+  size_t                i;
+
+  pthread_mutex_lock(&g_counter_mutex);
+  for (i = 0; i < OT_COUNTER_COUNT; ++i)
+    total[i] = atomic_load_explicit(g_retired + i, memory_order_relaxed);
+  for (ws = g_counter_contexts; ws; ws = ws->counters_next)
+    for (i = 0; i < OT_COUNTER_COUNT; ++i)
+      total[i] += atomic_load_explicit(ws->counters + i, memory_order_relaxed);
+  pthread_mutex_unlock(&g_counter_mutex);
+}
+
+#define COUNT(index, by) atomic_fetch_add_explicit(ws->counters + (index), (unsigned long long)(by), memory_order_relaxed)
+
 static const char        *ot_failed_request_names[] = {
     "302 Redirect", "400 Parse Error", "400 Invalid Parameter", "400 Invalid Parameter (compact=0)", "400 Not Modest",
     "402 Payment Required", "403 Access Denied", "404 Not found", "500 Internal Server Error"};
-static unsigned long long ot_renewed[OT_PEER_TIMEOUT];
-static unsigned long long ot_overall_sync_count;
-static unsigned long long ot_overall_stall_count;
 
 static time_t             ot_start_time;
 
@@ -178,8 +208,8 @@ static size_t stats_get_highscore_networks(stats_network_node *node, int depth,
       ++j;
     --j;
 
-    memcpy(scores, scores + 1, j * sizeof(*scores));
-    memcpy(networks, networks + 1, j * sizeof(*networks));
+    memmove(scores, scores + 1, j * sizeof(*scores));
+    memmove(networks, networks + 1, j * sizeof(*networks));
     scores[j] = node_score;
     memcpy(networks + j, node_value, sizeof(*networks));
   }
@@ -187,12 +217,11 @@ static size_t stats_get_highscore_networks(stats_network_node *node, int depth,
   return score;
 }
 
-static size_t stats_return_busy_networks(char *reply, stats_network_node *tree, size_t amount, int limit) {
+static void stats_return_busy_networks(char **reply, const char *reply_end, stats_network_node *tree, size_t amount, int limit) {
   ot_ip6 networks[amount];
   ot_ip6 node_value;
   size_t scores[amount];
   size_t i;
-  char  *r = reply;
 
   memset(scores, 0, sizeof(scores));
   memset(networks, 0, sizeof(networks));
@@ -200,24 +229,21 @@ static size_t stats_return_busy_networks(char *reply, stats_network_node *tree,
 
   stats_get_highscore_networks(tree, 0, node_value, scores, networks, amount, limit);
 
-  r += sprintf(r, "Networks, limit /%d:\n", limit + STATS_NETWORK_NODE_BITWIDTH);
+  fmt_append(reply, reply_end, "Networks, limit /%d:\n", limit + STATS_NETWORK_NODE_BITWIDTH);
   for (i = amount; i > 0; --i) {
     if (scores[i - 1]) {
-      r += sprintf(r, "%08zd: ", scores[i - 1]);
-      // #ifdef WANT_V6
-      r += ip_format(r, networks[i - 1]);
-      *r++ = '\n';
+      char ip_readable[64];
+
+      ip_readable[ip_format(ip_readable, networks[i - 1])] = 0;
+      fmt_append(reply, reply_end, "%08zd: %s\n", scores[i - 1], ip_readable);
     }
   }
-  *r++ = '\n';
-
-  return r - reply;
+  fmt_append(reply, reply_end, "\n");
 }
 
-static size_t stats_slash24s_txt(char *reply, size_t amount) {
+static void stats_slash24s_txt(char **reply, const char *reply_end, size_t amount) {
   stats_network_node *slash24s_network_counters_root = NULL;
-  char  *r = reply;
-  size_t i, bucket, peer_size = OT_PEER_SIZE4;
+  size_t              i, bucket, peer_size = OT_PEER_SIZE4;
 
   for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) {
     ot_vector *torrents_list = mutex_bucket_lock(bucket);
@@ -244,35 +270,28 @@ static size_t stats_slash24s_txt(char *reply, size_t amount) {
     }
     mutex_bucket_unlock(bucket, 0);
     if (!g_opentracker_running)
-      goto bailout_error;
+      goto done;
   }
 
   /* The tree is built. Now analyze */
-  r += stats_return_busy_networks(r, slash24s_network_counters_root, amount, STATS_NETWORK_NODE_MAXDEPTH);
-  r += stats_return_busy_networks(r, slash24s_network_counters_root, amount, STATS_NETWORK_NODE_LIMIT);
-  goto success;
+  stats_return_busy_networks(reply, reply_end, slash24s_network_counters_root, amount, STATS_NETWORK_NODE_MAXDEPTH);
+  stats_return_busy_networks(reply, reply_end, slash24s_network_counters_root, amount, STATS_NETWORK_NODE_LIMIT);
+  goto done;
 
 bailout_unlock:
   mutex_bucket_unlock(bucket, 0);
-bailout_error:
-  r = reply;
-success:
+done:
   stats_shift_down_network_count(&slash24s_network_counters_root, 0, sizeof(int) * 8 - 1);
-
-  return r - reply;
 }
 
 #ifdef WANT_SPOT_WOODPECKER
 static stats_network_node *stats_woodpeckers_tree;
 static pthread_mutex_t     g_woodpeckers_mutex = PTHREAD_MUTEX_INITIALIZER;
 
-static size_t              stats_return_woodpeckers(char *reply, int amount) {
-  char *r = reply;
-
+static void              stats_return_woodpeckers(char **reply, const char *reply_end, int amount) {
   pthread_mutex_lock(&g_woodpeckers_mutex);
-  r += stats_return_busy_networks(r, stats_woodpeckers_tree, amount, STATS_NETWORK_NODE_MAXDEPTH);
+  stats_return_busy_networks(reply, reply_end, stats_woodpeckers_tree, amount, STATS_NETWORK_NODE_MAXDEPTH);
   pthread_mutex_unlock(&g_woodpeckers_mutex);
-  return r - reply;
 }
 #endif
 
@@ -312,10 +331,10 @@ static inline void stats_insert_into_highscore(size_t val, ot_record* list, size
 
 
 /* Fetches stats from tracker */
-size_t stats_top_txt(char *reply, size_t amount) {
+void stats_top_txt(char **reply, const char *reply_end, size_t amount) {
   size_t    j, idx, bucket;
   ot_record top100s[100], top100c[100], top100l[100];
-  char     *r = reply, hex_out[42];
+  char      hex_out[42];
 
   if (amount > 100)
     amount = 100;
@@ -326,7 +345,7 @@ size_t stats_top_txt(char *reply, size_t amount) {
 
   /* Iterate over complete torrent list */
   for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) {
-    ot_vector *torrents_list  = mutex_bucket_lock(bucket);
+    ot_vector *torrents_list = mutex_bucket_lock(bucket);
     for (j = 0; j < torrents_list->size; ++j) {
       ot_torrent *torrent     = (ot_torrent *)(torrents_list->data) + j;
       size_t      peer_count  = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count;
@@ -339,114 +358,148 @@ size_t stats_top_txt(char *reply, size_t amount) {
     }
     mutex_bucket_unlock(bucket, 0);
     if (!g_opentracker_running)
-      return 0;
+      return;
   }
 
-  r += sprintf(r, "Top %zd torrents by peers:\n", amount);
+  fmt_append(reply, reply_end, "Top %zd torrents by peers:\n", amount);
   for (idx = 0; idx < amount; ++idx)
     if (top100c[idx].val)
-      r += sprintf(r, "\t%zd\t%s\n", top100c[idx].val, hex_fromhash(hex_out, top100c[idx].hash));
-  r += sprintf(r, "Top %zd torrents by seeds:\n", amount);
+      fmt_append(reply, reply_end, "\t%zd\t%s\n", top100c[idx].val, hex_from_hash(hex_out, top100c[idx].hash));
+  fmt_append(reply, reply_end, "Top %zd torrents by seeds:\n", amount);
   for (idx = 0; idx < amount; ++idx)
     if (top100s[idx].val)
-      r += sprintf(r, "\t%zd\t%s\n", top100s[idx].val, hex_fromhash(hex_out, top100s[idx].hash));
-  r += sprintf(r, "Top %zd torrents by leechers:\n", amount);
+      fmt_append(reply, reply_end, "\t%zd\t%s\n", top100s[idx].val, hex_from_hash(hex_out, top100s[idx].hash));
+  fmt_append(reply, reply_end, "Top %zd torrents by leechers:\n", amount);
   for (idx = 0; idx < amount; ++idx)
     if (top100l[idx].val)
-      r += sprintf(r, "\t%zd\t%s\n", top100l[idx].val, hex_fromhash(hex_out, top100l[idx].hash));
-
-  return r - reply;
+      fmt_append(reply, reply_end, "\t%zd\t%s\n", top100l[idx].val, hex_from_hash(hex_out, top100l[idx].hash));
 }
 
 static unsigned long events_per_time(unsigned long long events, time_t t) { return events / ((unsigned int)t ? (unsigned int)t : 1); }
 
-static size_t        stats_connections_mrtg(char *reply) {
+static void        stats_connections_mrtg(char **reply, const char *reply_end) {
   ot_time t = time(NULL) - ot_start_time;
-  return sprintf(reply, "%llu\n%llu\n%i seconds (%i hours)\nopentracker connections, %lu conns/s :: %lu success/s.",
-                        ot_overall_tcp_connections + ot_overall_udp_connections,
-                        ot_overall_tcp_successfulannounces + ot_overall_udp_successfulannounces + ot_overall_udp_connects, (int)t, (int)(t / 3600),
-                        events_per_time(ot_overall_tcp_connections + ot_overall_udp_connections, t),
-                        events_per_time(ot_overall_tcp_successfulannounces + ot_overall_udp_successfulannounces + ot_overall_udp_connects, t));
+  unsigned long long total[OT_COUNTER_COUNT];
+
+  counters_total(total);
+
+  fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker connections, %lu conns/s :: %lu success/s.",
+                    total[OT_COUNTER_TCP_CONNECTIONS] + total[OT_COUNTER_UDP_CONNECTIONS],
+                    total[OT_COUNTER_TCP_ANNOUNCES] + total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], (int)t, (int)(t / 3600),
+                    events_per_time(total[OT_COUNTER_TCP_CONNECTIONS] + total[OT_COUNTER_UDP_CONNECTIONS], t),
+                    events_per_time(total[OT_COUNTER_TCP_ANNOUNCES] + total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], t));
 }
 
-static size_t stats_udpconnections_mrtg(char *reply) {
+static void stats_udpconnections_mrtg(char **reply, const char *reply_end) {
   ot_time t = time(NULL) - ot_start_time;
-  return sprintf(reply, "%llu\n%llu\n%i seconds (%i hours)\nopentracker udp4 stats, %lu conns/s :: %lu success/s.", ot_overall_udp_connections,
-                 ot_overall_udp_successfulannounces + ot_overall_udp_connects, (int)t, (int)(t / 3600), events_per_time(ot_overall_udp_connections, t),
-                 events_per_time(ot_overall_udp_successfulannounces + ot_overall_udp_connects, t));
+  unsigned long long total[OT_COUNTER_COUNT];
+
+  counters_total(total);
+
+  fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker udp4 stats, %lu conns/s :: %lu success/s.", total[OT_COUNTER_UDP_CONNECTIONS],
+                    total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], (int)t, (int)(t / 3600),
+                    events_per_time(total[OT_COUNTER_UDP_CONNECTIONS], t),
+                    events_per_time(total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], t));
 }
 
-static size_t stats_tcpconnections_mrtg(char *reply) {
+static void stats_tcpconnections_mrtg(char **reply, const char *reply_end) {
   time_t t = time(NULL) - ot_start_time;
-  return sprintf(reply, "%llu\n%llu\n%i seconds (%i hours)\nopentracker tcp4 stats, %lu conns/s :: %lu success/s.", ot_overall_tcp_connections,
-                 ot_overall_tcp_successfulannounces, (int)t, (int)(t / 3600), events_per_time(ot_overall_tcp_connections, t),
-                 events_per_time(ot_overall_tcp_successfulannounces, t));
+  unsigned long long total[OT_COUNTER_COUNT];
+
+  counters_total(total);
+
+  fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker tcp4 stats, %lu conns/s :: %lu success/s.", total[OT_COUNTER_TCP_CONNECTIONS],
+                    total[OT_COUNTER_TCP_ANNOUNCES], (int)t, (int)(t / 3600), events_per_time(total[OT_COUNTER_TCP_CONNECTIONS], t),
+                    events_per_time(total[OT_COUNTER_TCP_ANNOUNCES], t));
 }
 
-static size_t stats_scrape_mrtg(char *reply) {
+static void stats_scrape_mrtg(char **reply, const char *reply_end) {
   time_t t = time(NULL) - ot_start_time;
-  return sprintf(reply, "%llu\n%llu\n%i seconds (%i hours)\nopentracker scrape stats, %lu scrape/s (tcp and udp)", ot_overall_tcp_successfulscrapes,
-                 ot_overall_udp_successfulscrapes, (int)t, (int)(t / 3600),
-                 events_per_time((ot_overall_tcp_successfulscrapes + ot_overall_udp_successfulscrapes), t));
+  unsigned long long total[OT_COUNTER_COUNT];
+
+  counters_total(total);
+
+  fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker scrape stats, %lu scrape/s (tcp and udp)",
+                    total[OT_COUNTER_TCP_SCRAPES], total[OT_COUNTER_UDP_SCRAPES], (int)t, (int)(t / 3600),
+                    events_per_time((total[OT_COUNTER_TCP_SCRAPES] + total[OT_COUNTER_UDP_SCRAPES]), t));
 }
 
-static size_t stats_fullscrapes_mrtg(char *reply) {
+static void stats_fullscrapes_mrtg(char **reply, const char *reply_end) {
   ot_time t = time(NULL) - ot_start_time;
-  return sprintf(reply, "%llu\n%llu\n%i seconds (%i hours)\nopentracker full scrape stats, %lu conns/s :: %lu bytes/s.", ot_full_scrape_count * 1000,
-                 ot_full_scrape_size, (int)t, (int)(t / 3600), events_per_time(ot_full_scrape_count, t), events_per_time(ot_full_scrape_size, t));
+  unsigned long long total[OT_COUNTER_COUNT];
+
+  counters_total(total);
+
+  fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker full scrape stats, %lu conns/s :: %lu bytes/s.",
+                    total[OT_COUNTER_FULLSCRAPES] * 1000, total[OT_COUNTER_FULLSCRAPE_BYTES], (int)t, (int)(t / 3600),
+                    events_per_time(total[OT_COUNTER_FULLSCRAPES], t),
+                    events_per_time(total[OT_COUNTER_FULLSCRAPE_BYTES], t));
 }
 
-static size_t stats_peers_mrtg(char *reply) {
+static void stats_peers_mrtg(char **reply, const char *reply_end) {
   torrent_stats stats = {0, 0, 0};
 
   iterate_all_torrents(torrent_statter, (uintptr_t)&stats);
 
-  return sprintf(reply, "%llu\n%llu\nopentracker serving %llu torrents\nopentracker", stats.peer_count, stats.seed_count, stats.torrent_count);
+  fmt_append(reply, reply_end, "%llu\n%llu\nopentracker serving %llu torrents\nopentracker", stats.peer_count, stats.seed_count,
+                    stats.torrent_count);
 }
 
-static size_t stats_torrents_mrtg(char *reply) {
+static void stats_torrents_mrtg(char **reply, const char *reply_end) {
   size_t torrent_count = mutex_get_torrent_count();
 
-  return sprintf(reply, "%zd\n%zd\nopentracker serving %zd torrents\nopentracker", torrent_count, (size_t)0, torrent_count);
+  fmt_append(reply, reply_end, "%zd\n%zd\nopentracker serving %zd torrents\nopentracker", torrent_count, (size_t)0, torrent_count);
 }
 
-static size_t stats_httperrors_txt(char *reply) {
-  return sprintf(reply, "302 RED %llu\n400 ... %llu\n400 PAR %llu\n400 COM %llu\n403 IP  %llu\n404 INV %llu\n500 SRV %llu\n", ot_failed_request_counts[0],
-                 ot_failed_request_counts[1], ot_failed_request_counts[2], ot_failed_request_counts[3], ot_failed_request_counts[4],
-                 ot_failed_request_counts[5], ot_failed_request_counts[6]);
+static void stats_httperrors_txt(char **reply, const char *reply_end) {
+  unsigned long long total[OT_COUNTER_COUNT];
+
+  counters_total(total);
+
+  fmt_append(reply, reply_end, "302 RED %llu\n400 ... %llu\n400 PAR %llu\n400 COM %llu\n403 IP  %llu\n404 INV %llu\n500 SRV %llu\n",
+                    total[OT_COUNTER_FAILED + 0], total[OT_COUNTER_FAILED + 1], total[OT_COUNTER_FAILED + 2], total[OT_COUNTER_FAILED + 3],
+                    total[OT_COUNTER_FAILED + 4], total[OT_COUNTER_FAILED + 5], total[OT_COUNTER_FAILED + 6]);
 }
 
-static size_t stats_return_renew_bucket(char *reply) {
-  char *r = reply;
-  int   i;
+static void stats_return_renew_bucket(char **reply, const char *reply_end) {
+  unsigned long long total[OT_COUNTER_COUNT];
+
+  counters_total(total);
+
+  int i;
 
   for (i = 0; i < OT_PEER_TIMEOUT; ++i)
-    r += sprintf(r, "%02i %llu\n", i, ot_renewed[i]);
-  return r - reply;
+    fmt_append(reply, reply_end, "%02i %llu\n", i, total[OT_COUNTER_RENEWED + i]);
 }
 
-static size_t stats_return_sync_mrtg(char *reply) {
+static void stats_return_sync_mrtg(char **reply, const char *reply_end) {
   ot_time t = time(NULL) - ot_start_time;
-  return sprintf(reply, "%llu\n%llu\n%i seconds (%i hours)\nopentracker connections, %lu conns/s :: %lu success/s.", ot_overall_sync_count, 0LL, (int)t,
-                 (int)(t / 3600), events_per_time(ot_overall_tcp_connections + ot_overall_udp_connections, t),
-                 events_per_time(ot_overall_tcp_successfulannounces + ot_overall_udp_successfulannounces + ot_overall_udp_connects, t));
+  unsigned long long total[OT_COUNTER_COUNT];
+
+  counters_total(total);
+
+  fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker connections, %lu conns/s :: %lu success/s.", total[OT_COUNTER_SYNCS], 0LL,
+                    (int)t, (int)(t / 3600), events_per_time(total[OT_COUNTER_TCP_CONNECTIONS] + total[OT_COUNTER_UDP_CONNECTIONS], t),
+                    events_per_time(total[OT_COUNTER_TCP_ANNOUNCES] + total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], t));
 }
 
-static size_t stats_return_completed_mrtg(char *reply) {
+static void stats_return_completed_mrtg(char **reply, const char *reply_end) {
   ot_time t = time(NULL) - ot_start_time;
+  unsigned long long total[OT_COUNTER_COUNT];
+
+  counters_total(total);
 
-  return sprintf(reply, "%llu\n%llu\n%i seconds (%i hours)\nopentracker, %lu completed/h.", ot_overall_completed, 0LL, (int)t, (int)(t / 3600),
-                 events_per_time(ot_overall_completed, t / 3600));
+
+  fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker, %lu completed/h.", total[OT_COUNTER_COMPLETED], 0LL, (int)t, (int)(t / 3600),
+                    events_per_time(total[OT_COUNTER_COMPLETED], t / 3600));
 }
 
 #ifdef WANT_LOG_NUMWANT
 extern unsigned long long numwants[201];
-static size_t             stats_return_numwants(char *reply) {
-  char *r = reply;
-  int   i;
+static void             stats_return_numwants(char **reply, const char *reply_end) {
+  int i;
   for (i = 0; i <= 200; ++i)
-    r += sprintf(r, "%03d => %lld\n", i, numwants[i]);
-  return r - reply;
+    fmt_append(reply, reply_end, "%03d => %lld\n", i, numwants[i]);
 }
 #endif
 
@@ -459,148 +512,162 @@ static void stats_return_fulllog(size_t *iovec_entries, struct iovec **iovector,
 
   while (loglist) {
     if (r + (loglist->size + 64) >= re) {
-      r = iovec_fix_increase_or_free(iovec_entries, iovector, r, 32 * OT_STATS_TMPSIZE);
+      r = iovec_next_chunk(iovec_entries, iovector, r, 32 * OT_STATS_TMPSIZE);
       if (!r)
         return;
       re = r + 32 * OT_STATS_TMPSIZE;
     }
-    r      += sprintf(r, "%08ld: ", loglist->time);
-    r      += ip_format(r, loglist->ip);
-    *r++    = '\n';
-    memcpy(r, loglist->data, loglist->size);
-    r      += loglist->size;
-    *r++    = '\n';
-    *r++    = '*';
-    *r++    = '\n';
-    *r++    = '\n';
+    char ip_readable[64];
+
+    ip_readable[ip_format(ip_readable, loglist->ip)] = 0;
+    fmt_append(&r, re, "%08ld: %s\n", loglist->time, ip_readable);
+    fmt_bytes(&r, re, loglist->data, loglist->size);
+    fmt_append(&r, re, "\n*\n\n");
 
-    llnext  = loglist->next;
+    llnext = loglist->next;
     free(loglist->data);
     free(loglist);
     loglist = llnext;
   }
-  iovec_fixlast(iovec_entries, iovector, r);
+  iovec_trim_last(iovec_entries, iovector, r);
 }
 #endif
 
-static size_t stats_return_everything(char *reply) {
+static void stats_return_everything(char **reply, const char *reply_end) {
+  unsigned long long total[OT_COUNTER_COUNT];
+
+  counters_total(total);
+
   torrent_stats stats = {0, 0, 0};
   int           i;
-  char         *r = reply;
 
   iterate_all_torrents(torrent_statter, (uintptr_t)&stats);
 
-  r += sprintf(r, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
-  r += sprintf(r, "<stats>\n");
-  r += sprintf(r, "  <tracker_id>%" PRIu32 "</tracker_id>\n", g_tracker_id);
-  r += sprintf(r, "  <version>\n");
-  r += stats_return_tracker_version(r);
-  r += sprintf(r, "  </version>\n");
-  r += sprintf(r, "  <uptime>%llu</uptime>\n", (unsigned long long)(time(NULL) - ot_start_time));
-  r += sprintf(r, "  <torrents>\n");
-  r += sprintf(r, "    <count_mutex>%zd</count_mutex>\n", mutex_get_torrent_count());
-  r += sprintf(r, "    <count_iterator>%llu</count_iterator>\n", stats.torrent_count);
-  r += sprintf(r, "  </torrents>\n");
-  r += sprintf(r, "  <peers>\n    <count>%llu</count>\n  </peers>\n", stats.peer_count);
-  r += sprintf(r, "  <seeds>\n    <count>%llu</count>\n  </seeds>\n", stats.seed_count);
-  r += sprintf(r, "  <completed>\n    <count>%llu</count>\n  </completed>\n", ot_overall_completed);
-  r += sprintf(r, "  <connections>\n");
-  r += sprintf(r, "    <tcp>\n      <accept>%llu</accept>\n      <announce>%llu</announce>\n      <scrape>%llu</scrape>\n    </tcp>\n",
-      ot_overall_tcp_connections, ot_overall_tcp_successfulannounces, ot_overall_tcp_successfulscrapes);
-  r += sprintf(r, "    <udp>\n      <overall>%llu</overall>\n      <connect>%llu</connect>\n      <announce>%llu</announce>\n      <scrape>%llu</scrape>\n      <missmatch>%llu</missmatch>\n    </udp>\n",
-      ot_overall_udp_connections, ot_overall_udp_connects, ot_overall_udp_successfulannounces, ot_overall_udp_successfulscrapes,
-      ot_overall_udp_connectionidmissmatches);
-  r += sprintf(r, "    <livesync>\n      <count>%llu</count>\n    </livesync>\n", ot_overall_sync_count);
-  r += sprintf(r, "  </connections>\n");
-  r += sprintf(r, "  <debug>\n");
-  r += sprintf(r, "    <renew>\n");
+  fmt_append(reply, reply_end, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+  fmt_append(reply, reply_end, "<stats>\n");
+  fmt_append(reply, reply_end, "  <tracker_id>%" PRIu32 "</tracker_id>\n", g_tracker_id);
+  fmt_append(reply, reply_end, "  <version>\n");
+  stats_return_tracker_version(reply, reply_end);
+  fmt_append(reply, reply_end, "  </version>\n");
+  fmt_append(reply, reply_end, "  <uptime>%llu</uptime>\n", (unsigned long long)(time(NULL) - ot_start_time));
+  fmt_append(reply, reply_end, "  <torrents>\n");
+  fmt_append(reply, reply_end, "    <count_mutex>%zd</count_mutex>\n", mutex_get_torrent_count());
+  fmt_append(reply, reply_end, "    <count_iterator>%llu</count_iterator>\n", stats.torrent_count);
+  fmt_append(reply, reply_end, "  </torrents>\n");
+  fmt_append(reply, reply_end, "  <peers>\n    <count>%llu</count>\n  </peers>\n", stats.peer_count);
+  fmt_append(reply, reply_end, "  <seeds>\n    <count>%llu</count>\n  </seeds>\n", stats.seed_count);
+  fmt_append(reply, reply_end, "  <completed>\n    <count>%llu</count>\n  </completed>\n", total[OT_COUNTER_COMPLETED]);
+  fmt_append(reply, reply_end, "  <connections>\n");
+  fmt_append(reply, reply_end, "    <tcp>\n      <accept>%llu</accept>\n      <announce>%llu</announce>\n      <scrape>%llu</scrape>\n    </tcp>\n",
+             total[OT_COUNTER_TCP_CONNECTIONS], total[OT_COUNTER_TCP_ANNOUNCES], total[OT_COUNTER_TCP_SCRAPES]);
+  fmt_append(reply, reply_end, "    <udp>\n      <overall>%llu</overall>\n      <connect>%llu</connect>\n      <announce>%llu</announce>\n      <scrape>%llu</scrape>\n      <missmatch>%llu</missmatch>\n    </udp>\n",
+             total[OT_COUNTER_UDP_CONNECTIONS], total[OT_COUNTER_UDP_CONNECTS], total[OT_COUNTER_UDP_ANNOUNCES], total[OT_COUNTER_UDP_SCRAPES],
+             total[OT_COUNTER_UDP_CONNID_MISSMATCH]);
+  fmt_append(reply, reply_end, "    <livesync>\n      <count>%llu</count>\n    </livesync>\n", total[OT_COUNTER_SYNCS]);
+  fmt_append(reply, reply_end, "  </connections>\n");
+  fmt_append(reply, reply_end, "  <debug>\n");
+  fmt_append(reply, reply_end, "    <renew>\n");
   for (i = 0; i < OT_PEER_TIMEOUT; ++i)
-    r += sprintf(r, "      <count interval=\"%02i\">%llu</count>\n", i, ot_renewed[i]);
-  r += sprintf(r, "    </renew>\n");
-  r += sprintf(r, "    <http_error>\n");
+    fmt_append(reply, reply_end, "      <count interval=\"%02i\">%llu</count>\n", i, total[OT_COUNTER_RENEWED + i]);
+  fmt_append(reply, reply_end, "    </renew>\n");
+  fmt_append(reply, reply_end, "    <http_error>\n");
   for (i = 0; i < CODE_HTTPERROR_COUNT; ++i)
-    r += sprintf(r, "      <count code=\"%s\">%llu</count>\n", ot_failed_request_names[i], ot_failed_request_counts[i]);
-  r += sprintf(r, "    </http_error>\n");
-  r += sprintf(r, "    <mutex_stall>\n      <count>%llu</count>\n    </mutex_stall>\n", ot_overall_stall_count);
-  r += sprintf(r, "  </debug>\n");
-  r += sprintf(r, "</stats>");
-  return r - reply;
+    fmt_append(reply, reply_end, "      <count code=\"%s\">%llu</count>\n", ot_failed_request_names[i], total[OT_COUNTER_FAILED + i]);
+  fmt_append(reply, reply_end, "    </http_error>\n");
+  fmt_append(reply, reply_end, "    <mutex_stall>\n      <count>%llu</count>\n    </mutex_stall>\n", total[OT_COUNTER_STALLS]);
+  fmt_append(reply, reply_end, "  </debug>\n");
+  fmt_append(reply, reply_end, "</stats>");
 }
 
-size_t stats_return_tracker_version(char *reply) {
+void stats_return_tracker_version(char **reply, const char *reply_end) {
 #define QUOTE(name) #name
 #define SQUOTE(name) QUOTE(name)
-  return sprintf(reply, "https://erdgeist.org/gitweb/opentracker/commit/?id=" SQUOTE(GIT_VERSION) "\n");
+  fmt_append(reply, reply_end, "https://erdgeist.org/gitweb/opentracker/commit/?id=" SQUOTE(GIT_VERSION) "\n");
 }
 
-size_t return_stats_for_tracker(char *reply, int mode, int format) {
+void return_stats_for_tracker(char **reply, const char *reply_end, int mode, int format) {
   (void)format;
   switch (mode & TASK_TASK_MASK) {
   case TASK_STATS_CONNS:
-    return stats_connections_mrtg(reply);
+    stats_connections_mrtg(reply, reply_end);
+    break;
   case TASK_STATS_SCRAPE:
-    return stats_scrape_mrtg(reply);
+    stats_scrape_mrtg(reply, reply_end);
+    break;
   case TASK_STATS_UDP:
-    return stats_udpconnections_mrtg(reply);
+    stats_udpconnections_mrtg(reply, reply_end);
+    break;
   case TASK_STATS_TCP:
-    return stats_tcpconnections_mrtg(reply);
+    stats_tcpconnections_mrtg(reply, reply_end);
+    break;
   case TASK_STATS_FULLSCRAPE:
-    return stats_fullscrapes_mrtg(reply);
+    stats_fullscrapes_mrtg(reply, reply_end);
+    break;
   case TASK_STATS_COMPLETED:
-    return stats_return_completed_mrtg(reply);
+    stats_return_completed_mrtg(reply, reply_end);
+    break;
   case TASK_STATS_HTTPERRORS:
-    return stats_httperrors_txt(reply);
+    stats_httperrors_txt(reply, reply_end);
+    break;
   case TASK_STATS_VERSION:
-    return stats_return_tracker_version(reply);
+    stats_return_tracker_version(reply, reply_end);
+    break;
   case TASK_STATS_RENEW:
-    return stats_return_renew_bucket(reply);
+    stats_return_renew_bucket(reply, reply_end);
+    break;
   case TASK_STATS_SYNCS:
-    return stats_return_sync_mrtg(reply);
+    stats_return_sync_mrtg(reply, reply_end);
+    break;
 #ifdef WANT_LOG_NUMWANT
   case TASK_STATS_NUMWANTS:
-    return stats_return_numwants(reply);
+    stats_return_numwants(reply, reply_end);
+    break;
 #endif
   default:
-    return 0;
+    break;
   }
 }
 
 static void stats_make(size_t *iovec_entries, struct iovec **iovector, ot_tasktype mode) {
-  char *r;
+  char       *r;
+  const char *r_end;
 
   *iovec_entries = 0;
   *iovector      = NULL;
-  if (!(r = iovec_increase(iovec_entries, iovector, OT_STATS_TMPSIZE)))
+  if (!(r = iovec_new_chunk(iovec_entries, iovector, OT_STATS_TMPSIZE)))
     return;
+  r_end = r + OT_STATS_TMPSIZE;
 
   switch (mode & TASK_TASK_MASK) {
   case TASK_STATS_TORRENTS:
-    r += stats_torrents_mrtg(r);
+    stats_torrents_mrtg(&r, r_end);
     break;
   case TASK_STATS_PEERS:
-    r += stats_peers_mrtg(r);
+    stats_peers_mrtg(&r, r_end);
     break;
   case TASK_STATS_SLASH24S:
-    r += stats_slash24s_txt(r, 128);
+    stats_slash24s_txt(&r, r_end, 128);
     break;
   case TASK_STATS_TOP10:
-    r += stats_top_txt(r, 10);
+    stats_top_txt(&r, r_end, 10);
     break;
   case TASK_STATS_TOP100:
-    r = iovec_fix_increase_or_free(iovec_entries, iovector, r, 4 * OT_STATS_TMPSIZE);
+    r = iovec_next_chunk(iovec_entries, iovector, r, 4 * OT_STATS_TMPSIZE);
     if (!r)
       return;
-    r += stats_top_txt(r, 100);
+    r_end = r + 4 * OT_STATS_TMPSIZE;
+    stats_top_txt(&r, r_end, 100);
     break;
   case TASK_STATS_EVERYTHING:
-    r = iovec_fix_increase_or_free(iovec_entries, iovector, r, OT_STATS_TMPSIZE + 64 * OT_PEER_TIMEOUT);
+    r = iovec_next_chunk(iovec_entries, iovector, r, OT_STATS_TMPSIZE + 64 * OT_PEER_TIMEOUT);
     if (!r)
       return;
-    r += stats_return_everything(r);
+    r_end = r + OT_STATS_TMPSIZE + 64 * OT_PEER_TIMEOUT;
+    stats_return_everything(&r, r_end);
     break;
 #ifdef WANT_SPOT_WOODPECKER
   case TASK_STATS_WOODPECKERS:
-    r += stats_return_woodpeckers(r, 128);
+    stats_return_woodpeckers(&r, r_end, 128);
     break;
 #endif
 #ifdef WANT_FULLLOG_NETWORKS
@@ -612,36 +679,38 @@ static void stats_make(size_t *iovec_entries, struct iovec **iovector, ot_taskty
     iovec_free(iovec_entries, iovector);
     return;
   }
-  iovec_fixlast(iovec_entries, iovector, r);
+  iovec_trim_last(iovec_entries, iovector, r);
 }
 
-void stats_issue_event(ot_status_event event, PROTO_FLAG proto, uintptr_t event_data) {
+void stats_issue_event(struct ot_workstruct *ws, ot_status_event event, PROTO_FLAG proto, uintptr_t event_data) {
+  if (!ws)
+    return;
+
   switch (event) {
   case EVENT_ACCEPT:
     if (proto == FLAG_TCP)
-      ot_overall_tcp_connections++;
+      COUNT(OT_COUNTER_TCP_CONNECTIONS, 1);
     else
-      ot_overall_udp_connections++;
+      COUNT(OT_COUNTER_UDP_CONNECTIONS, 1);
 #ifdef WANT_LOG_NETWORKS
     stat_increase_network_count(&stats_network_counters_root, 0, event_data);
 #endif
     break;
   case EVENT_ANNOUNCE:
     if (proto == FLAG_TCP)
-      ot_overall_tcp_successfulannounces++;
+      COUNT(OT_COUNTER_TCP_ANNOUNCES, 1);
     else
-      ot_overall_udp_successfulannounces++;
+      COUNT(OT_COUNTER_UDP_ANNOUNCES, 1);
     break;
   case EVENT_CONNECT:
     if (proto == FLAG_TCP)
-      ot_overall_tcp_connects++;
+      COUNT(OT_COUNTER_TCP_CONNECTS, 1);
     else
-      ot_overall_udp_connects++;
+      COUNT(OT_COUNTER_UDP_CONNECTS, 1);
     break;
   case EVENT_COMPLETED:
 #ifdef WANT_SYSLOGS
-    if (event_data) {
-      struct ot_workstruct *ws = (struct ot_workstruct *)event_data;
+    {
       char                  timestring[64];
       char                  hash_hex[42], peerid_hex[42], ip_readable[64];
       struct tm             time_now;
@@ -651,9 +720,9 @@ void stats_issue_event(ot_status_event event, PROTO_FLAG proto, uintptr_t event_
       localtime_r(&ttt, &time_now);
       strftime(timestring, sizeof(timestring), "%FT%T%z", &time_now);
 
-      hex_fromhash(hash_hex, *ws->hash);
+      hex_from_hash(hash_hex, *ws->hash);
       if (ws->peer_id)
-        hex_fromhash(peerid_hex, (uint8_t *)ws->peer_id);
+        hex_from_hash(peerid_hex, (uint8_t *)ws->peer_id);
       else {
         *peerid_hex = 0;
       }
@@ -662,47 +731,47 @@ void stats_issue_event(ot_status_event event, PROTO_FLAG proto, uintptr_t event_
       syslog(LOG_INFO, "time=%s event=completed info_hash=%s peer_id=%s ip=%s", timestring, hash_hex, peerid_hex, ip_readable);
     }
 #endif
-    ot_overall_completed++;
+    COUNT(OT_COUNTER_COMPLETED, 1);
     break;
   case EVENT_SCRAPE:
     if (proto == FLAG_TCP)
-      ot_overall_tcp_successfulscrapes++;
+      COUNT(OT_COUNTER_TCP_SCRAPES, 1);
     else
-      ot_overall_udp_successfulscrapes++;
+      COUNT(OT_COUNTER_UDP_SCRAPES, 1);
     break;
   case EVENT_FULLSCRAPE:
-    ot_full_scrape_count++;
-    ot_full_scrape_size += event_data;
+    COUNT(OT_COUNTER_FULLSCRAPES, 1);
+    COUNT(OT_COUNTER_FULLSCRAPE_BYTES, event_data);
     break;
   case EVENT_FULLSCRAPE_REQUEST: {
     ot_ip6 *ip = (ot_ip6 *)event_data; /* ugly hack to transfer ip to stats */
     char    _debug[512];
-    int     off  = snprintf(_debug, sizeof(_debug), "[%08d] scrp:  ", (unsigned int)(g_now_seconds - ot_start_time) / 60);
+    int     off  = snprintf(_debug, sizeof(_debug), "[%08d] scrp:  ", (unsigned int)(time_now_seconds() - ot_start_time) / 60);
     off         += ip_format(_debug + off, *ip);
     off         += snprintf(_debug + off, sizeof(_debug) - off, " - FULL SCRAPE\n");
     (void)write(2, _debug, off);
-    ot_full_scrape_request_count++;
+    COUNT(OT_COUNTER_FULLSCRAPE_REQUESTS, 1);
   } break;
   case EVENT_FULLSCRAPE_REQUEST_GZIP: {
     ot_ip6 *ip = (ot_ip6 *)event_data; /* ugly hack to transfer ip to stats */
     char    _debug[512];
-    int     off  = snprintf(_debug, sizeof(_debug), "[%08d] scrp:  ", (unsigned int)(g_now_seconds - ot_start_time) / 60);
+    int     off  = snprintf(_debug, sizeof(_debug), "[%08d] scrp:  ", (unsigned int)(time_now_seconds() - ot_start_time) / 60);
     off         += ip_format(_debug + off, *ip);
     off         += snprintf(_debug + off, sizeof(_debug) - off, " - FULL SCRAPE\n");
     (void)write(2, _debug, off);
-    ot_full_scrape_request_count++;
+    COUNT(OT_COUNTER_FULLSCRAPE_REQUESTS, 1);
   } break;
   case EVENT_FAILED:
-    ot_failed_request_counts[event_data]++;
+    COUNT(OT_COUNTER_FAILED + event_data, 1);
     break;
   case EVENT_RENEW:
-    ot_renewed[event_data]++;
+    COUNT(OT_COUNTER_RENEWED + event_data, 1);
     break;
   case EVENT_SYNC:
-    ot_overall_sync_count += event_data;
+    COUNT(OT_COUNTER_SYNCS, event_data);
     break;
   case EVENT_BUCKET_LOCKED:
-    ot_overall_stall_count++;
+    COUNT(OT_COUNTER_STALLS, 1);
     break;
 #ifdef WANT_SPOT_WOODPECKER
   case EVENT_WOODPECKER:
@@ -712,7 +781,7 @@ void stats_issue_event(ot_status_event event, PROTO_FLAG proto, uintptr_t event_
     break;
 #endif
   case EVENT_CONNID_MISSMATCH:
-    ++ot_overall_udp_connectionidmissmatches;
+    COUNT(OT_COUNTER_UDP_CONNID_MISSMATCH, 1);
   default:
     break;
   }
@@ -732,9 +801,11 @@ static void *stats_worker(void *args) {
 
   (void)args;
 
-  while (1) {
+  while (g_opentracker_running) {
     ot_tasktype tasktype = TASK_STATS;
     ot_taskid   taskid   = mutex_workqueue_poptask(&tasktype);
+    if (!taskid)
+      break;
     stats_make(&iovec_entries, &iovector, tasktype);
     if (mutex_workqueue_pushresult(taskid, iovec_entries, iovector))
       iovec_free(&iovec_entries, &iovector);
@@ -746,10 +817,11 @@ void stats_deliver(int64_t sock, int tasktype) { mutex_workqueue_pushtask(sock,
 
 static pthread_t thread_id;
 void stats_init(void) {
-  ot_start_time = g_now_seconds;
+  ot_start_time = time_now_seconds();
   pthread_create(&thread_id, NULL, stats_worker, NULL);
 }
 
 void stats_deinit(void) {
-    pthread_cancel(thread_id);
+  mutex_workqueue_wakeall();
+  pthread_join(thread_id, NULL);
 }