diff options
Diffstat (limited to 'ot_fullscrape.c')
| -rw-r--r-- | ot_fullscrape.c | 53 |
1 files changed, 29 insertions, 24 deletions
diff --git a/ot_fullscrape.c b/ot_fullscrape.c index 788e574..8518bd2 100644 --- a/ot_fullscrape.c +++ b/ot_fullscrape.c | |||
| @@ -17,15 +17,13 @@ | |||
| 17 | #include <zstd.h> | 17 | #include <zstd.h> |
| 18 | #endif | 18 | #endif |
| 19 | 19 | ||
| 20 | /* Libowfat */ | ||
| 21 | #include "io.h" | ||
| 22 | |||
| 23 | /* Opentracker */ | 20 | /* Opentracker */ |
| 21 | #include "ot_fmt.h" | ||
| 24 | #include "ot_hex.h" | 22 | #include "ot_hex.h" |
| 25 | #include "ot_fullscrape.h" | 23 | #include "ot_fullscrape.h" |
| 26 | #include "ot_iovec.h" | 24 | #include "ot_iovec.h" |
| 27 | #include "ot_mutex.h" | 25 | #include "ot_mutex.h" |
| 28 | #include "scan_urlencoded_query.h" | 26 | #include "ot_urlencoded.h" |
| 29 | #include "trackerlogic.h" | 27 | #include "trackerlogic.h" |
| 30 | 28 | ||
| 31 | /* Fetch full scrape info for all torrents | 29 | /* Fetch full scrape info for all torrents |
| @@ -86,10 +84,15 @@ void fullscrape_deliver( int64_t sock, ot_tasktype tasktype ) { | |||
| 86 | mutex_workqueue_pushtask( sock, TASK_FULLSCRAPE | tasktype ); | 84 | mutex_workqueue_pushtask( sock, TASK_FULLSCRAPE | tasktype ); |
| 87 | } | 85 | } |
| 88 | 86 | ||
| 89 | static char * fullscrape_write_one( ot_tasktype mode, char *r, ot_torrent *torrent, ot_hash *hash ) { | 87 | static void fullscrape_write_one( ot_tasktype mode, char **cursor, ot_torrent *torrent, ot_hash *hash ) { |
| 90 | size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; | 88 | /* Callers guarantee this much room: the chunk writer keeps its low |
| 91 | size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; | 89 | watermark that far from the end, the compressors hand over a buffer of |
| 92 | size_t down_count = torrent->peer_list6->down_count + torrent->peer_list4->down_count; | 90 | exactly that size */ |
| 91 | char *r = *cursor; | ||
| 92 | const char *r_end = r + OT_SCRAPE_MAXENTRYLEN; | ||
| 93 | size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; | ||
| 94 | size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; | ||
| 95 | size_t down_count = torrent->peer_list6->down_count + torrent->peer_list4->down_count; | ||
| 93 | 96 | ||
| 94 | switch (mode & TASK_TASK_MASK) { | 97 | switch (mode & TASK_TASK_MASK) { |
| 95 | case TASK_FULLSCRAPE: | 98 | case TASK_FULLSCRAPE: |
| @@ -101,18 +104,18 @@ static char * fullscrape_write_one( ot_tasktype mode, char *r, ot_torrent *torre | |||
| 101 | memcpy(r, hash, sizeof(ot_hash)); | 104 | memcpy(r, hash, sizeof(ot_hash)); |
| 102 | r += sizeof(ot_hash); | 105 | r += sizeof(ot_hash); |
| 103 | /* push rest of the scrape string */ | 106 | /* push rest of the scrape string */ |
| 104 | r += sprintf(r, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", seed_count, down_count, peer_count - seed_count); | 107 | fmt_append(&r, r_end, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", seed_count, down_count, peer_count - seed_count); |
| 105 | 108 | ||
| 106 | break; | 109 | break; |
| 107 | case TASK_FULLSCRAPE_TPB_ASCII: | 110 | case TASK_FULLSCRAPE_TPB_ASCII: |
| 108 | hex_fromhash(r, *hash); | 111 | hex_from_hash(r, *hash); |
| 109 | r += 2 * sizeof(ot_hash); | 112 | r += 2 * sizeof(ot_hash); |
| 110 | r += sprintf(r, ":%zd:%zd\n", seed_count, peer_count - seed_count); | 113 | fmt_append(&r, r_end, ":%zd:%zd\n", seed_count, peer_count - seed_count); |
| 111 | break; | 114 | break; |
| 112 | case TASK_FULLSCRAPE_TPB_ASCII_PLUS: | 115 | case TASK_FULLSCRAPE_TPB_ASCII_PLUS: |
| 113 | hex_fromhash(r, *hash); | 116 | hex_from_hash(r, *hash); |
| 114 | r += 2 * sizeof(ot_hash); | 117 | r += 2 * sizeof(ot_hash); |
| 115 | r += sprintf(r, ":%zd:%zd:%zd\n", seed_count, peer_count - seed_count, down_count); | 118 | fmt_append(&r, r_end, ":%zd:%zd:%zd\n", seed_count, peer_count - seed_count, down_count); |
| 116 | break; | 119 | break; |
| 117 | case TASK_FULLSCRAPE_TPB_BINARY: | 120 | case TASK_FULLSCRAPE_TPB_BINARY: |
| 118 | memcpy(r, *hash, sizeof(ot_hash)); | 121 | memcpy(r, *hash, sizeof(ot_hash)); |
| @@ -123,15 +126,15 @@ static char * fullscrape_write_one( ot_tasktype mode, char *r, ot_torrent *torre | |||
| 123 | break; | 126 | break; |
| 124 | case TASK_FULLSCRAPE_TPB_URLENCODED: | 127 | case TASK_FULLSCRAPE_TPB_URLENCODED: |
| 125 | r += urlencoded_format(r, *hash, sizeof(ot_hash)); | 128 | r += urlencoded_format(r, *hash, sizeof(ot_hash)); |
| 126 | r += sprintf(r, ":%zd:%zd\n", seed_count, peer_count - seed_count); | 129 | fmt_append(&r, r_end, ":%zd:%zd\n", seed_count, peer_count - seed_count); |
| 127 | break; | 130 | break; |
| 128 | case TASK_FULLSCRAPE_TRACKERSTATE: | 131 | case TASK_FULLSCRAPE_TRACKERSTATE: |
| 129 | hex_fromhash(r, *hash); | 132 | hex_from_hash(r, *hash); |
| 130 | r += 2 * sizeof(ot_hash); | 133 | r += 2 * sizeof(ot_hash); |
| 131 | r += sprintf(r, ":%zd:%zd\n", torrent->peer_list6->base, down_count); | 134 | fmt_append(&r, r_end, ":%zd:%zd\n", torrent->peer_list6->base, down_count); |
| 132 | break; | 135 | break; |
| 133 | } | 136 | } |
| 134 | return r; | 137 | *cursor = r; |
| 135 | } | 138 | } |
| 136 | 139 | ||
| 137 | static void fullscrape_make(int taskid, ot_tasktype mode) { | 140 | static void fullscrape_make(int taskid, ot_tasktype mode) { |
| @@ -148,7 +151,7 @@ static void fullscrape_make(int taskid, ot_tasktype mode) { | |||
| 148 | re = r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN; | 151 | re = r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN; |
| 149 | 152 | ||
| 150 | if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) | 153 | if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) |
| 151 | r += sprintf(r, "d5:filesd"); | 154 | fmt_append(&r, (char *)iovector.iov_base + OT_SCRAPE_CHUNK_SIZE, "d5:filesd"); |
| 152 | 155 | ||
| 153 | /* For each bucket... */ | 156 | /* For each bucket... */ |
| 154 | for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { | 157 | for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { |
| @@ -159,7 +162,7 @@ static void fullscrape_make(int taskid, ot_tasktype mode) { | |||
| 159 | 162 | ||
| 160 | /* For each torrent in this bucket.. */ | 163 | /* For each torrent in this bucket.. */ |
| 161 | for (i = 0; i < torrents_list->size; ++i) { | 164 | for (i = 0; i < torrents_list->size; ++i) { |
| 162 | r = fullscrape_write_one(mode, r, torrents + i, &torrents[i].hash); | 165 | fullscrape_write_one(mode, &r, torrents + i, &torrents[i].hash); |
| 163 | 166 | ||
| 164 | if (r > re) { | 167 | if (r > re) { |
| 165 | iovector.iov_len = r - (char *)iovector.iov_base; | 168 | iovector.iov_len = r - (char *)iovector.iov_base; |
| @@ -190,7 +193,7 @@ static void fullscrape_make(int taskid, ot_tasktype mode) { | |||
| 190 | } | 193 | } |
| 191 | 194 | ||
| 192 | if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) | 195 | if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) |
| 193 | r += sprintf(r, "ee"); | 196 | fmt_append(&r, (char *)iovector.iov_base + OT_SCRAPE_CHUNK_SIZE, "ee"); |
| 194 | 197 | ||
| 195 | /* Send rest of data */ | 198 | /* Send rest of data */ |
| 196 | iovector.iov_len = r - (char *)iovector.iov_base; | 199 | iovector.iov_len = r - (char *)iovector.iov_base; |
| @@ -233,7 +236,8 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) { | |||
| 233 | /* For each torrent in this bucket.. */ | 236 | /* For each torrent in this bucket.. */ |
| 234 | for (i = 0; i < torrents_list->size; ++i) { | 237 | for (i = 0; i < torrents_list->size; ++i) { |
| 235 | char compress_buffer[OT_SCRAPE_MAXENTRYLEN]; | 238 | char compress_buffer[OT_SCRAPE_MAXENTRYLEN]; |
| 236 | r = fullscrape_write_one(mode, compress_buffer, torrents + i, &torrents[i].hash); | 239 | r = compress_buffer; |
| 240 | fullscrape_write_one(mode, &r, torrents + i, &torrents[i].hash); | ||
| 237 | strm.next_in = (uint8_t *)compress_buffer; | 241 | strm.next_in = (uint8_t *)compress_buffer; |
| 238 | strm.avail_in = r - compress_buffer; | 242 | strm.avail_in = r - compress_buffer; |
| 239 | zres = deflate(&strm, Z_NO_FLUSH); | 243 | zres = deflate(&strm, Z_NO_FLUSH); |
| @@ -296,7 +300,7 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) { | |||
| 296 | iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); | 300 | iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); |
| 297 | 301 | ||
| 298 | if (!iovector.iov_base) { | 302 | if (!iovector.iov_base) { |
| 299 | fprintf(stderr, "Problem with iovec_fix_increase_or_free\n"); | 303 | fprintf(stderr, "Problem with iovec_next_chunk\n"); |
| 300 | deflateEnd(&strm); | 304 | deflateEnd(&strm); |
| 301 | return; | 305 | return; |
| 302 | } | 306 | } |
| @@ -362,7 +366,8 @@ static void fullscrape_make_zstd(int taskid, ot_tasktype mode) { | |||
| 362 | /* For each torrent in this bucket.. */ | 366 | /* For each torrent in this bucket.. */ |
| 363 | for (i = 0; i < torrents_list->size; ++i) { | 367 | for (i = 0; i < torrents_list->size; ++i) { |
| 364 | char compress_buffer[OT_SCRAPE_MAXENTRYLEN]; | 368 | char compress_buffer[OT_SCRAPE_MAXENTRYLEN]; |
| 365 | r = fullscrape_write_one(mode, compress_buffer, torrents + i, &torrents[i].hash); | 369 | r = compress_buffer; |
| 370 | fullscrape_write_one(mode, &r, torrents + i, &torrents[i].hash); | ||
| 366 | inbuf.src = compress_buffer; | 371 | inbuf.src = compress_buffer; |
| 367 | inbuf.size = r - compress_buffer; | 372 | inbuf.size = r - compress_buffer; |
| 368 | inbuf.pos = 0; | 373 | inbuf.pos = 0; |
| @@ -422,7 +427,7 @@ static void fullscrape_make_zstd(int taskid, ot_tasktype mode) { | |||
| 422 | iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); | 427 | iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); |
| 423 | 428 | ||
| 424 | if (!iovector.iov_base) { | 429 | if (!iovector.iov_base) { |
| 425 | fprintf(stderr, "Problem with iovec_fix_increase_or_free\n"); | 430 | fprintf(stderr, "Problem with iovec_next_chunk\n"); |
| 426 | ZSTD_freeCCtx(zstream); | 431 | ZSTD_freeCCtx(zstream); |
| 427 | return; | 432 | return; |
| 428 | } | 433 | } |
