diff options
| -rw-r--r-- | Makefile | 4 | ||||
| -rw-r--r-- | ot_fullscrape.c | 41 | ||||
| -rw-r--r-- | ot_http.c | 10 | ||||
| -rw-r--r-- | tests/testsuite2.sh | 24 |
4 files changed, 47 insertions, 32 deletions
| @@ -24,7 +24,7 @@ STRIP?=strip | |||
| 24 | #FEATURES+=-DWANT_DYNAMIC_ACCESSLIST | 24 | #FEATURES+=-DWANT_DYNAMIC_ACCESSLIST |
| 25 | 25 | ||
| 26 | #FEATURES+=-DWANT_SYNC_LIVE | 26 | #FEATURES+=-DWANT_SYNC_LIVE |
| 27 | #FEATURES+=-DWANT_IP_FROM_QUERY_STRING | 27 | FEATURES+=-DWANT_IP_FROM_QUERY_STRING |
| 28 | #FEATURES+=-DWANT_COMPRESSION_GZIP | 28 | #FEATURES+=-DWANT_COMPRESSION_GZIP |
| 29 | #FEATURES+=-DWANT_COMPRESSION_GZIP_ALWAYS | 29 | #FEATURES+=-DWANT_COMPRESSION_GZIP_ALWAYS |
| 30 | #FEATURES+=-DWANT_LOG_NETWORKS | 30 | #FEATURES+=-DWANT_LOG_NETWORKS |
| @@ -48,7 +48,7 @@ FEATURES+=-DWANT_FULLSCRAPE | |||
| 48 | OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage | 48 | OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage |
| 49 | OPTS_production=-O3 | 49 | OPTS_production=-O3 |
| 50 | 50 | ||
| 51 | CFLAGS+=-I$(LIBOWFAT_HEADERS) -Wall -pipe -pthread -Wextra #-ansi -pedantic | 51 | CFLAGS+=-I$(LIBOWFAT_HEADERS) -Wall -pipe -pthread -Wextra -Wincompatible-pointer-types #-ansi -pedantic |
| 52 | LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread -lz | 52 | LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread -lz |
| 53 | #LDFLAGS+=-lbsd | 53 | #LDFLAGS+=-lbsd |
| 54 | 54 | ||
diff --git a/ot_fullscrape.c b/ot_fullscrape.c index b147b6a..86b9e26 100644 --- a/ot_fullscrape.c +++ b/ot_fullscrape.c | |||
| @@ -262,33 +262,36 @@ fprintf(stderr, "GZIP path\n"); | |||
| 262 | if( deflate( &strm, Z_FINISH ) < Z_OK ) | 262 | if( deflate( &strm, Z_FINISH ) < Z_OK ) |
| 263 | fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); | 263 | fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); |
| 264 | 264 | ||
| 265 | if( !strm.avail_out ) { | 265 | iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base; |
| 266 | if (mutex_workqueue_pushchunked(taskid, &iovector) ) { | ||
| 267 | free(iovector.iov_base); | ||
| 268 | return mutex_bucket_unlock( bucket, 0 ); | ||
| 269 | } | ||
| 270 | |||
| 271 | { | ||
| 266 | unsigned int pending; | 272 | unsigned int pending; |
| 267 | int bits; | 273 | int bits; |
| 268 | deflatePending( &strm, &pending, &bits); | 274 | deflatePending( &strm, &pending, &bits); |
| 269 | pending += ( bits ? 1 : 0 ); | 275 | pending += ( bits ? 1 : 0 ); |
| 270 | 276 | ||
| 271 | iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base; | 277 | if (pending) { |
| 272 | if (mutex_workqueue_pushchunked(taskid, &iovector) ) { | 278 | /* Allocate a fresh output buffer */ |
| 273 | free(iovector.iov_base); | 279 | iovector.iov_base = malloc( pending ); |
| 280 | iovector.iov_len = pending; | ||
| 281 | |||
| 282 | if( !iovector.iov_base ) { | ||
| 283 | fprintf( stderr, "Problem with iovec_fix_increase_or_free\n" ); | ||
| 284 | deflateEnd(&strm); | ||
| 274 | return mutex_bucket_unlock( bucket, 0 ); | 285 | return mutex_bucket_unlock( bucket, 0 ); |
| 275 | } | 286 | } |
| 276 | /* Allocate a fresh output buffer */ | 287 | strm.next_out = iovector.iov_base; |
| 277 | iovector.iov_base = malloc( pending ); | 288 | strm.avail_out = pending; |
| 278 | iovector.iov_len = pending; | 289 | if( deflate( &strm, Z_FINISH ) < Z_OK ) |
| 290 | fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); | ||
| 279 | 291 | ||
| 280 | if( !iovector.iov_base ) { | 292 | if( mutex_workqueue_pushchunked(taskid, &iovector) ) |
| 281 | fprintf( stderr, "Problem with iovec_fix_increase_or_free\n" ); | 293 | free(iovector.iov_base); |
| 282 | deflateEnd(&strm); | ||
| 283 | return mutex_bucket_unlock( bucket, 0 ); | ||
| 284 | } | 294 | } |
| 285 | strm.next_out = iovector.iov_base; | ||
| 286 | strm.avail_out = pending; | ||
| 287 | if( deflate( &strm, Z_FINISH ) < Z_OK ) | ||
| 288 | fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); | ||
| 289 | |||
| 290 | if( mutex_workqueue_pushchunked(taskid, &iovector) ) | ||
| 291 | free(iovector.iov_base); | ||
| 292 | } | 295 | } |
| 293 | 296 | ||
| 294 | deflateEnd(&strm); | 297 | deflateEnd(&strm); |
| @@ -162,7 +162,7 @@ fprintf(stderr, "http_sendiovecdata sending %d iovec entries found cookie->batch | |||
| 162 | header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n%sContent-Length: %zd\r\n\r\n", encoding, size ); | 162 | header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n%sContent-Length: %zd\r\n\r\n", encoding, size ); |
| 163 | else { | 163 | else { |
| 164 | if ( !(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER )) { | 164 | if ( !(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER )) { |
| 165 | header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n%sTransfer-Encoding: chunked\r\n\r\n%zx\r\n", encoding, size ); | 165 | header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: application/octet-stream\r\n%sTransfer-Encoding: chunked\r\n\r\n%zx\r\n", encoding, size ); |
| 166 | cookie->flag |= STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER; | 166 | cookie->flag |= STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER; |
| 167 | } else | 167 | } else |
| 168 | header_size = sprintf( header, "%zx\r\n", size ); | 168 | header_size = sprintf( header, "%zx\r\n", size ); |
| @@ -269,12 +269,12 @@ static const ot_keywords keywords_format[] = | |||
| 269 | tai6464 t; | 269 | tai6464 t; |
| 270 | #ifdef WANT_COMPRESSION_GZIP | 270 | #ifdef WANT_COMPRESSION_GZIP |
| 271 | ws->request[ws->request_size] = 0; | 271 | ws->request[ws->request_size] = 0; |
| 272 | #ifdef WANT_COMPRESSION_GZIP_ALWAYS | 272 | #ifndef WANT_COMPRESSION_GZIP_ALWAYS |
| 273 | if( strstr( read_ptr - 1, "gzip" ) ) { | 273 | if( strstr( read_ptr - 1, "gzip" ) ) { |
| 274 | #endif | 274 | #endif |
| 275 | cookie->flag |= STRUCT_HTTP_FLAG_GZIP; | 275 | cookie->flag |= STRUCT_HTTP_FLAG_GZIP; |
| 276 | format |= TASK_FLAG_GZIP; | 276 | format |= TASK_FLAG_GZIP; |
| 277 | #ifdef WANT_COMPRESSION_GZIP_ALWAYS | 277 | #ifndef WANT_COMPRESSION_GZIP_ALWAYS |
| 278 | } | 278 | } |
| 279 | #endif | 279 | #endif |
| 280 | #endif | 280 | #endif |
| @@ -337,12 +337,16 @@ static ssize_t http_handle_fullscrape( const int64 sock, struct ot_workstruct *w | |||
| 337 | 337 | ||
| 338 | #ifdef WANT_COMPRESSION_GZIP | 338 | #ifdef WANT_COMPRESSION_GZIP |
| 339 | ws->request[ws->request_size-1] = 0; | 339 | ws->request[ws->request_size-1] = 0; |
| 340 | #ifndef WANT_COMPRESSION_GZIP_ALWAYS | ||
| 340 | if( strstr( ws->request, "gzip" ) ) { | 341 | if( strstr( ws->request, "gzip" ) ) { |
| 342 | #endif | ||
| 341 | cookie->flag |= STRUCT_HTTP_FLAG_GZIP; | 343 | cookie->flag |= STRUCT_HTTP_FLAG_GZIP; |
| 342 | format = TASK_FLAG_GZIP; | 344 | format = TASK_FLAG_GZIP; |
| 343 | stats_issue_event( EVENT_FULLSCRAPE_REQUEST_GZIP, 0, (uintptr_t)cookie->ip ); | 345 | stats_issue_event( EVENT_FULLSCRAPE_REQUEST_GZIP, 0, (uintptr_t)cookie->ip ); |
| 346 | #ifndef WANT_COMPRESSION_GZIP_ALWAYS | ||
| 344 | } else | 347 | } else |
| 345 | #endif | 348 | #endif |
| 349 | #endif | ||
| 346 | stats_issue_event( EVENT_FULLSCRAPE_REQUEST, 0, (uintptr_t)cookie->ip ); | 350 | stats_issue_event( EVENT_FULLSCRAPE_REQUEST, 0, (uintptr_t)cookie->ip ); |
| 347 | 351 | ||
| 348 | #ifdef _DEBUG_HTTPERROR | 352 | #ifdef _DEBUG_HTTPERROR |
diff --git a/tests/testsuite2.sh b/tests/testsuite2.sh index c9a5a6a..da5181b 100644 --- a/tests/testsuite2.sh +++ b/tests/testsuite2.sh | |||
| @@ -2,13 +2,21 @@ | |||
| 2 | 2 | ||
| 3 | while true; do | 3 | while true; do |
| 4 | request_string="GET /announce?info_hash=012345678901234567\ | 4 | request_string="GET /announce?info_hash=012345678901234567\ |
| 5 | %$(printf %02X $(( $RANDOM & 0xff )) )\ | 5 | $(printf %02X $(( $RANDOM & 0xff )) )\ |
| 6 | %$(printf %02X $(( $RANDOM & 0xff )) )\ | 6 | &ip=$(( $RANDOM & 0xff )).17.13.15&port=$(( $RANDOM & 0xff )) HTTP/1.0" |
| 7 | &ip=$(( $RANDOM & 0xff )).17.13.15&port=$(( $RANDOM & 0xff )) HTTP/1.0\n" | 7 | |
| 8 | 8 | # echo $request_string | |
| 9 | echo $request_string | 9 | # echo |
| 10 | echo | 10 | printf "%s\n\n" "$request_string" | nc 84.200.61.9 6969 | hexdump -C |
| 11 | echo $request_string | nc 23.23.23.237 6969 >/dev/null | 11 | |
| 12 | echo | 12 | request_string="GET /announce?info_hash=012345678901234567\ |
| 13 | $(printf %02X $(( $RANDOM & 0xff )) )\ | ||
| 14 | &ip=2001:1608:6:27::$(( $RANDOM & 0xff ))&port=$(( $RANDOM & 0xff )) HTTP/1.0" | ||
| 15 | printf "%s\n\n" "$request_string" | nc 2001:1608:6:27::9 6969 | hexdump -C | ||
| 16 | printf "%s\n\n" "$request_string" | ||
| 17 | |||
| 18 | request_string="GET /scrape?info_hash=012345678901234567\ | ||
| 19 | $(printf %02X $(( $RANDOM & 0xff )) ) HTTP/1.0" | ||
| 20 | printf "%s\n\n" "$request_string" | nc 2001:1608:6:27::9 6969 | hexdump -C | ||
| 13 | 21 | ||
| 14 | done | 22 | done |
