Clean up 1992 post processing

5 files changed, 23 insertions, 21 deletions

diff --git a/src/export/extract_version_1.c b/src/export/extract_version_1.c
index 8ec199e..6d67a5d 100644
--- a/src/export/extract_version_1.c
+++ b/src/export/extract_version_1.c
@@ -39,9 +39,9 @@ static uint8_t cp437_to_iso8859_1_table[] = {
 };
 
 static struct {
-  int      outfiles[15];
-  uint8_t *outbuf[15];
-  size_t   outfill[15];
+  int      outfiles[16];
+  uint8_t *outbuf[16];
+  size_t   outfill[16];
   char *   vorwahl;
   char     ort[1024];
   size_t   ort_len;
@@ -101,11 +101,13 @@ static void split_to_files( uint8_t *entries, int num_entries )
       *( g_state.outbuf[0] + g_state.outfill[0]++ ) = num_entries > 1 ? '1' : '0';
       memcpy( g_state.outbuf[12] + g_state.outfill[12], g_state.ort, g_state.ort_len ); g_state.outfill[12] += g_state.ort_len;
       memcpy( g_state.outbuf[13] + g_state.outfill[13], g_state.zip, g_state.zip_len ); g_state.outfill[13] += g_state.zip_len;
+      strcpy( g_state.outbuf[14] + g_state.outfill[14], g_state.vorwahl);               g_state.outfill[14] += strlen(g_state.vorwahl);
     }
 
     *( g_state.outbuf[0 ] + g_state.outfill[0 ]++ ) = '\n';
     *( g_state.outbuf[12] + g_state.outfill[12]++ ) = '\n';
     *( g_state.outbuf[13] + g_state.outfill[13]++ ) = '\n';
+    *( g_state.outbuf[14] + g_state.outfill[14]++ ) = '\n';
     if( !end )
       return;
   }
@@ -178,7 +180,7 @@ static void act_on_file( uint8_t *file )
   g_state.zip_len = snprintf( g_state.zip, sizeof(g_state.zip), "%s", zip );
   g_state.vorwahl = vorwahl;
 
-  /* printf( "Working on a %04d page and %06d records file, city: %4s %-32s with prefix %s\n", num_pages, num_records, zip, ort, vorwahl ); */
+  printf( "Working on a %04d page and %06d records file, city: %4s %-32s with prefix %s\n", num_pages, num_records, zip, ort, vorwahl );
   (void)num_records; /* silence warning about unused variable */
 
   for( page = 0; page < num_pages; ++page )
@@ -193,7 +195,7 @@ int main( )
   ssize_t temp = 0;
   int i;
 
-  for( i=0; i<14; ++i )
+  for( i=0; i<15; ++i )
   {
     sprintf( filename, "%02d_unknown", i+1 );
     g_state.outfiles[i] = open( filename, O_WRONLY | O_APPEND | O_CREAT, 0644 );
@@ -209,14 +211,14 @@ int main( )
     unmap_file( &f );
 
     /* Write out results */
-    for( i=0; i<14; ++i ) {
+    for( i=0; i<15; ++i ) {
       /* if( g_state.outfill[i] > 1024*1024*6 ) printf( "Large: %s %zd\n", g_state.ort, g_state.outfill[i] ); */
       temp += write( g_state.outfiles[i], g_state.outbuf[i], g_state.outfill[i] );
       g_state.outfill[i]  = 0;
     }
   }
 
-  for( i=0; i<14; ++i ) {
+  for( i=0; i<15; ++i ) {
     temp += write( g_state.outfiles[i], g_state.outbuf[i], g_state.outfill[i] );
     close( g_state.outfiles[i] );
   }
diff --git a/src/export/mystdlib.c b/src/export/mystdlib.c
index b65f63d..31c991e 100644
--- a/src/export/mystdlib.c
+++ b/src/export/mystdlib.c
@@ -21,7 +21,9 @@ MAP map_file( char *filename, int readonly )
     if( ( map->fh = open( filename, readonly ? O_RDONLY : O_RDWR ) ) >= 0 )
     {
       fstat( map->fh, &fstatus );
-      if( ( map->addr = mmap( NULL, map->size = (size_t)fstatus.st_size,
+      map->size = (size_t)fstatus.st_size;
+      if (!map->size) return map;
+      if( ( map->addr = mmap( NULL, map->size,
          PROT_READ | ( readonly ? 0 : PROT_WRITE), (readonly ? MAP_PRIVATE : MAP_SHARED), map->fh, 0) ) == MAP_FAILED )
       {
         fprintf( stderr, "Mapping file '%s' failed\n", filename );
diff --git a/src/postprocess/map_plz.c b/src/postprocess/map_plz.c
index ab0db71..9dec6bb 100644
--- a/src/postprocess/map_plz.c
+++ b/src/postprocess/map_plz.c
@@ -18,7 +18,7 @@ int main(int argc, char **args) {
   FILE *bfile, *streetfile_out;
   char *ptr, *input = malloc(65335);
   char *ort = malloc(65335), vorwahl_block[16];
-  int i, brutes_count = 0, report = 0;
+  unsigned int i, brutes_count = 0, report = 0;
   brute_t *brutes = malloc(200000*sizeof(brute_t));
 
   /* prepare io */
@@ -63,7 +63,7 @@ int main(int argc, char **args) {
   g_book_by_name = (entry_t*)malloc(g_book_size * sizeof(entry_t));
 
   /* Split pointers into input files into our arrays */
-  for (i = 0, ptr = (char*)tbuch->addr; i < g_book_size; ++i) {
+  for (i=0, ptr=(char*)tbuch->addr; i<g_book_size; ++i) {
     g_book[i].vorwahl = ptr; ptr += strlen(ptr) + 1;
     g_book[i].ort     = ptr; ptr += strlen(ptr) + 1;
     g_book[i].strasse = ptr; ptr += strlen(ptr) + 1;
diff --git a/src/postprocess/merge_entries.c b/src/postprocess/merge_entries.c
index f9ee67d..1dd7d50 100644
--- a/src/postprocess/merge_entries.c
+++ b/src/postprocess/merge_entries.c
@@ -148,9 +148,10 @@ static int sort_me(const void *f_a, const void *f_b) {
     outvec_t *oa_row = oa + row * COLUMNS;
     outvec_t *ob_row = ob + row * COLUMNS;
 
+    if ((res = STRCMP_n(oa_row[ 2].ptr, ob_row[ 2].ptr))) return res; /* PLZ */
+    if ((res = STRCMP_n(oa_row[ 9].ptr, ob_row[ 9].ptr))) return res; /* Ort */
     if ((res = STRCMP_n(oa_row[10].ptr, ob_row[10].ptr))) return res; /* Vorwahl */
     if ((res = STRCMP_n(oa_row[11].ptr, ob_row[11].ptr))) return res; /* Rufnummer */
-    if ((res = STRCMP_n(oa_row[ 2].ptr, ob_row[ 2].ptr))) return res; /* PLZ */
     if ((res = STRCMP_n(oa_row[ 6].ptr, ob_row[ 6].ptr))) return res; /* Strasse */
     if ((res = STRCMP_n(oa_row[ 7].ptr, ob_row[ 7].ptr))) return res; /* Hausnummer */
     if ((res = STRCMP_n(oa_row[ 3].ptr, ob_row[ 3].ptr))) return res; /* Nachname */
@@ -200,7 +201,7 @@ int main(int argc, char **args) {
   unsigned long current = 0, i, flag;
   uint64_t year_list = 0, revflag_list = 0, bizflag_list = 0;
 
-  if (argc != 1) exit(1);
+  if (argc != 2) exit(1);
   tbuch = map_file(args[1], 1);
 
   /* Estimate upper bound for amount of lines */
diff --git a/src/postprocess/postprocess-1992.sh b/src/postprocess/postprocess-1992.sh
index 1e685d2..6720991 100644
--- a/src/postprocess/postprocess-1992.sh
+++ b/src/postprocess/postprocess-1992.sh
@@ -4,19 +4,16 @@ paste 1992_Q2/{01_Flags,12_Vorwahl,12_Vorwahl_block,11_Ort,07_Strasse,08_Hausnum
 # Generate lookup file from 1995
 paste 1995_Q0/{12_Vorwahl,11_Ort,07_Strasse,08_Hausnummer,02_Nachname,03_Vorname,13_Rufnummer,10_Postleitzahl} | tr '\n\t' '\0' > 1995-voshnvrp.bin
 
-# To debug in lldb
-process launch -i 1992_testfile.txt -- 1995-vorwahl-ort-strasse-hnr-name-vorname-rufnummer-plz.bin
-
 # Compile plz mapper
 cc -O3 -o map_plz map_plz.c -I ../src/export/ ../src/export/mystdlib.c
 
+# To debug in lldb
+# cc -O0 -g -o map_plz map_plz.c -I ../src/export/ ../src/export/mystdlib.c
+# process launch -i 1992_testfile.txt -- 1995-vorwahl-ort-strasse-hnr-name-vorname-rufnummer-plz.bin
+
 # outputs mapped plz, generates brutemap.txt
 touch brutemap_input.bin zip_simple_map.bin
-./map_plz 1995-voshnvrp.bin zip_simple_map.bin brutemap_input.bin < 1992-fvvoshnvrpp.txt > 10_Postleitzahl
-
-# generate street name translation table from brutemap,
-# only taking into account similar street names
-# cut -f 3,4 brutemap.txt | tr '[:upper:]' '[:lower:]' | paste brutemap.txt - | cut -f 1-4,6,7 | ./jaro | cut -f 1-5 > brutemap_filtered.txt
+./map_plz 1995-voshnvrp.bin zip_simple_map.bin brutemap_input.bin < 1992-fvvoshnvrpp.txt > 1992_Q2/10_Postleitzahl
 
 # generate street name translation table from brutemap,
 # only taking into account similar street names, new style
@@ -29,4 +26,4 @@ cut -f 1-5 brutemap_simifiltered.txt | sort | uniq -c | sed -E $'s:^ *([[:digit:
 sort -u zip_mapfile.txt | tr '\n' '\0' > zip_simple_map.bin
 
 # Redo the mapping with the data from brutemap and zipmap
-./map_plz 1995-voshnvrp.bin zip_simple_map.bin brutemap_input.bin < 1992-fvvoshnvrpp.txt > 10_Postleitzahl
+./map_plz 1995-voshnvrp.bin zip_simple_map.bin brutemap_input.bin < 1992-fvvoshnvrpp.txt > 1992_Q2/10_Postleitzahl