path: root/src/postprocess/merge_entries.c

#include "mystdlib.h"

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <inttypes.h>
#include <assert.h>

enum { COLUMNS = 15 };
typedef struct {
    char *ptr;
    long rows;
    long outoff;
    long flag;
} entry_t;
typedef struct {
    char *ptr;
    size_t size;
} outvec_t;

const char *g_year_map[] = {
"1992_Q2", "1995_Q0", "1996_Q0", "1996_Q1", "1997_Q1", "1997_Q3", "1998_Q1", "1998_Q3", "1999_Q1", "1999_Q3", "2000_Q1", "2000_Q3", "2001_Q1", "2001_Q2", "2001_Q3", "2001_Q4", "2002_Q1",
"2002_Q3", "2003_Q1", "2003_Q3", "2004_Q1", "2004_Q3", "2005_Q1", "2005_Q3", "2006_Q1", "2006_Q3", "2007_Q1", "2007_Q3", "2008_Q1", "2008_Q3", "2009_Q1", "2009_Q3", "2010_Q1", "2010_Q3",
"2011_Q1", "2011_Q3", "2012_Q1", "2012_Q3", "2013_Q1", "2013_Q3", "2014_Q1", "2014_Q3", "2015_Q1", "2015_Q3", "2016_Q1", "2016_Q3", "2017_Q1", "2017_Q3", "2018_Q1", "2018_Q3", "2019_Q1",
0
};

void SKIP_1_COLUMN(char **ptr) { *ptr = strchr(*ptr, 10) + 1; }
void SKIP_2_COLUMNS(char **ptr) { SKIP_1_COLUMN(ptr); SKIP_1_COLUMN(ptr); }
void SKIP_3_COLUMNS(char **ptr) { SKIP_1_COLUMN(ptr); SKIP_1_COLUMN(ptr); SKIP_1_COLUMN(ptr); }

int year_to_offset(const char *year) {
  const char **y = g_year_map;
  int off = 0;
  while (*y) {
    if (!memcmp(year, *y, 7)) return off;
    ++off; ++y;
  }
  return -1;
}


int
STRCMP_n (const char *p1, const char *p2)
{
  const unsigned char *s1 = (const unsigned char *) p1;
  const unsigned char *s2 = (const unsigned char *) p2;
  unsigned char c1, c2;
  do
    {
      c1 = (unsigned char) *s1++;
      c2 = (unsigned char) *s2++;
      if (c1 == 10)
        return c1 - c2;
    }
  while (c1 == c2);
  return c1 - c2;
}

int compare_entries(entry_t*a, entry_t*b, int *prec) {
  char *pa = a->ptr, *pb = b->ptr;
  int col, row, res = 0, nprec = -1;

  /* Multi line entries never match single line entries */
  if (a->rows != b->rows)
    return -1;

  /* Assume house number precision first .. unless */
  if (!memcmp(pa,"2006_Q3",7))
    *prec = 2;
  else
    *prec = 3;

  if (!memcmp(pb,"2006_Q3",7))
    nprec = 2;
  else
    nprec = 3;

  /* Skip year and flags */
  SKIP_2_COLUMNS(&pa);
  SKIP_2_COLUMNS(&pb);

  /* Test all columns for identity */
  for (col=2; col<COLUMNS-1; ++col) {
    if (!res && STRCMP_n(pa, pb))
      res = -1;
    SKIP_1_COLUMN(&pa);
    SKIP_1_COLUMN(&pb);
  }

  /* If no coords, downgrade precision */
  if (*pa == 9) *prec = 1;
  if (*pb == 9) nprec = 1;

  /* If entries differ, return after precision has been found */
  if (res) return res;

  /* Only if precision is the same, difference in coordinates
     is significant.
  if (*prec == nprec && STRCMP_n(pa, pb))
    return -1; */

  /* Row 1 has been compared, check the rest of lines */
  for (row=0; row<a->rows; ++row) {

    /* Skip last row's coordinate columns, year and flags */
    SKIP_3_COLUMNS(&pa);
    SKIP_3_COLUMNS(&pb);

    for (col=2; col<COLUMNS-1; ++col) {
      if (STRCMP_n(pa, pb))
        return -1;
      SKIP_1_COLUMN(&pa);
      SKIP_1_COLUMN(&pb);
    }

    /* Only if precision is the same, difference in coordinates
       is significant.
    if (*prec == nprec && STRCMP_n(pa, pb))
      return -1; */
  }
  return 0;
}

int sort_me(const void *f_a, const void *f_b) {
  entry_t *e_a = (entry_t *)f_a; 
  entry_t *e_b = (entry_t *)f_b; 

  char * pa = (char*)e_a->ptr;
  char * pb = (char*)e_b->ptr;

  int results[COLUMNS], c;

  if (e_a->rows != e_b->rows)
    return e_a->rows - e_b->rows;

  for (c = 0; c<COLUMNS; ++c) {
    results[c] = STRCMP_n(pa, pb);
    SKIP_1_COLUMN(&pa);
    SKIP_1_COLUMN(&pb);
  }

  if (results[10]) return results[10]; /* Vorwahl */
  if (results[11]) return results[11]; /* Rufnummer */
  if (results[2])  return results[2];  /* PLZ */
  if (results[3])  return results[3];  /* Nachname */
  if (results[4])  return results[4];  /* Vorname */
  if (results[6])  return results[6];  /* Strasse */
  if (results[7])  return results[7];  /* Hausnummer */
  if (results[8])  return results[8];  /* Verweise */
  if (results[0])  return results[0];  /* Year */
  return 0;
}

enum { OUTPUT_BUFFER_SIZE = 1024*1024*128 };

static void do_escape_string(char * s, size_t len) {
  size_t i;

  putchar('"');
  for (i=0; i<len; ++i) {
    switch(s[i]) {
    case '\\':
      putchar('\\');
    case '"':
      putchar('\\');
    case ',':
      putchar('\\');
    default:
      break;
    }
    putchar(s[i]);
  }
  putchar('"');
}

static void escape_string(char * s, size_t len) {
  size_t i;
  if (len == 4 && ( ( s[0] | 0x20 ) == 'n' ) && ( ( s[1] | 0x20 ) == 'u' ) && ( ( s[2] | 0x20 ) == 'l' ) && ( ( s[3] | 0x20 ) == 'l' ) )
    return do_escape_string(s, len);
  for (i=0; i<len; ++i)
    switch(s[i]) {
      case '{': case '\\': case '}': case '"': case ',': return do_escape_string(s, len);
    }
  fwrite(s, len, 1, stdout);
}

int main(int argc, char **args) {
  MAP tbuch = map_file(args[1], 1);
  char *ptr, *start;
  entry_t * sort_array;
  outvec_t * out_array;
  int current = -1, outoff = 0, lines = 1, i, truth = 0, truth_prec = -1;
  uint64_t year_list = 0, revflag_list = 0, bizflag_list = 0;
  long flag = 0;

  /* Estimate upper bound for amount of lines */
  for (i=0; i<tbuch->size; ++i)
    if (tbuch->addr[i] == 10)
      ++lines;

  sort_array = (entry_t*)malloc((lines / COLUMNS) * sizeof(entry_t));
  out_array = (outvec_t*)malloc((lines / COLUMNS) * sizeof(outvec_t));

  ptr = (char*)tbuch->addr;
  start = ptr;

  while (ptr < (char*)tbuch->addr + tbuch->size) {
    int c;

    start = ptr;

    /* Look for field terminator */
    for (c=0; c<COLUMNS; ++c) {
      char * end = strchr(ptr, 10);
      if (c==1) {
        flag = strtoul(ptr, 0, 16);
        out_array[outoff].ptr = end + 1;
      }
      ptr = end + 1;
    }

    if (flag&2) {
      assert( current >= 0);
      sort_array[current].rows++;
    } else {
      sort_array[++current].ptr = start;
      sort_array[current].rows = 0;
      sort_array[current].outoff = outoff;
      sort_array[current].flag = flag;
    }
    out_array[outoff].size = ptr - out_array[outoff].ptr;
    outoff++;
  }

  /* Sort the whole thing */
  qsort(sort_array, current, sizeof(entry_t), sort_me);

  for (i=0; i<=current; ++i) {
    int j, dump = 0, prec;

    int year = year_to_offset(sort_array[i].ptr);

    year_list |= 1LL << year;
    if (sort_array[i].flag & 0x80 ) bizflag_list |= 1LL << year;
    if (sort_array[i].flag & 0x40 ) revflag_list |= 1LL << year;

    /* The last entry always needs to be dumped, but check if its
       precision is better than the old truth's
       The second comparision checks for equality of entries (modulo
       coordinate mismatch)
    */
    if (i == current) {
      compare_entries(sort_array+i, sort_array+i, &prec);
      dump = 1;
    } else if (compare_entries(sort_array+i, sort_array+i+1, &prec))
      dump = 1;

    /* If this entry's precision is higher than the one of possible
       earlier matches, then the current entry becomes the truth */
    if (prec >= truth_prec) {
      truth = i;
      truth_prec = prec;
    }

    if (dump) {
      printf("%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t", year_list, bizflag_list, revflag_list);
      for (int c=0; c<COLUMNS-2; ++c) {
        outvec_t * out = out_array + sort_array[truth].outoff;
        int started = 0, skipped = 0;
        for (j=0; j<=sort_array[truth].rows; ++j) {
          char *s = strchr(out->ptr, 10);
          size_t len = s - out->ptr;
          if (!len || out->ptr[0] == 9)
            skipped++;
          else {
            if (!started++)
              putchar('{');
            else
              putchar(',');
            for (int x=0; x<skipped; ++x) fputs("null,", stdout);
            if (c != COLUMNS-3)
              escape_string(out->ptr, len);
            else {
              char coords[64], *tab;
//              memcpy(coords, "POINT(", 6);
//              memcpy(coords + 6, out->ptr, len);
//              tab = memchr(coords + 6, 9, len);
//              if (tab) *tab = ' ';
//              coords[6+len] = ')';
//              fwrite(coords, 7 + len, 1, stdout);
                memcpy(coords, out->ptr, len);
                tab = memchr(coords, 9, len);
                if (tab) *tab = ' ';
                fwrite(coords, len, 1, stdout);
            }
            skipped = 0;
          }
          out->ptr = s + 1;
          ++out;
        }
        if (started) putchar('}');
        if (c<COLUMNS-3) putchar(9);
      }
      putchar(10);

      truth_prec = -1;
      year_list = 0;
      bizflag_list = 0;
      revflag_list = 0;
    }
  }
}