added largefile support. added 'bankdump' subcommand, which uses 'readrec'. figured out .dat and .tad header bytes. replaced option with separate subcommand: .

1 files changed, 153 insertions, 52 deletions

diff --git a/crodump.py b/crodump.py
index 80daab9..148b3cf 100644
--- a/crodump.py
+++ b/crodump.py
@@ -7,6 +7,7 @@ from hexdump import hexdump, asasc, tohex, unhex, strescape
 from koddecoder import kodecode
 from readers import ByteReader
 import zlib
+from collections import defaultdict
 
 """
 python3 crodump.py crodump chechnya_proverki_ul_2012
@@ -32,23 +33,50 @@ def enumunreferenced(ranges, filesize):
 
 class Datafile:
     """ Represent a single .dat with it's .tad index file """
-    def __init__(self, dat, tad, need_decode=False):
+    def __init__(self, name, dat, tad):
+        self.name = name
         self.dat = dat
         self.tad = tad
 
-        self.need_decode = need_decode
-
+        self.readdathdr()
         self.readtad()
 
         self.dat.seek(0, io.SEEK_END)
         self.datsize = self.dat.tell()
 
+
+    def readdathdr(self):
+        self.dat.seek(0)
+        hdrdata = self.dat.read(19)
+
+        magic, self.hdrunk, self.version, self.encoding, self.blocksize = struct.unpack("<8sH5sHH", hdrdata)
+        if magic != b"CroFile\x00":
+            print("unknown magic: ", magic)
+            raise Exception("not a Crofile")
+        self.use64bit = self.version == b'01.03'
+
+        # blocksize
+        #   0040 -> Bank
+        #   0400 -> Index or Sys
+        #   0200 -> Stru  or Sys
+
+        # encoding
+        #   0000
+        #   0001  --> 'KOD encoded'
+        #   0002
+        #   0003  --> encrypted
+
     def readtad(self):
         self.tad.seek(0)
         hdrdata = self.tad.read(2*4)
-        self.tadhdr = struct.unpack("<2L", hdrdata)
+        self.nrdeleted, self.firstdeleted = struct.unpack("<2L", hdrdata)
         indexdata = self.tad.read()
-        self.tadidx = [ struct.unpack_from("<3L", indexdata, 12*_) for _ in range(len(indexdata)//12) ]
+        if self.use64bit:
+            # 01.03 has 64 bit file offsets
+            self.tadidx = [ struct.unpack_from("<QLL", indexdata, 16*_) for _ in range(len(indexdata)//16) ]
+        else:
+            # 01.02  and 01.04  have 32 bit offsets.
+            self.tadidx = [ struct.unpack_from("<LLL", indexdata, 12*_) for _ in range(len(indexdata)//12) ]
 
     def readdata(self, ofs, size):
         self.dat.seek(ofs)
@@ -68,25 +96,27 @@ class Datafile:
         if not dat:
             # empty record
             encdat = dat
-        elif self.need_decode and not flags:
+        elif not flags:
             extofs, extlen = struct.unpack("<LL", dat[:8])
             encdat = dat[8:]
             while len(encdat)<extlen:
-                dat = self.readdata(extofs, 0x200)
+                dat = self.readdata(extofs, self.blocksize)
                 extofs, = struct.unpack("<L", dat[:4])
                 encdat += dat[4:]
             encdat = encdat[:extlen]
         else:
             encdat = dat
 
-        if not self.need_decode:
-            return encdat
-        else:
-            return kodecode(idx, encdat)
+        if self.encoding == 1:
+            encdat = kodecode(idx, encdat)
+        if self.iscompressed(encdat):
+            encdat = self.decompress(encdat)
+
+        return encdat
 
 
     def dump(self, args):
-        print("tadhdr: %08x %08x" % tuple(self.tadhdr))
+        print("hdr: %-6s dat: %04x %s enc:%04x bs:%04x, tad: %08x %08x" % (self.name, self.hdrunk, self.version, self.encoding, self.blocksize, self.nrdeleted, self.firstdeleted))
         ranges = []  # keep track of used bytes in the .dat file.
         for i, (ofs, ln, chk) in enumerate(self.tadidx):
             if ln==0xFFFFFFFF:
@@ -97,39 +127,50 @@ class Datafile:
             ln &= 0xFFFFFFF
             dat = self.readdata(ofs, ln)
             ranges.append((ofs, ofs+ln, "item #%d" % i))
-            decflag = ' '
+            decflags = [' ', ' ']
             infostr = ""
             tail = b''
 
             if not dat:
                 # empty record
                 encdat = dat
-            elif self.need_decode and not flags:
-                extofs, extlen = struct.unpack("<LL", dat[:8])
+            elif not flags:
+                if self.use64bit:
+                    extofs, extlen = struct.unpack("<QL", dat[:12])
+                    o = 12
+                else:
+                    extofs, extlen = struct.unpack("<LL", dat[:8])
+                    o = 8
                 infostr = "%08x;%08x" % (extofs, extlen)
-                encdat = dat[8:]
+                encdat = dat[o:]
                 while len(encdat)<extlen:
-                    dat = self.readdata(extofs, 0x200)
-                    ranges.append((extofs, extofs+0x200, "item #%d ext" % i))
-                    extofs, = struct.unpack("<L", dat[:4])
+                    dat = self.readdata(extofs, self.blocksize)
+                    ranges.append((extofs, extofs+self.blocksize, "item #%d ext" % i))
+                    if self.use64bit:
+                        extofs, = struct.unpack("<Q", dat[:8])
+                        o = 8
+                    else:
+                        extofs, = struct.unpack("<L", dat[:4])
+                        o = 4
                     infostr += ";%08x" % (extofs)
-                    encdat += dat[4:]
+                    encdat += dat[o:]
                 tail = encdat[extlen:]
                 encdat = encdat[:extlen]
-                decflag = '+'
+                decflags[0] = '+'
             else:
                 encdat = dat
-                decflag = '*'
+                decflags[0] = '*'
 
-            if self.need_decode:
+            if self.encoding == 1:
                 decdat = kodecode(i+1, encdat)
             else:
                 decdat = encdat
-                decflag = ' '
+                decflags[0] = ' '
 
             if args.decompress and self.iscompressed(decdat):
                 decdat = self.decompress(decdat)
-            print("%5d: %08x-%08x: (%02x:%08x) %s %s%s %s" % (i+1, ofs, ofs+ln, flags, chk, infostr, decflag, toout(args, decdat), tohex(tail)))
+                decflags[1] = '@'
+            print("%5d: %08x-%08x: (%02x:%08x) %s %s%s %s" % (i+1, ofs, ofs+ln, flags, chk, infostr, "".join(decflags), toout(args, decdat), tohex(tail)))
 
         if args.verbose:
             # output parts not referenced in the .tad file.
@@ -231,19 +272,19 @@ class Database:
     def __init__(self, dbdir):
         self.dbdir = dbdir
 
-        self.stru = self.getfile("Stru", need_decode=True)
+        self.stru = self.getfile("Stru")
         self.index = self.getfile("Index")
         self.bank  = self.getfile("Bank")
-        self.sys = self.getfile("Sys", need_decode=True)
+        self.sys = self.getfile("Sys")
         # BankTemp, Int
 
 
-    def getfile(self, name, need_decode=False):
+    def getfile(self, name):
         try:
             datname = self.getname(name, "dat")
             tadname = self.getname(name, "tad")
             if datname and tadname:
-                return Datafile(open(datname, "rb"), open(tadname, "rb"), need_decode)
+                return Datafile(name, open(datname, "rb"), open(tadname, "rb"))
         except IOError:
             return
 
@@ -259,21 +300,20 @@ class Database:
 
     def dump(self, args):
         if self.stru:
-            print("stru")
             self.stru.dump(args)
-            if args.struonly:
-                self.dumptabledefs(args)
-                return
         if self.index:
-            print("index")
             self.index.dump(args)
         if self.bank:
-            print("bank")
             self.bank.dump(args)
         if self.sys:
-            print("sys")
             self.sys.dump(args)
 
+    def strudump(self, args):
+        if not self.stru:
+            print("missing CroStru file")
+            return
+        self.dumptabledefs(args)
+
     def dumptabledefs(self, args):
         dbinfo = self.stru.readrec(1)
         dbdef = destruct_bank_definition(args, dbinfo)
@@ -289,6 +329,39 @@ class Database:
                     tbinfo = struct.pack("<B", 4) + idx
                 tbdef = destruct_base_definition(args, tbinfo)
 
+    def bankdump(self, args):
+        if not self.bank:
+            print("No CroBank.dat found")
+            return
+        nerr = 0
+        xref = defaultdict(int)
+        for i in range(args.maxrecs):
+            try:
+                data = self.bank.readrec(i)
+                if not args.stats:
+                    if data is None:
+                        print("%5d: <deleted>" % i)
+                    else:
+                        print("%5d: %s" % (i, toout(args, data)))
+                else:
+                    if data is None:
+                        xref["None"] += 1
+                    elif not len(data):
+                        xref["Empty"] += 1
+                    else:
+                        xref["%02x" % data[0]] += 1
+                nerr = 0
+            except IndexError:
+                break
+            except Exception as e:
+                print("%5d: <%s>" % (i, e))
+                nerr += 1
+                if nerr > 5:
+                    break
+        if args.stats:
+            print("-- stats --")
+            for k, v in xref.items():
+                print("%5d * %s" % (v, k))
 
 def incdata(data, s):
     """
@@ -359,9 +432,23 @@ def kod_hexdump(args):
 def cro_dump(args):
     """ handle 'crodump' subcommand """
     db = Database(args.dbdir)
-
     db.dump(args)
 
+def stru_dump(args):
+    """ handle 'strudump' subcommand """
+    db = Database(args.dbdir)
+    db.strudump(args)
+
+def bank_dump(args):
+    """ hexdump all records """
+    if args.maxrecs:
+        args.maxrecs = int(args.maxrecs, 0)
+    else:
+        # an arbitrarily large number.
+        args.maxrecs = 0xFFFFFFFF
+
+    db = Database(args.dbdir)
+    db.bankdump(args)
 
 def destruct(args):
     """
@@ -396,21 +483,35 @@ def main():
     ko.add_argument('filename', type=str, nargs='?', help="dump either stdin, or the specified file")
     ko.set_defaults(handler=kod_hexdump)
 
-    cro = subparsers.add_parser('crodump', help='CROdumper')
-    cro.add_argument('--verbose', '-v', action='store_true')
-    cro.add_argument('--kodecode', '-k', action='store_true')
-    cro.add_argument('--ascdump', '-a', action='store_true')
-    cro.add_argument('--nokod', '-n', action='store_true')
-    cro.add_argument('--struonly', action='store_true')
-    cro.add_argument('--nodecompress', action='store_false', dest='decompress', default='true')
-    cro.add_argument('dbdir', type=str)
-    cro.set_defaults(handler=cro_dump)
-
-    des = subparsers.add_parser('destruct', help='Stru dumper')
-    des.add_argument('--verbose', '-v', action='store_true')
-    des.add_argument('--ascdump', '-a', action='store_true')
-    des.add_argument('--type', '-t', type=int, help='what type of record to destruct')
-    des.set_defaults(handler=destruct)
+    p = subparsers.add_parser('crodump', help='CROdumper')
+    p.add_argument('--verbose', '-v', action='store_true')
+    p.add_argument('--kodecode', '-k', action='store_true')
+    p.add_argument('--ascdump', '-a', action='store_true')
+    p.add_argument('--nokod', '-n', action='store_true')
+    p.add_argument('--nodecompress', action='store_false', dest='decompress', default='true')
+    p.add_argument('dbdir', type=str)
+    p.set_defaults(handler=cro_dump)
+
+    p = subparsers.add_parser('bankdump', help='BANKdumper')
+    p.add_argument('--verbose', '-v', action='store_true')
+    p.add_argument('--ascdump', '-a', action='store_true')
+    p.add_argument('--maxrecs', '-n', type=str, help="max nr or recots to output")
+    p.add_argument('--stats', action='store_true', help='calc table stats from the first byte of each record')
+    p.add_argument('dbdir', type=str)
+    p.set_defaults(handler=bank_dump)
+
+    p = subparsers.add_parser('strudump', help='STRUdumper')
+    p.add_argument('--verbose', '-v', action='store_true')
+    p.add_argument('--ascdump', '-a', action='store_true')
+    p.add_argument('dbdir', type=str)
+    p.set_defaults(handler=stru_dump)
+
+
+    p = subparsers.add_parser('destruct', help='Stru dumper')
+    p.add_argument('--verbose', '-v', action='store_true')
+    p.add_argument('--ascdump', '-a', action='store_true')
+    p.add_argument('--type', '-t', type=int, help='what type of record to destruct')
+    p.set_defaults(handler=destruct)
 
     args = parser.parse_args()