Lots of changes, expand to read

- Add notes folder with MDBook documentation (the NOTES.md file was getting kind of large)
- Add rz_analyze.py, does the same a r2_analyze.py just with Rizin instead of radare2 so the project can be loaded in Cutter (*and* it's faster)
- Add Scrap.rzdb, Rizin database for the Scrap.exe executable
- Add Scrapper_rs, Rust version of .packed extractor and repacker
- replace helplib.txt with helplib.md
- add Py_Docs folder which contains generated documentation for the binary python modules built into Scrap.exe

tools/analyze_read_trace.py

@@ -8,10 +8,13 @@ import string
 import re
 from binascii import hexlify
 
+
 def gen():
     with open(sys.argv[1], "rb") as fh:
         size = os.stat(sys.argv[1]).st_size
-        progbar = tqdm(total=size, unit="bytes", unit_scale=True, unit_divisor=1024)
+        progbar = tqdm(
+            total=size, unit="bytes", unit_scale=True, unit_divisor=1024, leave=False
+        )
         pos = 0
         for entry in mp.Unpacker(fh, raw=True):
             progbar.update(fh.tell() - pos)
@@ -27,14 +30,10 @@ def gen():
             yield entry
 
 
-
 def strdump(data):
     printable_chars = set(bytes(string.printable, "ascii")) - set(b"\n\r\t\x0b\x0c")
     return "".join(chr(c) if c in printable_chars else "." for c in data)
 
-def tohex(data):
-    return str(hexlify(data), "utf8").upper()
-
 
 # best=sorted(tqdm(gen(),ascii=True),key=lambda v:len(v['data']),reverse=True)
 
@@ -53,16 +52,24 @@ def tohex(data):
 #                 entry['infos'][fmt]=[v[0] for v in struct.iter_unpack(fmt,data)]
 #     return entry
 
-filters=[re.compile(s) for s in sys.argv[2:]]
-
+filters = sys.argv[2:]
 with open("all.log", "w") as of:
     for entry in gen():
-        fm=[(f.match(entry['filename']) is not None) for f in filters]
-        if filters and not any(fm):
+        fm = any(
+            all(s.lower() in entry["filename"].lower() for s in f.split("|"))
+            for f in filters
+        )
+        if filters and not fm:
             continue
+        is_magic = bytes(entry["block_id"], "utf8").ljust(4, b"\0") == entry["data"]
         entry["data_len"] = len(entry["data"])
         entry["str"] = strdump(entry["data"])
-        entry["data"] = tohex(entry["data"])
+        entry["data"] = entry["data"].hex().upper()
+        if is_magic:
+            print("#" * 50, file=of)
         print(
-            "{timestamp} {block_id} {filename} {data_len:08X} {data} {str}".format(**entry), file=of
+            "{timestamp} {block_id} {filename} {data_len:08X} {data} {str}".format(
+                **entry
+            ),
+            file=of,
         )

tools/parse_chunked_new.py

@@ -38,12 +38,12 @@ def seek_to(fh, offset, pos=None):
     yield
     fh.seek(pos)
 
-def read_array(s,fh):
-    ret=[]
+
+def read_array(s, fh):
+    ret = []
     count = read_struct("<I", fh)[0]
-    size = struct.calcsize(s)
     for _ in range(count):
-        ret.append(read_struct(s,fh))
+        ret.append(read_struct(s, fh))
     return ret
 
 
@@ -93,7 +93,10 @@ class Parser:
         print("{}[{}] {} bytes".format("  " * self.depth, magic, len(data)))
         self.depth += 1
         fh = BytesIO(data)
-        ret = getattr(self, magic, lambda fh: self._default(magic, fh))(fh)
+        if len(data) != 0:
+            ret = getattr(self, magic, lambda fh: self._default(magic, fh))(fh)
+        else:
+            ret = None
         pos = fh.tell()
         leftover = len(fh.read())
         fh.seek(pos)
@@ -198,17 +201,27 @@ class Parser:
             dum = {}
             dum["name"] = read_str(fh)
             dum["pos"] = read_struct("<fff", fh)
-            dum["rot"] = read_struct("<fff", fh)
+            dum["ang"] = read_struct("<fff", fh)
             dum["has_ini"] = read_struct("<I", fh)[0]
             if dum["has_ini"]:
-                dum['ini']=self.parse_block(fh)
-            dum["has_next"] = read_struct("<I", fh)[0]
+                dum["ini"] = self.parse_block(fh)
+            dum["unk_1"] = read_struct("<I", fh)[0]
             ret["dummies"].append(dum)
         assert fh.read() == b"", "Leftover Data"
         return ret
 
-    # def AMC(self, fh):
-    #     return len(fh.read())
+    def AMC(self, fh):
+        ret = {}
+        ret["unk_1"] = read_struct("<I", fh)[0]
+        ret["unk_2"] = read_struct("<I", fh)[0]
+        ret["floats"] = read_struct("<16f", fh)  # ???
+        ret["cmsh_1"] = [self.parse_block(fh) for _ in range(2)]
+        ret["num_submeshes"] = read_struct("<I", fh)[0]
+        ret["cmsh_2"] = [self.parse_block(fh) for _ in range(ret["num_submeshes"])]
+        return ret
+
+    def CMSH(self, fh):
+        return {"ret": len(fh.read())}
 
     # def EMI(self, fh):
     #     return len(fh.read())
@@ -218,30 +231,14 @@ class Parser:
 
 basedir = r"D:/Games/Deep Silver/Scrapland/extracted/Data.packed"
 
-files = [
-    r"Models/Chars/Dtritus/Dtritus.sm3",
-    r"Models/Elements/AnilloEstructuraA/AnilloEstructuraA.SM3",
-    r"models/elements/antenaa/antenaa.lod1.sm3",
-    # r"models/elements/abshield/anm/loop.cm3",
-    # r"levels/fake/map/map3d.amc",
-    # r"levels/shipedit/map/map3d.dum",
-    # r"levels/menu/map/map3d.emi",
-    r"Models/Skies/Menu/Sky.SM3",
-    r"Levels/Menu/Map/Map3D.SM3",
-    r"Models/Elements/AnilloEstructuraD/AnilloEstructuraD.LOD1.SM3",
-    # r"levels/menu/map/map3d.amc",
-    # r"levels/menu/map/map3d.dum",
-    # r"levels/menu/map/scenecamera/anm/loop.cm3",
-    r"models/chars/boss/boss.sm3",
-    # r"models/chars/boss/anm/boss_walk.cm3",
-]
-
 filt = [s.lower() for s in sys.argv[1:]]
 
-for root, folders, files in os.walk(basedir):
+for root, _, files in os.walk(basedir):
     for file in files:
-        path = os.path.join(root, file).replace("\\","/")
-        if not path.lower().endswith(".dum".lower()):
+        path = os.path.join(root, file).replace("\\", "/")
+        if not path.lower().endswith(".amc".lower()):
+            continue
+        if "menu" not in path.lower():
             continue
         print("Parsing", path)
         p = Parser(debug=True)
@@ -252,4 +249,3 @@ for root, folders, files in os.walk(basedir):
                     break
                 pprint(parsed, compact=False, indent=4)
         print("#" * 50)
-

tools/parse_save.py

@@ -0,0 +1,21 @@
+import sys
+from construct import *
+from pprint import pprint
+
+ScrapSaveVar = Struct(
+    "name" / PascalString(Int32ul, encoding="utf-8"),
+    "data" / PascalString(Int32ul, encoding="utf-8"),
+)
+ScrapSave = "ScarpSaveGame" / Struct(
+    "title" / PascalString(Int32ul, encoding="utf-8"),
+    "id" / PascalString(Int32ul, encoding="utf-8"),
+    "data" / PrefixedArray(Int32ul, ScrapSaveVar),
+    Terminated,
+)
+with open(sys.argv[1], "rb") as sav_file:
+    save = ScrapSave.parse_stream(sav_file)
+    print("ID:", save.id)
+    print("Title:", save.title)
+    for var in save.data:
+        print("   {}: {}".format(var.name, var.data))
+

tools/scrapper.py

@@ -3,29 +3,18 @@ from collections import OrderedDict
 import glob
 import os
 import shutil
-from construct import (
-    Struct,
-    PascalString,
-    Int32ul,
-    Lazy,
-    Pointer,
-    Bytes,
-    this,
-    PrefixedArray,
-    Const,
-    Debugger
-)
+from construct import *
 from tqdm import tqdm
 
+setglobalstringencoding(None)
+
 ScrapFile = Struct(
-    "path" / PascalString(Int32ul, encoding="ascii"),
+    "path" / PascalString(Int32ul),
     "size" / Int32ul,
     "offset" / Int32ul,
-    "data" / Lazy(Pointer(this.offset, Bytes(this.size))),
-)
-DummyFile = Struct(
-    "path" / PascalString(Int32ul, encoding="u8"), "size" / Int32ul, "offset" / Int32ul
+    "data" / OnDemandPointer(this.offset, Bytes(this.size)),
 )
+DummyFile = Struct("path" / PascalString(Int32ul), "size" / Int32ul, "offset" / Int32ul)
 
 PackedHeader = Struct(
     Const(b"BFPK"), Const(b"\0\0\0\0"), "files" / PrefixedArray(Int32ul, ScrapFile)