The numbers of the Diamond 3.7 executable and of Football Limited stay built in, the game texts go into data/openbmh.dat, which the release ships next to the programs (found in OPENBMH_DAT, the game folder, the program folder, the current folder). Without the file those texts are empty. A replay of a recorded session gives the same output as with the real BMH.exe when the file is there, and differs only in the seven text lines when it is missing. week_check no longer counts its probe re-rating of the team as draws of the port (the apparent deviation from minute 10 of the DOSBox recording was this measuring error). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
265 lines
12 KiB
Python
265 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Generates the embedded game data of OpenBMH, so that the game runs from the asset files alone (no BMH.exe, no
|
|
FLTD.EXE): src/formats/diamond_constants.inc (German, Diamond 3.7) and src/formats/fltd_constants.inc (English
|
|
Football Limited) hold every non-zero byte of the numbers the engine reads - tables, rates, positions, prices - as runs of
|
|
bytes; zero runs are not stored. The game texts (live commentary, messages) are NOT in the source: they go into the data
|
|
file data/openbmh.dat that ships with the release (formats/diamond_image.hpp); the .inc files only list where they are.
|
|
After regenerating, put the new SHA-256 of data/openbmh.dat into REVIEWED_BINARY_FILES of tools/public_release/check_public_tree.py.
|
|
|
|
German (a Diamond 3.7 BMH.exe, read here and never copied into the repository):
|
|
python generate.py german --exe BMH.exe --trace trace1.txt [--trace trace2.txt ...] --out ../../src/formats/diamond_constants.inc
|
|
The file backed part of the data object, plus the few bytes of the code object that the program reads as numbers or
|
|
tables (instruction immediates such as the calendar setup). The code bytes come from the read traces of OPENBMH_LE_TRACE
|
|
(formats/le.hpp: run the data dependent tests and scenarios with the variable set to a file; the lines are
|
|
"<flag> <start> <length>" in hex, flag 1 = number read, 2 = string read, 4 = start of a bulk read) and from a window
|
|
at every layout constant that names a place in the code objects.
|
|
|
|
English (the German layout image of FLTD.EXE, the cached "BMH.exe" of an English game that translate_fltd() made):
|
|
python generate.py english --image BMH.exe --out ../../src/formats/fltd_constants.inc
|
|
Every non-zero byte of the image.
|
|
"""
|
|
import argparse
|
|
import re
|
|
import struct
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent))
|
|
import classify # noqa: E402
|
|
|
|
DATA_LO, DATA_PAGES = 0xA0000, 12 # the initialised pages of the data object (file backed)
|
|
DATA_HI = DATA_LO + DATA_PAGES * 4096
|
|
CODE = (0x10000, 0x10000 + 438272)
|
|
PATCH = (0x80000, 0x81000)
|
|
EN_LO, EN_HI = 0x6B000, 0x776A8 # the English data segment, also placed at its own addresses
|
|
|
|
|
|
def load_le(path):
|
|
"""The loaded image of a DOS/4GW LE executable (objects at their link addresses, internal fixups applied)."""
|
|
raw = Path(path).read_bytes()
|
|
u16 = lambda o: struct.unpack_from("<H", raw, o)[0]
|
|
u32 = lambda o: struct.unpack_from("<I", raw, o)[0]
|
|
le = u32(0x3C)
|
|
if u16(le) != 0x454C:
|
|
raise SystemExit("no LE header")
|
|
h = lambda off: u32(le + off)
|
|
pages, page_size, last_page = h(0x14), h(0x28), h(0x2C)
|
|
object_table, page_map = le + h(0x40), le + h(0x48)
|
|
fixup_pages, fixup_records = le + h(0x68), le + h(0x6C)
|
|
objects, data_pages = h(0x44), h(0x80)
|
|
obj, lo, hi = [], 0xFFFFFFFF, 0
|
|
for i in range(objects):
|
|
o = object_table + i * 24
|
|
ob = (u32(o), u32(o + 4), u32(o + 8), u32(o + 12), u32(o + 16)) # vsize, base, flags, page index, pages
|
|
obj.append(ob)
|
|
span = max(ob[0], ob[4] * page_size)
|
|
lo, hi = min(lo, ob[1]), max(hi, ob[1] + (span + page_size - 1) // page_size * page_size)
|
|
data = bytearray(hi - lo)
|
|
page_address = [0] * (pages + 1)
|
|
for vsize, base, flags, index, count in obj:
|
|
for p in range(count):
|
|
logical = index + p
|
|
e = page_map + (logical - 1) * 4
|
|
file_page = raw[e] << 16 | raw[e + 1] << 8 | raw[e + 2]
|
|
va = base + p * page_size
|
|
if logical <= pages:
|
|
page_address[logical] = va
|
|
if raw[e + 3] != 0:
|
|
continue
|
|
size = last_page if file_page == pages else page_size
|
|
src = data_pages + (file_page - 1) * page_size
|
|
size = min(size, len(raw) - src)
|
|
data[va - lo:va - lo + size] = raw[src:src + size]
|
|
|
|
def put(pos, fmt, v):
|
|
struct.pack_into(fmt, data, pos, v & ((1 << (8 * struct.calcsize(fmt))) - 1))
|
|
|
|
for pg in range(1, pages + 1):
|
|
r = fixup_records + u32(fixup_pages + (pg - 1) * 4)
|
|
end = fixup_records + u32(fixup_pages + pg * 4)
|
|
while r < end:
|
|
src, flags = raw[r], raw[r + 1]
|
|
r += 2
|
|
ftype, is_list = src & 0x0F, bool(src & 0x20)
|
|
count, source_offset = 1, 0
|
|
if is_list:
|
|
count = raw[r]; r += 1
|
|
else:
|
|
source_offset = struct.unpack_from("<h", raw, r)[0]; r += 2
|
|
if flags & 3:
|
|
raise SystemExit("external fixup")
|
|
if flags & 0x40:
|
|
target_object = u16(r); r += 2
|
|
else:
|
|
target_object = raw[r]; r += 1
|
|
target_offset = 0
|
|
if ftype != 0x02:
|
|
if flags & 0x10:
|
|
target_offset = u32(r); r += 4
|
|
else:
|
|
target_offset = u16(r); r += 2
|
|
offsets = []
|
|
if is_list:
|
|
for _ in range(count):
|
|
offsets.append(struct.unpack_from("<h", raw, r)[0]); r += 2
|
|
else:
|
|
offsets.append(source_offset)
|
|
target = obj[target_object - 1][1] + target_offset
|
|
for so in offsets:
|
|
sa = page_address[pg] + so
|
|
pos = sa - lo
|
|
if pos < 0 or pos + 4 > len(data):
|
|
continue
|
|
if ftype == 0x07: put(pos, "<I", target)
|
|
elif ftype == 0x08: put(pos, "<I", target - (sa + 4))
|
|
elif ftype == 0x05: put(pos, "<H", target)
|
|
elif ftype == 0x02: put(pos, "<H", target_object)
|
|
elif ftype == 0x06: put(pos, "<I", target); put(pos + 4, "<H", target_object)
|
|
return lo, data
|
|
|
|
|
|
def load_openbmh_image(path):
|
|
raw = Path(path).read_bytes()
|
|
if raw[:14] != b"OPENBMH-IMAGE\0":
|
|
raise SystemExit("not an OPENBMH-IMAGE file")
|
|
base, size = struct.unpack_from("<II", raw, 14)
|
|
data = raw[22:]
|
|
if len(data) != size:
|
|
raise SystemExit("OPENBMH-IMAGE: wrong size")
|
|
return base, bytearray(data)
|
|
|
|
|
|
def read_traces(paths):
|
|
numbers, bulk = [], []
|
|
for path in paths:
|
|
for line in Path(path).read_text().splitlines():
|
|
flag, start, length = (int(x, 16) for x in line.split())
|
|
if flag == 1:
|
|
numbers.append((start, length))
|
|
elif flag == 4:
|
|
bulk.append((start, length))
|
|
return numbers, bulk
|
|
|
|
|
|
def read_string_traces(paths):
|
|
out = []
|
|
for path in paths:
|
|
for line in Path(path).read_text().splitlines():
|
|
flag, start, length = (int(x, 16) for x in line.split())
|
|
if flag == 2:
|
|
out.append((start, length))
|
|
return out
|
|
|
|
|
|
def merge(ranges, gap=0):
|
|
out = []
|
|
for start, end in sorted((s, s + l) for s, l in ranges):
|
|
if out and start <= out[-1][1] + gap:
|
|
out[-1][1] = max(out[-1][1], end)
|
|
else:
|
|
out.append([start, end])
|
|
return out
|
|
|
|
|
|
def nonzero_runs(data, base, lo, hi, gap=8):
|
|
"""[start, end) runs that hold every non-zero byte of lo..hi; zero gaps up to `gap` bytes stay inside a run."""
|
|
runs, cur = [], None
|
|
for address in range(lo, hi):
|
|
if data[address - base]:
|
|
if cur and address - cur[1] <= gap:
|
|
cur[1] = address + 1
|
|
else:
|
|
cur = [address, address + 1]
|
|
runs.append(cur)
|
|
return runs
|
|
|
|
|
|
def write_inc(path, title, base, data, chunks, texts):
|
|
out = ["// SPDX-License-Identifier: GPL-3.0-only",
|
|
f"// Generated by tools/diamond_constants/generate.py from {title} - do not edit.",
|
|
"// Every non-zero byte of the numbers the engine reads (tables, rates, positions, prices) as runs; the game texts",
|
|
"// are not part of it: kTextRanges lists where they lie, their bytes come from openbmh.dat.",
|
|
"// Included inside a namespace by formats/diamond_image.cpp.",
|
|
f"inline constexpr uint32_t kImageBase = {base}, kImageSize = {len(data)};"]
|
|
total = 0
|
|
for i, (start, end) in enumerate(chunks):
|
|
body = data[start - base:end - base]
|
|
total += len(body)
|
|
rows = [", ".join(f"0x{b:02X}" for b in body[k:k + 24]) for k in range(0, len(body), 24)]
|
|
out.append(f"inline constexpr uint8_t kChunk{i}[] = {{\n " + ",\n ".join(rows) + "};")
|
|
out.append("inline constexpr Chunk kChunks[] = {")
|
|
for i, (start, end) in enumerate(chunks):
|
|
out.append(f" {{{start}, {end - start}, kChunk{i}}},")
|
|
out.append("};")
|
|
out.append("inline constexpr Range kTextRanges[] = {")
|
|
for start, length in texts:
|
|
out.append(f" {{{start}, {length}}},")
|
|
out.append("};")
|
|
Path(path).write_text("\n".join(out) + "\n")
|
|
print(f"{path}: {len(chunks)} chunks, {total} bytes of numbers; {len(texts)} text ranges, {sum(l for _, l in texts)} bytes of text")
|
|
|
|
|
|
def zeroed(data, base, texts):
|
|
out = bytearray(data)
|
|
for start, length in texts:
|
|
out[start - base:start - base + length] = bytes(length)
|
|
return out
|
|
|
|
|
|
def dat_set(edition, data, base, texts):
|
|
out = struct.pack("<II", edition, len(texts))
|
|
for start, length in texts:
|
|
out += struct.pack("<II", start, length)
|
|
for start, length in texts:
|
|
out += bytes(data[start - base:start - base + length])
|
|
return out
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--exe", required=True, help="a Diamond 3.7 BMH.exe")
|
|
ap.add_argument("--english-image", required=True, help="the cached German layout image of FLTD.EXE (OPENBMH-IMAGE)")
|
|
ap.add_argument("--trace", action="append", default=[])
|
|
ap.add_argument("--layout", default=str(Path(__file__).parents[2] / "src/formats/executable_layout.hpp"))
|
|
ap.add_argument("--out-dir", required=True, help="src/formats")
|
|
ap.add_argument("--dat", required=True, help="data/openbmh.dat")
|
|
args = ap.parse_args()
|
|
layout = Path(args.layout).read_text()
|
|
out_dir = Path(args.out_dir)
|
|
|
|
# German (Diamond 3.7)
|
|
base, data = load_le(args.exe)
|
|
numbers_read, bulk = read_traces(args.trace)
|
|
strings_read = [(s, l) for s, l in read_string_traces(args.trace) if DATA_LO <= s < DATA_HI]
|
|
pointers = [int(a, 16) for n, a in re.findall(r"inline constexpr auto (\w*TextPointer\w*) = 0x([0-9a-fA-F]+);", layout)]
|
|
texts_de = classify.text_ranges(data, base, [(DATA_LO, DATA_HI)], [(DATA_LO, DATA_HI)], [CODE, PATCH], pointers, strings_read)
|
|
wanted = []
|
|
for s, l in numbers_read:
|
|
if CODE[0] <= s < CODE[1] or PATCH[0] <= s < PATCH[1]:
|
|
wanted.append((max(s - 4, 0), l + 8))
|
|
for s, l in bulk:
|
|
if CODE[0] <= s < CODE[1] or PATCH[0] <= s < PATCH[1]:
|
|
wanted.append((s, 256))
|
|
for name, address in re.findall(r"inline constexpr auto (\w+) = 0x([0-9a-fA-F]+);", layout):
|
|
a = int(address, 16)
|
|
if CODE[0] <= a < CODE[1] or PATCH[0] <= a < PATCH[1]:
|
|
wanted.append((a, 16))
|
|
numbers_de = zeroed(data, base, texts_de)
|
|
chunks_de = sorted(nonzero_runs(numbers_de, base, DATA_LO, DATA_HI) + merge(wanted, 16))
|
|
write_inc(out_dir / "diamond_constants.inc", "BMH 2026-27 Diamond edition 3.7", base, numbers_de, chunks_de, texts_de)
|
|
|
|
# English (Football Limited, in the German address space)
|
|
ebase, edata = load_openbmh_image(args.english_image)
|
|
en_segment = (EN_LO, EN_HI)
|
|
texts_en = classify.text_ranges(edata, ebase, [(DATA_LO, DATA_HI), en_segment], [(DATA_LO, DATA_HI), en_segment], [], [], [])
|
|
numbers_en = zeroed(edata, ebase, texts_en)
|
|
write_inc(out_dir / "fltd_constants.inc", "the German layout image of FLTD.EXE (Football Limited)", ebase, numbers_en,
|
|
nonzero_runs(numbers_en, ebase, ebase, ebase + len(edata)), texts_en)
|
|
|
|
blob = b"OPENBMH-DAT\0" + struct.pack("<I", 2) + dat_set(1, data, base, texts_de) + dat_set(2, edata, ebase, texts_en)
|
|
Path(args.dat).write_bytes(blob)
|
|
print(f"{args.dat}: {len(blob)} bytes")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |