Move the game texts out of the source into openbmh.dat

The numbers of the Diamond 3.7 executable and of Football Limited stay built in,
the game texts go into data/openbmh.dat, which the release ships next to the
programs (found in OPENBMH_DAT, the game folder, the program folder, the current
folder). Without the file those texts are empty. A replay of a recorded session
gives the same output as with the real BMH.exe when the file is there, and differs
only in the seven text lines when it is missing. week_check no longer counts its
probe re-rating of the team as draws of the port (the apparent deviation from
minute 10 of the DOSBox recording was this measuring error).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ClaudeandClaude Sonnet 5.5 committed 2026-11-08 14:32:27 +01:00
1 parent 524b9e4158
commit 9e593f921f
16 files changed
+2015 -1753

No files matched your search

+134
View File
@@ -0,0 +1,134 @@
"""Finds the game texts in a German-layout image (a Diamond BMH.exe, or the translated English FLTD.EXE).
generate.py keeps the numbers of the image in the source files (formats/*_constants.inc) and puts the game texts into
the data file openbmh.dat that ships with the release. A range counts as text when it is
* a NUL terminated string that a pointer variable of the data object (or of the English data segment) points to, or
* a string that a 32 bit immediate of the code points to (a "push offset text"), or
* a long printable run with a space (a sentence), or
* a string the program read as a string while a trace was recorded (OPENBMH_LE_TRACE, kind 2).
Letter permutation tables of the cup draw ('abcdefghijklmnopqr', all letters different) and file names are numbers /
names, not texts, and stay in the constants. Used by generate.py.
"""
import re
import struct
PRINTABLE = set(range(0x20, 0x7F))
UMLAUTS = {0x84, 0x94, 0x81, 0x8E, 0x99, 0x9A, 0xE1} # CP437 letters seen in character sets
TEXT_BYTES = PRINTABLE | UMLAUTS
FILE_NAME = re.compile(r"[A-Za-z0-9_./\\*?-]*[./\\][A-Za-z0-9_./\\*?-]*")
def string_end(data, base, start):
"""Address after the NUL of the string starting at `start`, or None when it is not a clean string."""
i = start - base
n = 0
while i < len(data) and data[i] != 0:
if data[i] not in TEXT_BYTES and data[i] < 0x20:
return None
i += 1
n += 1
if n > 400:
return None
return i + 1 + base if i < len(data) else None
def is_permutation(text):
letters = text.lower()
return letters.isalpha() and len(set(letters)) == len(letters) and len(letters) in (4, 6, 10, 12, 16, 18, 20)
def ascending(text, n=4):
"""True when the text holds n or more ascending consecutive characters (a character set 'abcdef', not a word)."""
run = 1
for a, b in zip(text, text[1:]):
run = run + 1 if ord(b) == ord(a) + 1 else 1
if run >= n:
return True
return False
def is_text(text):
if not re.search(r"[A-Za-z]{3}", text) or is_permutation(text) or text.startswith(("f\\", "Ã")) or FILE_NAME.fullmatch(text):
return False
# a single token in one case is a name or a table fragment (BLASTER, KRON_FBM, V3.7, 'njpmd' from a fixture row), not a
# sentence or a label of the game; letter and character-set tables are not texts either
if " " not in text and (text.isupper() or text.islower() or re.fullmatch(r"[A-Z][A-Za-z0-9_.]*\d[A-Za-z0-9_.]*", text)):
return False
if ascending(text) or sum(c.isalpha() or c == " " for c in text) < 0.6 * len(text):
return False
return True
def text_ranges(data, base, targets, sources, code, layout_pointers, traced_strings):
"""data: the image bytes at address `base`. targets: [(lo, hi)] where texts may lie, sources: [(lo, hi)] where
pointer variables lie, code: [(lo, hi)] where immediates are searched. Returns sorted (start, length) incl. the NUL."""
found = {}
def in_targets(address):
return any(lo <= address < hi for lo, hi in targets)
def add(start, end):
if in_targets(start) and end > start:
found[start] = max(found.get(start, 0), end - start)
def text_at(value):
end = string_end(data, base, value)
if end and end - value >= 4:
text = bytes(data[value - base:end - 1 - base]).decode("latin1")
if is_text(text):
return end
return None
# 1. strings that pointer variables point to
for lo, hi in sources:
for at in range(lo, hi - 3, 4):
value = struct.unpack_from("<I", data, at - base)[0]
if in_targets(value) and value != at:
end = text_at(value)
if end:
add(value, end)
for pointer in layout_pointers:
if pointer - base < 0 or pointer - base + 4 > len(data):
continue
value = struct.unpack_from("<I", data, pointer - base)[0]
if in_targets(value):
end = string_end(data, base, value)
if end:
add(value, end)
# 1b. strings the code points to: after the fixups a "push offset text" holds the address as a 32 bit immediate
for lo, hi in code:
for at in range(lo, hi - 3):
value = struct.unpack_from("<I", data, at - base)[0]
if in_targets(value) and data[value - 1 - base] == 0:
end = text_at(value)
if end:
add(value, end)
# 2. long printable runs with a space (sentences)
for lo, hi in targets:
i = lo - base
while i < hi - base:
if data[i] in PRINTABLE:
j = i
while j < hi - base and data[j] in TEXT_BYTES:
j += 1
text = bytes(data[i:j]).decode("latin1")
if j - i >= 8 and " " in text and re.search(r"[A-Za-z]{3}", text):
add(i + base, j + base + (1 if j < len(data) and data[j] == 0 else 0))
i = j
else:
i += 1
# 3. strings the program read while a trace was recorded
for start, length in traced_strings:
text = bytes(data[start - base:start - base + max(length - 1, 0)]).decode("latin1")
if is_text(text):
add(start, start + length)
merged = []
for start in sorted(found):
end = start + found[start]
if merged and start <= merged[-1][1]:
merged[-1][1] = max(merged[-1][1], end)
else:
merged.append([start, end])
return [(s, e - s) for s, e in merged]
+75 -30
View File
@@ -1,8 +1,10 @@
#!/usr/bin/env python3
"""Generates the embedded game images of OpenBMH, so that the game runs from the asset files alone (no BMH.exe, no
"""Generates the embedded game data of OpenBMH, so that the game runs from the asset files alone (no BMH.exe, no
FLTD.EXE): src/formats/diamond_constants.inc (German, Diamond 3.7) and src/formats/fltd_constants.inc (English
Football Limited). Both hold every non-zero byte of the data the engine reads - tables, rates, positions, prices and the
game texts - as runs of bytes; zero runs are not stored.
Football Limited) hold every non-zero byte of the numbers the engine reads - tables, rates, positions, prices - as runs of
bytes; zero runs are not stored. The game texts (live commentary, messages) are NOT in the source: they go into the data
file data/openbmh.dat that ships with the release (formats/diamond_image.hpp); the .inc files only list where they are.
After regenerating, put the new SHA-256 of data/openbmh.dat into REVIEWED_BINARY_FILES of tools/public_release/check_public_tree.py.
German (a Diamond 3.7 BMH.exe, read here and never copied into the repository):
python generate.py german --exe BMH.exe --trace trace1.txt [--trace trace2.txt ...] --out ../../src/formats/diamond_constants.inc
@@ -19,12 +21,17 @@ English (the German layout image of FLTD.EXE, the cached "BMH.exe" of an English
import argparse
import re
import struct
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
import classify # noqa: E402
DATA_LO, DATA_PAGES = 0xA0000, 12 # the initialised pages of the data object (file backed)
DATA_HI = DATA_LO + DATA_PAGES * 4096
CODE = (0x10000, 0x10000 + 438272)
PATCH = (0x80000, 0x81000)
EN_LO, EN_HI = 0x6B000, 0x776A8 # the English data segment, also placed at its own addresses
def load_le(path):
@@ -134,6 +141,16 @@ def read_traces(paths):
return numbers, bulk
def read_string_traces(paths):
out = []
for path in paths:
for line in Path(path).read_text().splitlines():
flag, start, length = (int(x, 16) for x in line.split())
if flag == 2:
out.append((start, length))
return out
def merge(ranges, gap=0):
out = []
for start, end in sorted((s, s + l) for s, l in ranges):
@@ -157,10 +174,11 @@ def nonzero_runs(data, base, lo, hi, gap=8):
return runs
def write_inc(path, title, base, data, chunks):
def write_inc(path, title, base, data, chunks, texts):
out = ["// SPDX-License-Identifier: GPL-3.0-only",
f"// Generated by tools/diamond_constants/generate.py from {title} - do not edit.",
"// Every non-zero byte of the data the engine reads (tables, rates, positions, prices, game texts), as runs.",
"// Every non-zero byte of the numbers the engine reads (tables, rates, positions, prices) as runs; the game texts",
"// are not part of it: kTextRanges lists where they lie, their bytes come from openbmh.dat.",
"// Included inside a namespace by formats/diamond_image.cpp.",
f"inline constexpr uint32_t kImageBase = {base}, kImageSize = {len(data)};"]
total = 0
@@ -173,48 +191,75 @@ def write_inc(path, title, base, data, chunks):
for i, (start, end) in enumerate(chunks):
out.append(f" {{{start}, {end - start}, kChunk{i}}},")
out.append("};")
out.append("inline constexpr Range kTextRanges[] = {")
for start, length in texts:
out.append(f" {{{start}, {length}}},")
out.append("};")
Path(path).write_text("\n".join(out) + "\n")
print(f"{len(chunks)} chunks, {total} bytes")
print(f"{path}: {len(chunks)} chunks, {total} bytes of numbers; {len(texts)} text ranges, {sum(l for _, l in texts)} bytes of text")
def zeroed(data, base, texts):
out = bytearray(data)
for start, length in texts:
out[start - base:start - base + length] = bytes(length)
return out
def dat_set(edition, data, base, texts):
out = struct.pack("<II", edition, len(texts))
for start, length in texts:
out += struct.pack("<II", start, length)
for start, length in texts:
out += bytes(data[start - base:start - base + length])
return out
def main():
ap = argparse.ArgumentParser()
sub = ap.add_subparsers(dest="mode", required=True)
de = sub.add_parser("german")
de.add_argument("--exe", required=True)
de.add_argument("--trace", action="append", default=[])
de.add_argument("--layout", default=str(Path(__file__).parents[2] / "src/formats/executable_layout.hpp"))
de.add_argument("--out", required=True)
en = sub.add_parser("english")
en.add_argument("--image", required=True)
en.add_argument("--out", required=True)
ap.add_argument("--exe", required=True, help="a Diamond 3.7 BMH.exe")
ap.add_argument("--english-image", required=True, help="the cached German layout image of FLTD.EXE (OPENBMH-IMAGE)")
ap.add_argument("--trace", action="append", default=[])
ap.add_argument("--layout", default=str(Path(__file__).parents[2] / "src/formats/executable_layout.hpp"))
ap.add_argument("--out-dir", required=True, help="src/formats")
ap.add_argument("--dat", required=True, help="data/openbmh.dat")
args = ap.parse_args()
if args.mode == "english":
base, data = load_openbmh_image(args.image)
write_inc(args.out, "the German layout image of FLTD.EXE (Football Limited)", base, data,
nonzero_runs(data, base, base, base + len(data)))
return
base, data = load_le(args.exe)
numbers, bulk = read_traces(args.trace)
layout = Path(args.layout).read_text()
# numbers read from the code or patch object: the bytes read, a margin, and a window after bulk pointers
out_dir = Path(args.out_dir)
# German (Diamond 3.7)
base, data = load_le(args.exe)
numbers_read, bulk = read_traces(args.trace)
strings_read = [(s, l) for s, l in read_string_traces(args.trace) if DATA_LO <= s < DATA_HI]
pointers = [int(a, 16) for n, a in re.findall(r"inline constexpr auto (\w*TextPointer\w*) = 0x([0-9a-fA-F]+);", layout)]
texts_de = classify.text_ranges(data, base, [(DATA_LO, DATA_HI)], [(DATA_LO, DATA_HI)], [CODE, PATCH], pointers, strings_read)
wanted = []
for s, l in numbers:
for s, l in numbers_read:
if CODE[0] <= s < CODE[1] or PATCH[0] <= s < PATCH[1]:
wanted.append((max(s - 4, 0), l + 8))
for s, l in bulk:
if CODE[0] <= s < CODE[1] or PATCH[0] <= s < PATCH[1]:
wanted.append((s, 256))
# and, whatever the traces saw, a window at every layout constant that names a place in the code objects
for name, address in re.findall(r"inline constexpr auto (\w+) = 0x([0-9a-fA-F]+);", layout):
a = int(address, 16)
if CODE[0] <= a < CODE[1] or PATCH[0] <= a < PATCH[1]:
wanted.append((a, 16))
chunks = sorted(nonzero_runs(data, base, DATA_LO, DATA_HI) + merge(wanted, 16))
write_inc(args.out, "BMH 2026-27 Diamond edition 3.7", base, data, chunks)
numbers_de = zeroed(data, base, texts_de)
chunks_de = sorted(nonzero_runs(numbers_de, base, DATA_LO, DATA_HI) + merge(wanted, 16))
write_inc(out_dir / "diamond_constants.inc", "BMH 2026-27 Diamond edition 3.7", base, numbers_de, chunks_de, texts_de)
# English (Football Limited, in the German address space)
ebase, edata = load_openbmh_image(args.english_image)
en_segment = (EN_LO, EN_HI)
texts_en = classify.text_ranges(edata, ebase, [(DATA_LO, DATA_HI), en_segment], [(DATA_LO, DATA_HI), en_segment], [], [], [])
numbers_en = zeroed(edata, ebase, texts_en)
write_inc(out_dir / "fltd_constants.inc", "the German layout image of FLTD.EXE (Football Limited)", ebase, numbers_en,
nonzero_runs(numbers_en, ebase, ebase, ebase + len(edata)), texts_en)
blob = b"OPENBMH-DAT\0" + struct.pack("<I", 2) + dat_set(1, data, base, texts_de) + dat_set(2, edata, ebase, texts_en)
Path(args.dat).write_bytes(blob)
print(f"{args.dat}: {len(blob)} bytes")
if __name__ == "__main__":
main()
main()