Move the game texts out of the source into openbmh.dat
The numbers of the Diamond 3.7 executable and of Football Limited stay built in, the game texts go into data/openbmh.dat, which the release ships next to the programs (found in OPENBMH_DAT, the game folder, the program folder, the current folder). Without the file those texts are empty. A replay of a recorded session gives the same output as with the real BMH.exe when the file is there, and differs only in the seven text lines when it is missing. week_check no longer counts its probe re-rating of the team as draws of the port (the apparent deviation from minute 10 of the DOSBox recording was this measuring error). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
524b9e4158
commit
9e593f921f
16 files changed
+2015
-1753
No files matched your search
@@ -0,0 +1,134 @@
|
||||
"""Finds the game texts in a German-layout image (a Diamond BMH.exe, or the translated English FLTD.EXE).
|
||||
|
||||
generate.py keeps the numbers of the image in the source files (formats/*_constants.inc) and puts the game texts into
|
||||
the data file openbmh.dat that ships with the release. A range counts as text when it is
|
||||
|
||||
* a NUL terminated string that a pointer variable of the data object (or of the English data segment) points to, or
|
||||
* a string that a 32 bit immediate of the code points to (a "push offset text"), or
|
||||
* a long printable run with a space (a sentence), or
|
||||
* a string the program read as a string while a trace was recorded (OPENBMH_LE_TRACE, kind 2).
|
||||
|
||||
Letter permutation tables of the cup draw ('abcdefghijklmnopqr', all letters different) and file names are numbers /
|
||||
names, not texts, and stay in the constants. Used by generate.py.
|
||||
"""
|
||||
import re
|
||||
import struct
|
||||
|
||||
PRINTABLE = set(range(0x20, 0x7F))
|
||||
UMLAUTS = {0x84, 0x94, 0x81, 0x8E, 0x99, 0x9A, 0xE1} # CP437 letters seen in character sets
|
||||
TEXT_BYTES = PRINTABLE | UMLAUTS
|
||||
FILE_NAME = re.compile(r"[A-Za-z0-9_./\\*?-]*[./\\][A-Za-z0-9_./\\*?-]*")
|
||||
|
||||
|
||||
def string_end(data, base, start):
|
||||
"""Address after the NUL of the string starting at `start`, or None when it is not a clean string."""
|
||||
i = start - base
|
||||
n = 0
|
||||
while i < len(data) and data[i] != 0:
|
||||
if data[i] not in TEXT_BYTES and data[i] < 0x20:
|
||||
return None
|
||||
i += 1
|
||||
n += 1
|
||||
if n > 400:
|
||||
return None
|
||||
return i + 1 + base if i < len(data) else None
|
||||
|
||||
|
||||
def is_permutation(text):
|
||||
letters = text.lower()
|
||||
return letters.isalpha() and len(set(letters)) == len(letters) and len(letters) in (4, 6, 10, 12, 16, 18, 20)
|
||||
|
||||
|
||||
def ascending(text, n=4):
|
||||
"""True when the text holds n or more ascending consecutive characters (a character set 'abcdef', not a word)."""
|
||||
run = 1
|
||||
for a, b in zip(text, text[1:]):
|
||||
run = run + 1 if ord(b) == ord(a) + 1 else 1
|
||||
if run >= n:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_text(text):
|
||||
if not re.search(r"[A-Za-z]{3}", text) or is_permutation(text) or text.startswith(("f\\", "Ã")) or FILE_NAME.fullmatch(text):
|
||||
return False
|
||||
# a single token in one case is a name or a table fragment (BLASTER, KRON_FBM, V3.7, 'njpmd' from a fixture row), not a
|
||||
# sentence or a label of the game; letter and character-set tables are not texts either
|
||||
if " " not in text and (text.isupper() or text.islower() or re.fullmatch(r"[A-Z][A-Za-z0-9_.]*\d[A-Za-z0-9_.]*", text)):
|
||||
return False
|
||||
if ascending(text) or sum(c.isalpha() or c == " " for c in text) < 0.6 * len(text):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def text_ranges(data, base, targets, sources, code, layout_pointers, traced_strings):
|
||||
"""data: the image bytes at address `base`. targets: [(lo, hi)] where texts may lie, sources: [(lo, hi)] where
|
||||
pointer variables lie, code: [(lo, hi)] where immediates are searched. Returns sorted (start, length) incl. the NUL."""
|
||||
found = {}
|
||||
|
||||
def in_targets(address):
|
||||
return any(lo <= address < hi for lo, hi in targets)
|
||||
|
||||
def add(start, end):
|
||||
if in_targets(start) and end > start:
|
||||
found[start] = max(found.get(start, 0), end - start)
|
||||
|
||||
def text_at(value):
|
||||
end = string_end(data, base, value)
|
||||
if end and end - value >= 4:
|
||||
text = bytes(data[value - base:end - 1 - base]).decode("latin1")
|
||||
if is_text(text):
|
||||
return end
|
||||
return None
|
||||
|
||||
# 1. strings that pointer variables point to
|
||||
for lo, hi in sources:
|
||||
for at in range(lo, hi - 3, 4):
|
||||
value = struct.unpack_from("<I", data, at - base)[0]
|
||||
if in_targets(value) and value != at:
|
||||
end = text_at(value)
|
||||
if end:
|
||||
add(value, end)
|
||||
for pointer in layout_pointers:
|
||||
if pointer - base < 0 or pointer - base + 4 > len(data):
|
||||
continue
|
||||
value = struct.unpack_from("<I", data, pointer - base)[0]
|
||||
if in_targets(value):
|
||||
end = string_end(data, base, value)
|
||||
if end:
|
||||
add(value, end)
|
||||
# 1b. strings the code points to: after the fixups a "push offset text" holds the address as a 32 bit immediate
|
||||
for lo, hi in code:
|
||||
for at in range(lo, hi - 3):
|
||||
value = struct.unpack_from("<I", data, at - base)[0]
|
||||
if in_targets(value) and data[value - 1 - base] == 0:
|
||||
end = text_at(value)
|
||||
if end:
|
||||
add(value, end)
|
||||
# 2. long printable runs with a space (sentences)
|
||||
for lo, hi in targets:
|
||||
i = lo - base
|
||||
while i < hi - base:
|
||||
if data[i] in PRINTABLE:
|
||||
j = i
|
||||
while j < hi - base and data[j] in TEXT_BYTES:
|
||||
j += 1
|
||||
text = bytes(data[i:j]).decode("latin1")
|
||||
if j - i >= 8 and " " in text and re.search(r"[A-Za-z]{3}", text):
|
||||
add(i + base, j + base + (1 if j < len(data) and data[j] == 0 else 0))
|
||||
i = j
|
||||
else:
|
||||
i += 1
|
||||
# 3. strings the program read while a trace was recorded
|
||||
for start, length in traced_strings:
|
||||
text = bytes(data[start - base:start - base + max(length - 1, 0)]).decode("latin1")
|
||||
if is_text(text):
|
||||
add(start, start + length)
|
||||
merged = []
|
||||
for start in sorted(found):
|
||||
end = start + found[start]
|
||||
if merged and start <= merged[-1][1]:
|
||||
merged[-1][1] = max(merged[-1][1], end)
|
||||
else:
|
||||
merged.append([start, end])
|
||||
return [(s, e - s) for s, e in merged]
|
||||
@@ -1,8 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generates the embedded game images of OpenBMH, so that the game runs from the asset files alone (no BMH.exe, no
|
||||
"""Generates the embedded game data of OpenBMH, so that the game runs from the asset files alone (no BMH.exe, no
|
||||
FLTD.EXE): src/formats/diamond_constants.inc (German, Diamond 3.7) and src/formats/fltd_constants.inc (English
|
||||
Football Limited). Both hold every non-zero byte of the data the engine reads - tables, rates, positions, prices and the
|
||||
game texts - as runs of bytes; zero runs are not stored.
|
||||
Football Limited) hold every non-zero byte of the numbers the engine reads - tables, rates, positions, prices - as runs of
|
||||
bytes; zero runs are not stored. The game texts (live commentary, messages) are NOT in the source: they go into the data
|
||||
file data/openbmh.dat that ships with the release (formats/diamond_image.hpp); the .inc files only list where they are.
|
||||
After regenerating, put the new SHA-256 of data/openbmh.dat into REVIEWED_BINARY_FILES of tools/public_release/check_public_tree.py.
|
||||
|
||||
German (a Diamond 3.7 BMH.exe, read here and never copied into the repository):
|
||||
python generate.py german --exe BMH.exe --trace trace1.txt [--trace trace2.txt ...] --out ../../src/formats/diamond_constants.inc
|
||||
@@ -19,12 +21,17 @@ English (the German layout image of FLTD.EXE, the cached "BMH.exe" of an English
|
||||
import argparse
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
import classify # noqa: E402
|
||||
|
||||
DATA_LO, DATA_PAGES = 0xA0000, 12 # the initialised pages of the data object (file backed)
|
||||
DATA_HI = DATA_LO + DATA_PAGES * 4096
|
||||
CODE = (0x10000, 0x10000 + 438272)
|
||||
PATCH = (0x80000, 0x81000)
|
||||
EN_LO, EN_HI = 0x6B000, 0x776A8 # the English data segment, also placed at its own addresses
|
||||
|
||||
|
||||
def load_le(path):
|
||||
@@ -134,6 +141,16 @@ def read_traces(paths):
|
||||
return numbers, bulk
|
||||
|
||||
|
||||
def read_string_traces(paths):
|
||||
out = []
|
||||
for path in paths:
|
||||
for line in Path(path).read_text().splitlines():
|
||||
flag, start, length = (int(x, 16) for x in line.split())
|
||||
if flag == 2:
|
||||
out.append((start, length))
|
||||
return out
|
||||
|
||||
|
||||
def merge(ranges, gap=0):
|
||||
out = []
|
||||
for start, end in sorted((s, s + l) for s, l in ranges):
|
||||
@@ -157,10 +174,11 @@ def nonzero_runs(data, base, lo, hi, gap=8):
|
||||
return runs
|
||||
|
||||
|
||||
def write_inc(path, title, base, data, chunks):
|
||||
def write_inc(path, title, base, data, chunks, texts):
|
||||
out = ["// SPDX-License-Identifier: GPL-3.0-only",
|
||||
f"// Generated by tools/diamond_constants/generate.py from {title} - do not edit.",
|
||||
"// Every non-zero byte of the data the engine reads (tables, rates, positions, prices, game texts), as runs.",
|
||||
"// Every non-zero byte of the numbers the engine reads (tables, rates, positions, prices) as runs; the game texts",
|
||||
"// are not part of it: kTextRanges lists where they lie, their bytes come from openbmh.dat.",
|
||||
"// Included inside a namespace by formats/diamond_image.cpp.",
|
||||
f"inline constexpr uint32_t kImageBase = {base}, kImageSize = {len(data)};"]
|
||||
total = 0
|
||||
@@ -173,48 +191,75 @@ def write_inc(path, title, base, data, chunks):
|
||||
for i, (start, end) in enumerate(chunks):
|
||||
out.append(f" {{{start}, {end - start}, kChunk{i}}},")
|
||||
out.append("};")
|
||||
out.append("inline constexpr Range kTextRanges[] = {")
|
||||
for start, length in texts:
|
||||
out.append(f" {{{start}, {length}}},")
|
||||
out.append("};")
|
||||
Path(path).write_text("\n".join(out) + "\n")
|
||||
print(f"{len(chunks)} chunks, {total} bytes")
|
||||
print(f"{path}: {len(chunks)} chunks, {total} bytes of numbers; {len(texts)} text ranges, {sum(l for _, l in texts)} bytes of text")
|
||||
|
||||
|
||||
def zeroed(data, base, texts):
|
||||
out = bytearray(data)
|
||||
for start, length in texts:
|
||||
out[start - base:start - base + length] = bytes(length)
|
||||
return out
|
||||
|
||||
|
||||
def dat_set(edition, data, base, texts):
|
||||
out = struct.pack("<II", edition, len(texts))
|
||||
for start, length in texts:
|
||||
out += struct.pack("<II", start, length)
|
||||
for start, length in texts:
|
||||
out += bytes(data[start - base:start - base + length])
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
sub = ap.add_subparsers(dest="mode", required=True)
|
||||
de = sub.add_parser("german")
|
||||
de.add_argument("--exe", required=True)
|
||||
de.add_argument("--trace", action="append", default=[])
|
||||
de.add_argument("--layout", default=str(Path(__file__).parents[2] / "src/formats/executable_layout.hpp"))
|
||||
de.add_argument("--out", required=True)
|
||||
en = sub.add_parser("english")
|
||||
en.add_argument("--image", required=True)
|
||||
en.add_argument("--out", required=True)
|
||||
ap.add_argument("--exe", required=True, help="a Diamond 3.7 BMH.exe")
|
||||
ap.add_argument("--english-image", required=True, help="the cached German layout image of FLTD.EXE (OPENBMH-IMAGE)")
|
||||
ap.add_argument("--trace", action="append", default=[])
|
||||
ap.add_argument("--layout", default=str(Path(__file__).parents[2] / "src/formats/executable_layout.hpp"))
|
||||
ap.add_argument("--out-dir", required=True, help="src/formats")
|
||||
ap.add_argument("--dat", required=True, help="data/openbmh.dat")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.mode == "english":
|
||||
base, data = load_openbmh_image(args.image)
|
||||
write_inc(args.out, "the German layout image of FLTD.EXE (Football Limited)", base, data,
|
||||
nonzero_runs(data, base, base, base + len(data)))
|
||||
return
|
||||
|
||||
base, data = load_le(args.exe)
|
||||
numbers, bulk = read_traces(args.trace)
|
||||
layout = Path(args.layout).read_text()
|
||||
# numbers read from the code or patch object: the bytes read, a margin, and a window after bulk pointers
|
||||
out_dir = Path(args.out_dir)
|
||||
|
||||
# German (Diamond 3.7)
|
||||
base, data = load_le(args.exe)
|
||||
numbers_read, bulk = read_traces(args.trace)
|
||||
strings_read = [(s, l) for s, l in read_string_traces(args.trace) if DATA_LO <= s < DATA_HI]
|
||||
pointers = [int(a, 16) for n, a in re.findall(r"inline constexpr auto (\w*TextPointer\w*) = 0x([0-9a-fA-F]+);", layout)]
|
||||
texts_de = classify.text_ranges(data, base, [(DATA_LO, DATA_HI)], [(DATA_LO, DATA_HI)], [CODE, PATCH], pointers, strings_read)
|
||||
wanted = []
|
||||
for s, l in numbers:
|
||||
for s, l in numbers_read:
|
||||
if CODE[0] <= s < CODE[1] or PATCH[0] <= s < PATCH[1]:
|
||||
wanted.append((max(s - 4, 0), l + 8))
|
||||
for s, l in bulk:
|
||||
if CODE[0] <= s < CODE[1] or PATCH[0] <= s < PATCH[1]:
|
||||
wanted.append((s, 256))
|
||||
# and, whatever the traces saw, a window at every layout constant that names a place in the code objects
|
||||
for name, address in re.findall(r"inline constexpr auto (\w+) = 0x([0-9a-fA-F]+);", layout):
|
||||
a = int(address, 16)
|
||||
if CODE[0] <= a < CODE[1] or PATCH[0] <= a < PATCH[1]:
|
||||
wanted.append((a, 16))
|
||||
chunks = sorted(nonzero_runs(data, base, DATA_LO, DATA_HI) + merge(wanted, 16))
|
||||
write_inc(args.out, "BMH 2026-27 Diamond edition 3.7", base, data, chunks)
|
||||
numbers_de = zeroed(data, base, texts_de)
|
||||
chunks_de = sorted(nonzero_runs(numbers_de, base, DATA_LO, DATA_HI) + merge(wanted, 16))
|
||||
write_inc(out_dir / "diamond_constants.inc", "BMH 2026-27 Diamond edition 3.7", base, numbers_de, chunks_de, texts_de)
|
||||
|
||||
# English (Football Limited, in the German address space)
|
||||
ebase, edata = load_openbmh_image(args.english_image)
|
||||
en_segment = (EN_LO, EN_HI)
|
||||
texts_en = classify.text_ranges(edata, ebase, [(DATA_LO, DATA_HI), en_segment], [(DATA_LO, DATA_HI), en_segment], [], [], [])
|
||||
numbers_en = zeroed(edata, ebase, texts_en)
|
||||
write_inc(out_dir / "fltd_constants.inc", "the German layout image of FLTD.EXE (Football Limited)", ebase, numbers_en,
|
||||
nonzero_runs(numbers_en, ebase, ebase, ebase + len(edata)), texts_en)
|
||||
|
||||
blob = b"OPENBMH-DAT\0" + struct.pack("<I", 2) + dat_set(1, data, base, texts_de) + dat_set(2, edata, ebase, texts_en)
|
||||
Path(args.dat).write_bytes(blob)
|
||||
print(f"{args.dat}: {len(blob)} bytes")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
main()
|
||||
@@ -11,7 +11,7 @@ import zipfile
|
||||
|
||||
def validate(directory, windows):
|
||||
products = {'OpenBMH.exe', 'openbmh_cli.exe'} if windows else {'OpenBMH', 'openbmh_cli'}
|
||||
documents = {'README.md', 'LICENSE', 'THIRD_PARTY_NOTICES.md', 'ACKNOWLEDGEMENTS.md', 'CONTRIBUTING.md'}
|
||||
documents = {'README.md', 'LICENSE', 'THIRD_PARTY_NOTICES.md', 'ACKNOWLEDGEMENTS.md', 'CONTRIBUTING.md', 'openbmh.dat'}
|
||||
licenses = {'README.md', 'LGPL-2.1.txt', 'Nuked-OPL3-notices.txt', 'LZMA-SDK.txt', 'SDL2-HIDAPI-BSD.txt',
|
||||
'SDL2-HIDAPI-options.txt', 'SDL2-HIDAPI-original.txt', 'SDL2-yuv2rgb.txt',
|
||||
'SDL2.txt', 'miniz-source-notices.txt', 'miniz.txt', 'nativefiledialog-extended.txt', 'stb_image.txt'}
|
||||
|
||||
@@ -31,7 +31,9 @@ ASSET_SUFFIXES = {
|
||||
}
|
||||
# Newly supplied project artwork and an unchanged MIT/public-domain decoder.
|
||||
# Exact hashes keep this exception from admitting original-game asset dumps.
|
||||
REVIEWED_BINARY_FILES = {"assets/start-screen.jpg": "28df5a72cd7b6951c9eebf30532ea9557007a46f0a73614ba22401c403e379dc"}
|
||||
# data/openbmh.dat: the game texts of the original executables (tools/diamond_constants/generate.py); regenerate = new hash here
|
||||
REVIEWED_BINARY_FILES = {"assets/start-screen.jpg": "28df5a72cd7b6951c9eebf30532ea9557007a46f0a73614ba22401c403e379dc",
|
||||
"data/openbmh.dat": "ae313060d3c34a9b45adae8565715618182137443ab20c1485b2da6e2c9e62a4"}
|
||||
REVIEWED_UPSTREAM_FILES = {"src/app/third_party/stb_image.h": "594c2fe35d49488b4382dbfaec8f98366defca819d916ac95becf3e75f4200b3"}
|
||||
PRIVATE_PREFIXES = (
|
||||
("data", "original"), ("docs", "manuals"), ("docs", "analyses"),
|
||||
|
||||
Reference in new issue
Block a user