Move the game texts out of the source into openbmh.dat

The numbers of the Diamond 3.7 executable and of Football Limited stay built in,
the game texts go into data/openbmh.dat, which the release ships next to the
programs (found in OPENBMH_DAT, the game folder, the program folder, the current
folder). Without the file those texts are empty. A replay of a recorded session
gives the same output as with the real BMH.exe when the file is there, and differs
only in the seven text lines when it is missing. week_check no longer counts its
probe re-rating of the team as draws of the port (the apparent deviation from
minute 10 of the DOSBox recording was this measuring error).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ClaudeandClaude Sonnet 5.5 committed 2026-11-08 14:32:27 +01:00
1 parent 524b9e4158
commit 9e593f921f
16 files changed
+2015 -1753

No files matched your search

+11 -4
View File
@@ -1087,9 +1087,16 @@ set(BMH_TEST_SPORTSCHAU_APP_TESTS_INPUT_1 "" CACHE PATH "External input 1 for sp
bmh_add_data_test(NAME sportschau_app_tests REQUIRED_OPTIONS BMH_TEST_SPORTSCHAU_APP_TESTS_INPUT_1 COMMAND sportschau_app_tests "${BMH_DATA}"
"${BMH_TEST_SPORTSCHAU_APP_TESTS_INPUT_1}")
# The runtime component deliberately contains only the two products and notices.
# openbmh.dat (the game texts of the original executables, tools/diamond_constants) ships next to the programs and is copied
# next to the built programs as well, where the game looks for it.
foreach(product IN ITEMS openbmh_play openbmh_cli)
add_custom_command(TARGET ${product} POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_CURRENT_SOURCE_DIR}/data/openbmh.dat" "$<TARGET_FILE_DIR:${product}>")
endforeach()
# The runtime component deliberately contains only the two products, openbmh.dat and notices.
install(TARGETS openbmh_play openbmh_cli RUNTIME DESTINATION . COMPONENT Runtime)
install(FILES README.md LICENSE THIRD_PARTY_NOTICES.md ACKNOWLEDGEMENTS.md CONTRIBUTING.md
install(FILES README.md LICENSE THIRD_PARTY_NOTICES.md ACKNOWLEDGEMENTS.md CONTRIBUTING.md data/openbmh.dat
DESTINATION . COMPONENT Runtime)
install(DIRECTORY docs/ DESTINATION docs COMPONENT Runtime)
if(MSVC)
@@ -1194,8 +1201,8 @@ target_link_libraries(transfer_seller_actions_tests PRIVATE bmhcore)
add_test(NAME transfer_seller_actions_tests COMMAND transfer_seller_actions_tests)
add_executable(diamond_image_tests tests/diamond_image_tests.cpp)
target_link_libraries(diamond_image_tests PRIVATE bmhcore)
add_test(NAME diamond_image_tests COMMAND diamond_image_tests)
bmh_add_data_test(NAME diamond_image_data_tests COMMAND diamond_image_tests "${BMH_DATA}")
add_test(NAME diamond_image_tests COMMAND diamond_image_tests - "${CMAKE_CURRENT_SOURCE_DIR}/data/openbmh.dat")
bmh_add_data_test(NAME diamond_image_data_tests COMMAND diamond_image_tests "${BMH_DATA}" "${CMAKE_CURRENT_SOURCE_DIR}/data/openbmh.dat")
add_executable(search_profile_tests tests/search_profile_tests.cpp)
target_link_libraries(search_profile_tests PRIVATE bmhcore)
add_test(NAME search_profile_tests COMMAND search_profile_tests)
+1 -1
View File
@@ -15,7 +15,7 @@ This is an independent community project, with no affiliation with or endorsemen
Check the [releases page](https://github.com/Weyoune/openbmh/releases) for available packages and their platform notes. A Windows package uses the name `OpenBMH-<version>-windows-x64.zip`. Availability and native validation are stated for each release; source build instructions do not establish platform support on their own.
1. Have your own supported game installation available. OpenBMH needs its asset files (texts, master database, pictures, fonts, sounds); neither `BMH.EXE` nor `FLTD.EXE` is required: the numbers and texts of the Diamond 3.7 executable and of the English Football Limited are built in.
1. Have your own supported game installation available. OpenBMH needs its asset files (texts, master database, pictures, fonts, sounds); neither `BMH.EXE` nor `FLTD.EXE` is required: the numbers of the Diamond 3.7 executable and of the English Football Limited are built in, and their game texts (live commentary, messages) are in `openbmh.dat`, which ships next to the programs in the release package (without that file those texts are empty).
2. Extract the OpenBMH package into a separate directory, or alongside the game files.
3. Start `OpenBMH` (`OpenBMH.exe` on Windows). Select the game directory (or `BMH.EXE`) when prompted. A ZIP or 7z containing one game installation can also be selected.
4. A BMH.EXE (or FLTD.EXE) that stays in the game directory is used as before; it carries its own edition's numbers and texts.
BIN
View File
Binary file not shown.
File diff suppressed because it is too large. Load diff
+129 -8
View File
@@ -1,16 +1,32 @@
// SPDX-License-Identifier: GPL-3.0-only
// Implementation of diamond_image.hpp: the images built from the embedded data of the generated .inc files.
// Implementation of diamond_image.hpp: the images built from the embedded numbers of the generated .inc files plus the
// texts of openbmh.dat.
#include "formats/diamond_image.hpp"
#include <algorithm>
#include <cctype>
#include <cstdlib>
#include <cstring>
#include <string>
#include <system_error>
#ifdef _WIN32
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#elif defined(__APPLE__)
#include <mach-o/dyld.h>
#else
#include <unistd.h>
#endif
namespace bmh {
namespace {
struct Chunk { uint32_t address; uint32_t size; const uint8_t* bytes; };
struct Range { uint32_t address; uint32_t length; };
namespace german {
#include "formats/diamond_constants.inc"
@@ -19,6 +35,19 @@ namespace english {
#include "formats/fltd_constants.inc"
} // namespace english
constexpr char kMagic[12] = "OPENBMH-DAT"; // 11 characters and the NUL
constexpr char kFileName[] = "openbmh.dat";
std::string lower(std::string s) {
for (char& c : s) c = char(std::tolower(static_cast<unsigned char>(c)));
return s;
}
uint32_t word(const Bytes& b, size_t at) {
if (at + 4 > b.size()) throw FormatError("openbmh.dat: truncated");
return uint32_t(b[at]) | uint32_t(b[at + 1]) << 8 | uint32_t(b[at + 2]) << 16 | uint32_t(b[at + 3]) << 24;
}
LeImage build(uint32_t base, uint32_t size, const Chunk* first, const Chunk* last) {
Bytes bytes(size, 0);
for (const Chunk* chunk = first; chunk != last; ++chunk)
@@ -26,19 +55,111 @@ LeImage build(uint32_t base, uint32_t size, const Chunk* first, const Chunk* las
return LeImage::from_bytes(base, std::move(bytes));
}
std::string lower(std::string s) {
for (char& c : s) c = char(std::tolower(static_cast<unsigned char>(c)));
return s;
std::filesystem::path executable_directory() {
std::error_code ec;
#ifdef _WIN32
wchar_t buffer[32768];
const DWORD n = GetModuleFileNameW(nullptr, buffer, DWORD(std::size(buffer)));
if (n == 0 || n >= std::size(buffer)) return {};
return std::filesystem::path(buffer).parent_path();
#elif defined(__APPLE__)
char buffer[4096];
uint32_t size = sizeof buffer;
if (_NSGetExecutablePath(buffer, &size) != 0) return {};
return std::filesystem::weakly_canonical(buffer, ec).parent_path();
#else
char buffer[4096];
const ssize_t n = readlink("/proc/self/exe", buffer, sizeof buffer - 1);
if (n <= 0) return {};
buffer[n] = 0;
return std::filesystem::path(buffer).parent_path();
#endif
}
std::filesystem::path file_in(const std::filesystem::path& directory) {
if (directory.empty()) return {};
std::error_code ec;
for (const auto& entry : std::filesystem::directory_iterator(directory, ec))
if (entry.is_regular_file(ec) && lower(entry.path().filename().string()) == kFileName) return entry.path();
return {};
}
void add_texts(LeImage& image, int edition, const std::filesystem::path& game_directory) {
const auto file = find_texts_file(game_directory);
if (file.empty()) return;
try {
apply_texts(image, edition, read_file(file));
} catch (const std::exception&) {
// a damaged texts file leaves the texts empty; the game itself still runs
}
}
} // namespace
LeImage diamond_constants_image() {
return build(german::kImageBase, german::kImageSize, std::begin(german::kChunks), std::end(german::kChunks));
std::filesystem::path find_texts_file(const std::filesystem::path& game_directory) {
if (const char* env = std::getenv("OPENBMH_DAT"); env && *env) {
std::error_code ec;
if (std::filesystem::is_regular_file(env, ec)) return env;
}
for (const auto& directory : {game_directory, executable_directory(), std::filesystem::path(".")})
if (auto file = file_in(directory); !file.empty()) return file;
return {};
}
LeImage english_constants_image() {
return build(english::kImageBase, english::kImageSize, std::begin(english::kChunks), std::end(english::kChunks));
std::string texts_signature(const std::filesystem::path& game_directory) {
const auto file = find_texts_file(game_directory);
if (file.empty()) return {};
try {
const Bytes data = read_file(file);
uint64_t hash = 1469598103934665603ull; // FNV-1a
for (uint8_t b : data) hash = (hash ^ b) * 1099511628211ull;
return std::to_string(data.size()) + "-" + std::to_string(hash);
} catch (const std::exception&) {
return {};
}
}
bool apply_texts(LeImage& image, int edition, const Bytes& file) {
if (file.size() < sizeof kMagic + 4 || !std::equal(kMagic, kMagic + sizeof kMagic, file.begin()))
throw FormatError("openbmh.dat: not an OpenBMH data file");
const Range* known = edition == 1 ? std::begin(german::kTextRanges) : std::begin(english::kTextRanges);
const Range* known_end = edition == 1 ? std::end(german::kTextRanges) : std::end(english::kTextRanges);
const uint32_t sets = word(file, sizeof kMagic);
if (sets > 16) throw FormatError("openbmh.dat: too many sets");
size_t at = sizeof kMagic + 4;
bool found = false;
for (uint32_t s = 0; s < sets; ++s) {
const uint32_t id = word(file, at), count = word(file, at + 4);
if (count > 65536) throw FormatError("openbmh.dat: too many ranges");
const size_t table = at + 8;
size_t data = table + size_t(count) * 8;
for (uint32_t n = 0; n < count; ++n) {
const uint32_t address = word(file, table + n * 8ull), length = word(file, table + n * 8ull + 4);
if (data + length > file.size()) throw FormatError("openbmh.dat: truncated");
if (int(id) == edition) {
const bool allowed = std::any_of(known, known_end,
[&](const Range& r) { return r.address == address && length <= r.length; });
if (!allowed) throw FormatError("openbmh.dat: a range is not a text of the game");
image.patch(address, file.data() + data, length);
found = true;
}
data += length;
}
at = data;
}
return found;
}
LeImage diamond_constants_image(const std::filesystem::path& game_directory) {
LeImage image = build(german::kImageBase, german::kImageSize, std::begin(german::kChunks), std::end(german::kChunks));
add_texts(image, 1, game_directory);
return image;
}
LeImage english_constants_image(const std::filesystem::path& game_directory) {
LeImage image = build(english::kImageBase, english::kImageSize, std::begin(english::kChunks), std::end(english::kChunks));
add_texts(image, 2, game_directory);
return image;
}
bool is_executable_name(const std::filesystem::path& file_name) { return lower(file_name.string()) == "bmh.exe"; }
+27 -8
View File
@@ -1,7 +1,9 @@
// SPDX-License-Identifier: GPL-3.0-only
// The game without BMH.exe and FLTD.EXE: OpenBMH carries the numbers and texts of the Diamond 3.7 executable (tables,
// rates, positions, prices, live commentary, messages; the generated formats/diamond_constants.inc) and of the English
// Football Limited (formats/fltd_constants.inc) and needs only the asset files of the game.
// The game without BMH.exe and FLTD.EXE: OpenBMH carries the numbers of the Diamond 3.7 executable (tables, rates,
// positions, prices; the generated formats/diamond_constants.inc) and of the English Football Limited
// (formats/fltd_constants.inc) and needs only the asset files of the game. The game texts that live in the executables
// (live commentary, messages) are not part of the source: they are in the data file openbmh.dat, which ships with the
// release next to the programs (tools/diamond_constants/generate.py makes it). Without that file those texts are empty.
//
// load_le() returns diamond_constants_image() for a BMH.exe that does not exist, so every reader of the executable image
// works unchanged. A BMH.exe that is there is used as before (it carries its own edition's numbers and texts). For an
@@ -10,18 +12,35 @@
#pragma once
#include <filesystem>
#include <string>
#include "formats/binary.hpp"
#include "formats/le.hpp"
namespace bmh {
// The German (Diamond 3.7) image built from the embedded data.
LeImage diamond_constants_image();
// openbmh.dat: "OPENBMH-DAT" NUL, u32 set count, then per set u32 edition (1 German, 2 English), u32 range count, count
// times (u32 address, u32 length), then the bytes of all ranges. A range must be one of the edition's text ranges, so
// the file can never change a number of the image.
//
// The file is looked for in: the environment variable OPENBMH_DAT, the folder of the game data, the folder of the
// running program, the current folder. Returns an empty path when there is none.
std::filesystem::path find_texts_file(const std::filesystem::path& game_directory);
// The English Football Limited image in the German address space, built from the embedded data (what
// launch::translate_fltd() makes from FLTD.EXE).
LeImage english_constants_image();
// A short text that changes when the texts file changes (empty without a file): used to rebuild cached English games.
std::string texts_signature(const std::filesystem::path& game_directory);
// The German (Diamond 3.7) image built from the embedded numbers plus the texts of openbmh.dat (found for
// `game_directory`).
LeImage diamond_constants_image(const std::filesystem::path& game_directory = {});
// The English Football Limited image in the German address space (what launch::translate_fltd() makes from
// FLTD.EXE), built from the embedded numbers plus the texts of openbmh.dat.
LeImage english_constants_image(const std::filesystem::path& game_directory = {});
// Puts the texts of an openbmh.dat file (edition 1 or 2) into `image`. Throws FormatError for malformed data or for a
// range that is not a text range of that edition. Returns false when the file has no set of that edition.
bool apply_texts(LeImage& image, int edition, const Bytes& file);
// Whether `file_name` is the name of the executable (BMH.exe in any case).
bool is_executable_name(const std::filesystem::path& file_name);
File diff suppressed because it is too large. Load diff
+3 -3
View File
@@ -99,12 +99,12 @@ void save_image(const LeImage& image, const std::filesystem::path& path) {
LeImage load_le(const std::filesystem::path& path) {
le_trace::enable_from_environment();
// The game runs from its asset files alone: for a BMH.exe that is not there the embedded numbers and texts of the
// Diamond 3.7 executable stand in for it.
// The game runs from its asset files alone: for a BMH.exe that is not there the embedded numbers of the Diamond 3.7 executable
// and the texts of openbmh.dat stand in for it.
{
std::error_code ec;
if (is_executable_name(path.filename()) && !std::filesystem::exists(path, ec))
return diamond_constants_image();
return diamond_constants_image(path.has_parent_path() ? path.parent_path() : std::filesystem::path("."));
}
const Bytes raw = read_file(path);
if (raw.size() >= sizeof kImageMagic + 8 && std::equal(kImageMagic, kImageMagic + sizeof kImageMagic, raw.begin())) {
+2 -2
View File
@@ -170,9 +170,9 @@ bool prepare_english_game(const fs::path& source, const fs::path& target, std::s
}
if (auto f = find_file(source, "TEXTE.BLK")) copy(*f, target / "TEXTD.BLK");
if (auto f = find_file(source, "M3E.MAN")) copy(*f, target / "M3D.MAN");
// FLTD.EXE is optional: without it the numbers and texts built into OpenBMH stand in (formats/diamond_image.hpp)
// FLTD.EXE is optional: without it the numbers built into OpenBMH and the texts of openbmh.dat stand in (formats/diamond_image.hpp)
const auto exe = find_file(source, "FLTD.EXE");
save_image(exe ? translate_fltd(*exe) : english_constants_image(), target / "BMH.exe");
save_image(exe ? translate_fltd(*exe) : english_constants_image(source), target / "BMH.exe");
return true;
} catch (const std::exception& e) {
error = e.what();
+5 -3
View File
@@ -426,21 +426,23 @@ ResolvedGame resolve_game(const fs::path& input, const fs::path& cache) {
const fs::path marker = prepared / ".openbmh-english";
// the version of the translation (launch/fltd.cpp, fltd_map.inc): a cache built
// by another version is built again
constexpr const char* kTranslation = "english translation 18"; // 18: numbers and texts built in (no FLTD.EXE needed), 17: English monthly environment tables (16: overwrite question)
constexpr const char* kTranslation = "english translation 18"; // 18: numbers built in (no FLTD.EXE needed), 17: English monthly environment tables (16: overwrite question)
const std::string texts = texts_signature(game.check.directory); // an English game made without openbmh.dat is made again with it
const std::string version = std::string(kTranslation) + (game.check.exe.empty() ? " " + texts : std::string());
std::string built;
{
std::ifstream in(marker);
std::getline(in, built);
}
std::error_code ec;
if (built != kTranslation) {
if (built != version) {
fs::remove_all(prepared, ec);
std::string error;
if (!prepare_english_game(game.check.directory, prepared, error)) {
game.error = "The English version cannot be prepared: " + error;
return game;
}
std::ofstream(marker) << kTranslation << "\n" << path_to_utf8(game.check.directory) << "\n";
std::ofstream(marker) << version << "\n" << path_to_utf8(game.check.directory) << "\n";
}
game.check.directory = prepared;
game.check.playable = true;
+36 -11
View File
@@ -1,11 +1,12 @@
// SPDX-License-Identifier: GPL-3.0-only
// The game without BMH.exe and FLTD.EXE (formats/diamond_image.*): the images built from the embedded numbers and texts
// of the Diamond 3.7 executable and of the English Football Limited. Usage: diamond_image_tests [<BMH data dir>]
// of the Diamond 3.7 executable and of the English Football Limited. Usage: diamond_image_tests [<BMH data dir> | -] [<openbmh.dat>]
// (with a data directory the embedded German image is also compared byte for byte with its BMH.exe).
#include <algorithm>
#include <filesystem>
#include <iostream>
#include <string>
#include <string_view>
#include "formats/diamond_image.hpp"
#include "formats/le.hpp"
@@ -26,21 +27,42 @@ bool contains_text(const LeImage& image, const std::string& text) {
return std::search(b.begin(), b.end(), text.begin(), text.end()) != b.end();
}
void german() {
const LeImage image = diamond_constants_image();
// Without the texts file the images hold the numbers only; with it (argv[2], data/openbmh.dat) they also hold the texts.
void german(const Bytes& dat) {
LeImage image = diamond_constants_image();
require(image.size() > kMinimumSize && image.base() == kImageStart, "the German image has the layout of the Diamond data object");
// the game texts are built in: a line of the live commentary
require(contains_text(image, "DER LINIENRICHTER HAT DIE FAHNE OBEN"), "the German texts are embedded");
// the letter tables of the cup draw and fixtures are numbers of the image
require(contains_text(image, "abcdefghijklmnopqrst"), "the fixture letter tables are embedded");
if (dat.empty()) return;
require(apply_texts(image, 1, dat), "openbmh.dat has the German texts");
require(contains_text(image, "DER LINIENRICHTER HAT DIE FAHNE OBEN"), "the German texts come from openbmh.dat");
}
void english() {
const LeImage image = english_constants_image();
void english(const Bytes& dat) {
LeImage image = english_constants_image();
require(image.base() == kImageStart && image.size() > kMinimumSize, "the English image lies in the German address space");
require(contains_text(image, "Competition closed"), "the English texts are embedded");
if (dat.empty()) return;
require(apply_texts(image, 2, dat), "openbmh.dat has the English texts");
require(contains_text(image, "Competition closed"), "the English texts come from openbmh.dat");
}
void damaged_file(const Bytes& dat) {
LeImage image = diamond_constants_image();
bool threw = false;
try { apply_texts(image, 1, Bytes{'n', 'o', 't', ' ', 'a', ' ', 'f', 'i', 'l', 'e', ' ', ' ', ' ', ' ', ' ', ' '}); } catch (const FormatError&) { threw = true; }
require(threw, "a file without the magic is refused");
if (dat.size() < 40) return;
Bytes cut(dat.begin(), dat.begin() + 40);
threw = false;
try { apply_texts(image, 1, cut); } catch (const FormatError&) { threw = true; }
require(threw, "a truncated file is refused");
// a range that is not a text of the game cannot be written: the file never changes a number of the image
Bytes bad = dat;
for (size_t at = 12 + 4 + 8; at < 12 + 4 + 12; ++at) bad[at] ^= 0x55; // the address of the first German range
threw = false;
try { apply_texts(image, 1, bad); } catch (const FormatError&) { threw = true; }
require(threw, "a range that is no text range is refused");
}
void missing_executable() {
// a BMH.exe that is not there stands for the embedded German image
const fs::path nowhere = fs::temp_directory_path() / "openbmh-no-such-directory" / "BMH.exe";
@@ -73,10 +95,13 @@ void against_real_executable(const fs::path& directory) {
} // namespace
int main(int argc, char** argv) try {
german();
english();
Bytes dat;
if (argc > 2) dat = read_file(argv[2]);
german(dat);
english(dat);
damaged_file(dat);
missing_executable();
if (argc > 1) against_real_executable(argv[1]);
if (argc > 1 && std::string_view(argv[1]) != "-") against_real_executable(argv[1]);
if (failures) return 1;
std::cout << "PASS embedded German and English images\n";
return 0;
+4 -1
View File
@@ -271,7 +271,10 @@ bool replay_matchday(GameState& b, std::vector<MatchGroup>& groups, GameRandom&
MatchEngine trial{sc, gc, gg, wr, engine.referees};
trial.month = engine.month;
MinuteReport tr;
trial.rerate_team(team, tr);
{
NoHook no_hook; // (the trial draws on copies: they must not count as draws of the port)
trial.rerate_team(team, tr);
}
const bool hit = !tr.unsupported && gg.seed() == int32_t(std::stoll(field(after.line, "game_seed"))) &&
wr.seed() == uint32_t(std::stoull(field(after.line, "rand_seed")));
if (!hit) continue;
+134
View File
@@ -0,0 +1,134 @@
"""Finds the game texts in a German-layout image (a Diamond BMH.exe, or the translated English FLTD.EXE).
generate.py keeps the numbers of the image in the source files (formats/*_constants.inc) and puts the game texts into
the data file openbmh.dat that ships with the release. A range counts as text when it is
* a NUL terminated string that a pointer variable of the data object (or of the English data segment) points to, or
* a string that a 32 bit immediate of the code points to (a "push offset text"), or
* a long printable run with a space (a sentence), or
* a string the program read as a string while a trace was recorded (OPENBMH_LE_TRACE, kind 2).
Letter permutation tables of the cup draw ('abcdefghijklmnopqr', all letters different) and file names are numbers /
names, not texts, and stay in the constants. Used by generate.py.
"""
import re
import struct
PRINTABLE = set(range(0x20, 0x7F))
UMLAUTS = {0x84, 0x94, 0x81, 0x8E, 0x99, 0x9A, 0xE1} # CP437 letters seen in character sets
TEXT_BYTES = PRINTABLE | UMLAUTS
FILE_NAME = re.compile(r"[A-Za-z0-9_./\\*?-]*[./\\][A-Za-z0-9_./\\*?-]*")
def string_end(data, base, start):
"""Address after the NUL of the string starting at `start`, or None when it is not a clean string."""
i = start - base
n = 0
while i < len(data) and data[i] != 0:
if data[i] not in TEXT_BYTES and data[i] < 0x20:
return None
i += 1
n += 1
if n > 400:
return None
return i + 1 + base if i < len(data) else None
def is_permutation(text):
letters = text.lower()
return letters.isalpha() and len(set(letters)) == len(letters) and len(letters) in (4, 6, 10, 12, 16, 18, 20)
def ascending(text, n=4):
"""True when the text holds n or more ascending consecutive characters (a character set 'abcdef', not a word)."""
run = 1
for a, b in zip(text, text[1:]):
run = run + 1 if ord(b) == ord(a) + 1 else 1
if run >= n:
return True
return False
def is_text(text):
if not re.search(r"[A-Za-z]{3}", text) or is_permutation(text) or text.startswith(("f\\", "Ã")) or FILE_NAME.fullmatch(text):
return False
# a single token in one case is a name or a table fragment (BLASTER, KRON_FBM, V3.7, 'njpmd' from a fixture row), not a
# sentence or a label of the game; letter and character-set tables are not texts either
if " " not in text and (text.isupper() or text.islower() or re.fullmatch(r"[A-Z][A-Za-z0-9_.]*\d[A-Za-z0-9_.]*", text)):
return False
if ascending(text) or sum(c.isalpha() or c == " " for c in text) < 0.6 * len(text):
return False
return True
def text_ranges(data, base, targets, sources, code, layout_pointers, traced_strings):
"""data: the image bytes at address `base`. targets: [(lo, hi)] where texts may lie, sources: [(lo, hi)] where
pointer variables lie, code: [(lo, hi)] where immediates are searched. Returns sorted (start, length) incl. the NUL."""
found = {}
def in_targets(address):
return any(lo <= address < hi for lo, hi in targets)
def add(start, end):
if in_targets(start) and end > start:
found[start] = max(found.get(start, 0), end - start)
def text_at(value):
end = string_end(data, base, value)
if end and end - value >= 4:
text = bytes(data[value - base:end - 1 - base]).decode("latin1")
if is_text(text):
return end
return None
# 1. strings that pointer variables point to
for lo, hi in sources:
for at in range(lo, hi - 3, 4):
value = struct.unpack_from("<I", data, at - base)[0]
if in_targets(value) and value != at:
end = text_at(value)
if end:
add(value, end)
for pointer in layout_pointers:
if pointer - base < 0 or pointer - base + 4 > len(data):
continue
value = struct.unpack_from("<I", data, pointer - base)[0]
if in_targets(value):
end = string_end(data, base, value)
if end:
add(value, end)
# 1b. strings the code points to: after the fixups a "push offset text" holds the address as a 32 bit immediate
for lo, hi in code:
for at in range(lo, hi - 3):
value = struct.unpack_from("<I", data, at - base)[0]
if in_targets(value) and data[value - 1 - base] == 0:
end = text_at(value)
if end:
add(value, end)
# 2. long printable runs with a space (sentences)
for lo, hi in targets:
i = lo - base
while i < hi - base:
if data[i] in PRINTABLE:
j = i
while j < hi - base and data[j] in TEXT_BYTES:
j += 1
text = bytes(data[i:j]).decode("latin1")
if j - i >= 8 and " " in text and re.search(r"[A-Za-z]{3}", text):
add(i + base, j + base + (1 if j < len(data) and data[j] == 0 else 0))
i = j
else:
i += 1
# 3. strings the program read while a trace was recorded
for start, length in traced_strings:
text = bytes(data[start - base:start - base + max(length - 1, 0)]).decode("latin1")
if is_text(text):
add(start, start + length)
merged = []
for start in sorted(found):
end = start + found[start]
if merged and start <= merged[-1][1]:
merged[-1][1] = max(merged[-1][1], end)
else:
merged.append([start, end])
return [(s, e - s) for s, e in merged]
+75 -30
View File
@@ -1,8 +1,10 @@
#!/usr/bin/env python3
"""Generates the embedded game images of OpenBMH, so that the game runs from the asset files alone (no BMH.exe, no
"""Generates the embedded game data of OpenBMH, so that the game runs from the asset files alone (no BMH.exe, no
FLTD.EXE): src/formats/diamond_constants.inc (German, Diamond 3.7) and src/formats/fltd_constants.inc (English
Football Limited). Both hold every non-zero byte of the data the engine reads - tables, rates, positions, prices and the
game texts - as runs of bytes; zero runs are not stored.
Football Limited) hold every non-zero byte of the numbers the engine reads - tables, rates, positions, prices - as runs of
bytes; zero runs are not stored. The game texts (live commentary, messages) are NOT in the source: they go into the data
file data/openbmh.dat that ships with the release (formats/diamond_image.hpp); the .inc files only list where they are.
After regenerating, put the new SHA-256 of data/openbmh.dat into REVIEWED_BINARY_FILES of tools/public_release/check_public_tree.py.
German (a Diamond 3.7 BMH.exe, read here and never copied into the repository):
python generate.py german --exe BMH.exe --trace trace1.txt [--trace trace2.txt ...] --out ../../src/formats/diamond_constants.inc
@@ -19,12 +21,17 @@ English (the German layout image of FLTD.EXE, the cached "BMH.exe" of an English
import argparse
import re
import struct
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
import classify # noqa: E402
DATA_LO, DATA_PAGES = 0xA0000, 12 # the initialised pages of the data object (file backed)
DATA_HI = DATA_LO + DATA_PAGES * 4096
CODE = (0x10000, 0x10000 + 438272)
PATCH = (0x80000, 0x81000)
EN_LO, EN_HI = 0x6B000, 0x776A8 # the English data segment, also placed at its own addresses
def load_le(path):
@@ -134,6 +141,16 @@ def read_traces(paths):
return numbers, bulk
def read_string_traces(paths):
out = []
for path in paths:
for line in Path(path).read_text().splitlines():
flag, start, length = (int(x, 16) for x in line.split())
if flag == 2:
out.append((start, length))
return out
def merge(ranges, gap=0):
out = []
for start, end in sorted((s, s + l) for s, l in ranges):
@@ -157,10 +174,11 @@ def nonzero_runs(data, base, lo, hi, gap=8):
return runs
def write_inc(path, title, base, data, chunks):
def write_inc(path, title, base, data, chunks, texts):
out = ["// SPDX-License-Identifier: GPL-3.0-only",
f"// Generated by tools/diamond_constants/generate.py from {title} - do not edit.",
"// Every non-zero byte of the data the engine reads (tables, rates, positions, prices, game texts), as runs.",
"// Every non-zero byte of the numbers the engine reads (tables, rates, positions, prices) as runs; the game texts",
"// are not part of it: kTextRanges lists where they lie, their bytes come from openbmh.dat.",
"// Included inside a namespace by formats/diamond_image.cpp.",
f"inline constexpr uint32_t kImageBase = {base}, kImageSize = {len(data)};"]
total = 0
@@ -173,48 +191,75 @@ def write_inc(path, title, base, data, chunks):
for i, (start, end) in enumerate(chunks):
out.append(f" {{{start}, {end - start}, kChunk{i}}},")
out.append("};")
out.append("inline constexpr Range kTextRanges[] = {")
for start, length in texts:
out.append(f" {{{start}, {length}}},")
out.append("};")
Path(path).write_text("\n".join(out) + "\n")
print(f"{len(chunks)} chunks, {total} bytes")
print(f"{path}: {len(chunks)} chunks, {total} bytes of numbers; {len(texts)} text ranges, {sum(l for _, l in texts)} bytes of text")
def zeroed(data, base, texts):
out = bytearray(data)
for start, length in texts:
out[start - base:start - base + length] = bytes(length)
return out
def dat_set(edition, data, base, texts):
out = struct.pack("<II", edition, len(texts))
for start, length in texts:
out += struct.pack("<II", start, length)
for start, length in texts:
out += bytes(data[start - base:start - base + length])
return out
def main():
ap = argparse.ArgumentParser()
sub = ap.add_subparsers(dest="mode", required=True)
de = sub.add_parser("german")
de.add_argument("--exe", required=True)
de.add_argument("--trace", action="append", default=[])
de.add_argument("--layout", default=str(Path(__file__).parents[2] / "src/formats/executable_layout.hpp"))
de.add_argument("--out", required=True)
en = sub.add_parser("english")
en.add_argument("--image", required=True)
en.add_argument("--out", required=True)
ap.add_argument("--exe", required=True, help="a Diamond 3.7 BMH.exe")
ap.add_argument("--english-image", required=True, help="the cached German layout image of FLTD.EXE (OPENBMH-IMAGE)")
ap.add_argument("--trace", action="append", default=[])
ap.add_argument("--layout", default=str(Path(__file__).parents[2] / "src/formats/executable_layout.hpp"))
ap.add_argument("--out-dir", required=True, help="src/formats")
ap.add_argument("--dat", required=True, help="data/openbmh.dat")
args = ap.parse_args()
if args.mode == "english":
base, data = load_openbmh_image(args.image)
write_inc(args.out, "the German layout image of FLTD.EXE (Football Limited)", base, data,
nonzero_runs(data, base, base, base + len(data)))
return
base, data = load_le(args.exe)
numbers, bulk = read_traces(args.trace)
layout = Path(args.layout).read_text()
# numbers read from the code or patch object: the bytes read, a margin, and a window after bulk pointers
out_dir = Path(args.out_dir)
# German (Diamond 3.7)
base, data = load_le(args.exe)
numbers_read, bulk = read_traces(args.trace)
strings_read = [(s, l) for s, l in read_string_traces(args.trace) if DATA_LO <= s < DATA_HI]
pointers = [int(a, 16) for n, a in re.findall(r"inline constexpr auto (\w*TextPointer\w*) = 0x([0-9a-fA-F]+);", layout)]
texts_de = classify.text_ranges(data, base, [(DATA_LO, DATA_HI)], [(DATA_LO, DATA_HI)], [CODE, PATCH], pointers, strings_read)
wanted = []
for s, l in numbers:
for s, l in numbers_read:
if CODE[0] <= s < CODE[1] or PATCH[0] <= s < PATCH[1]:
wanted.append((max(s - 4, 0), l + 8))
for s, l in bulk:
if CODE[0] <= s < CODE[1] or PATCH[0] <= s < PATCH[1]:
wanted.append((s, 256))
# and, whatever the traces saw, a window at every layout constant that names a place in the code objects
for name, address in re.findall(r"inline constexpr auto (\w+) = 0x([0-9a-fA-F]+);", layout):
a = int(address, 16)
if CODE[0] <= a < CODE[1] or PATCH[0] <= a < PATCH[1]:
wanted.append((a, 16))
chunks = sorted(nonzero_runs(data, base, DATA_LO, DATA_HI) + merge(wanted, 16))
write_inc(args.out, "BMH 2026-27 Diamond edition 3.7", base, data, chunks)
numbers_de = zeroed(data, base, texts_de)
chunks_de = sorted(nonzero_runs(numbers_de, base, DATA_LO, DATA_HI) + merge(wanted, 16))
write_inc(out_dir / "diamond_constants.inc", "BMH 2026-27 Diamond edition 3.7", base, numbers_de, chunks_de, texts_de)
# English (Football Limited, in the German address space)
ebase, edata = load_openbmh_image(args.english_image)
en_segment = (EN_LO, EN_HI)
texts_en = classify.text_ranges(edata, ebase, [(DATA_LO, DATA_HI), en_segment], [(DATA_LO, DATA_HI), en_segment], [], [], [])
numbers_en = zeroed(edata, ebase, texts_en)
write_inc(out_dir / "fltd_constants.inc", "the German layout image of FLTD.EXE (Football Limited)", ebase, numbers_en,
nonzero_runs(numbers_en, ebase, ebase, ebase + len(edata)), texts_en)
blob = b"OPENBMH-DAT\0" + struct.pack("<I", 2) + dat_set(1, data, base, texts_de) + dat_set(2, edata, ebase, texts_en)
Path(args.dat).write_bytes(blob)
print(f"{args.dat}: {len(blob)} bytes")
if __name__ == "__main__":
main()
main()
+1 -1
View File
@@ -11,7 +11,7 @@ import zipfile
def validate(directory, windows):
products = {'OpenBMH.exe', 'openbmh_cli.exe'} if windows else {'OpenBMH', 'openbmh_cli'}
documents = {'README.md', 'LICENSE', 'THIRD_PARTY_NOTICES.md', 'ACKNOWLEDGEMENTS.md', 'CONTRIBUTING.md'}
documents = {'README.md', 'LICENSE', 'THIRD_PARTY_NOTICES.md', 'ACKNOWLEDGEMENTS.md', 'CONTRIBUTING.md', 'openbmh.dat'}
licenses = {'README.md', 'LGPL-2.1.txt', 'Nuked-OPL3-notices.txt', 'LZMA-SDK.txt', 'SDL2-HIDAPI-BSD.txt',
'SDL2-HIDAPI-options.txt', 'SDL2-HIDAPI-original.txt', 'SDL2-yuv2rgb.txt',
'SDL2.txt', 'miniz-source-notices.txt', 'miniz.txt', 'nativefiledialog-extended.txt', 'stb_image.txt'}
+3 -1
View File
@@ -31,7 +31,9 @@ ASSET_SUFFIXES = {
}
# Newly supplied project artwork and an unchanged MIT/public-domain decoder.
# Exact hashes keep this exception from admitting original-game asset dumps.
REVIEWED_BINARY_FILES = {"assets/start-screen.jpg": "28df5a72cd7b6951c9eebf30532ea9557007a46f0a73614ba22401c403e379dc"}
# data/openbmh.dat: the game texts of the original executables (tools/diamond_constants/generate.py); regenerate = new hash here
REVIEWED_BINARY_FILES = {"assets/start-screen.jpg": "28df5a72cd7b6951c9eebf30532ea9557007a46f0a73614ba22401c403e379dc",
"data/openbmh.dat": "ae313060d3c34a9b45adae8565715618182137443ab20c1485b2da6e2c9e62a4"}
REVIEWED_UPSTREAM_FILES = {"src/app/third_party/stb_image.h": "594c2fe35d49488b4382dbfaec8f98366defca819d916ac95becf3e75f4200b3"}
PRIVATE_PREFIXES = (
("data", "original"), ("docs", "manuals"), ("docs", "analyses"),