From d4a461faec62ca1efdd69209a89fe7b079b15a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Barroso=20Galindo?= Date: Thu, 22 Dec 2022 01:15:26 +0100 Subject: [PATCH] Simplifying cheats and term aliases. --- .github/calculate_db.py | 65 +++++++++--------- .github/download_distribution.py | 104 ++++++++++++----------------- .github/tests/test_calculate_db.py | 2 +- 3 files changed, 77 insertions(+), 94 deletions(-) diff --git a/.github/calculate_db.py b/.github/calculate_db.py index b50592118..a5987a5c3 100755 --- a/.github/calculate_db.py +++ b/.github/calculate_db.py @@ -46,7 +46,7 @@ def main(dryrun): check_test = os.getenv('CHECK_TEST', 'true') != 'false' download_metadata_json = os.getenv('DOWNLOAD_METADATA_JSON', '/tmp/download_metadata.json') - tags = Tags(try_read_json(download_metadata_json, {'home': {}})) + tags = Tags(try_read_json(download_metadata_json, None)) db = create_db('.', { 'sha': sha, @@ -127,22 +127,11 @@ distribution_mister_aliases = [ ['megadrive', 'genesis'], ['megacd', 'segacd'], ['sms', 'mastersystem', 'segamark3'], - ['coleco', 'colecovision'], ['gb', 'gameboy'], ['gbc', 'gameboycolor'], ['sgb', 'supergameboy'], ['gba', 'gameboyadvance'], - # Computers - ['vector06c', 'vector06'], - ['amiga', 'minimig'], - ['zxspectrum', 'spectrum'], - ['pc88', 'pc8801'], - ['laser310', 'laser'], - - # Other - ['flappybird', 'flappy'], - # General ['console-cores', 'console'], ['arcade-cores', 'arcade'], @@ -156,6 +145,10 @@ filter_part_regex = re.compile("[-_a-z0-9.]$", ) main_binaries = ['MiSTer', 'menu.rbf'] class Metadata: + @staticmethod + def new_props(): + return {'home': {}, 'aliases': []} + def __init__(self, props): self._props = props @@ -165,21 +158,39 @@ class Metadata: def category_by_home(self, home): return None if home not in self._props['home'] else self._props['home'][home]['category'] + def aliases(self): + return self._props['aliases'] + class Tags: - def __init__(self, metadata) -> None: - self._metadata = Metadata(metadata) + def __init__(self, metadata_props) -> None: + self._metadata = Metadata(metadata_props if metadata_props is not None else Metadata.new_props()) self._dict = {} self._alternatives = {} self._index = 0 self._report_set = set() self._used = set() + self._init = False def init_aliases(self, aliases): - for alias_list in aliases: - for alias in alias_list: - self._dict[self._clean_term(alias)] = self._index - self._index += 1 + if self._init: + raise Exception("Can only be initialised once.") + self._init = True + for alias_list in [*self._metadata.aliases(), *aliases]: + clean_terms = [self._clean_term(alias) for alias in alias_list] + index = self._matching_dict_index(clean_terms) + for term in clean_terms: + self._dict[term] = index + + def _matching_dict_index(self, clean_terms): + for term in clean_terms: + if term in self._dict: + return self._dict[term] + + result = self._index + self._index += 1 + return result + def get_tags_for_file(self, path: Path): return sorted(self._get_tags_for_file(path)) @@ -240,6 +251,9 @@ class Tags: if rbf is not None: self._append(result, self._use_term(Path(rbf).name.lower())) + if stem in ['menu', 'mister']: + self._append(result, self._use_term('essential')) + if parent in ['games', 'docs']: first_level = path.parts[1].lower() self._append(result, self._use_term(first_level)) @@ -268,14 +282,11 @@ class Tags: elif parent == 'cheats': self._append(result, self._use_term(path.parts[1].lower())) - if parent in ['gamma', 'filters', 'filters_audio', 'shadow_masks']: + elif parent in ['gamma', 'filters', 'filters_audio', 'shadow_masks']: self._append(result, self._use_term('all_filters')) - if parent in ['gamma', 'filters', 'shadow_masks']: - self._append(result, self._use_term('filters_video')) - - if stem in ['menu', 'mister']: - self._append(result, self._use_term('essential')) + if parent in ['gamma', 'filters', 'shadow_masks']: + self._append(result, self._use_term('filters_video')) return result @@ -669,8 +680,6 @@ class MultiSourcesZipCreator: print('Created zip: ' + self._zip_name) def create_summary(finder: Finder, tags: Tags, source): - delete_list_regex = re.compile("^(.*_)[0-9]{8}(\.[a-zA-Z0-9]+)*$", ) - summary = { 'files': dict(), 'folders': dict() @@ -688,10 +697,6 @@ def create_summary(finder: Finder, tags: Tags, source): "hash": hash(file), "tags": tags.get_tags_for_file(file) } - - delete_list = create_delete_list(strfile, delete_list_regex) - if len(delete_list) > 0: - summary["files"][strfile]["delete"] = delete_list file_name = file.name.lower() diff --git a/.github/download_distribution.py b/.github/download_distribution.py index 796a16863..d6b1e4436 100755 --- a/.github/download_distribution.py +++ b/.github/download_distribution.py @@ -17,7 +17,7 @@ import xml.etree.ElementTree as ET import sys amount_of_cores_validation_limit = 200 -amount_of_extra_content_urls_validation_limit = 25 +amount_of_extra_content_urls_validation_limit = 20 def main(): @@ -176,24 +176,9 @@ def fetch_extra_content_urls(): result = [] result.extend(['https://github.com/MiSTer-devel/Main_MiSTer', 'https://github.com/MiSTer-devel/Menu_MiSTer']) result.extend(['user-content-mra-alternatives', 'https://github.com/MiSTer-devel/MRA-Alternatives_MiSTer']) - result.extend(["user-cheats"]) # @TODO Modify this mapping whenever there is a new system with cheats - result.extend(["fds|NES"]) - result.extend(["gb|GameBoy"]) - result.extend(["gba|GBA"]) - result.extend(["gbc|GameBoy"]) - result.extend(["gen|Genesis"]) - result.extend(["gg|SMS"]) - result.extend(["lnx|AtariLynx"]) - result.extend(["nes|NES"]) - result.extend(["pce|TGFX16"]) - result.extend(["pcd|TGFX16-CD"]) - result.extend(["psx|PSX"]) - result.extend(["scd|MegaCD"]) - result.extend(["sms|SMS"]) - result.extend(["snes|SNES"]) - #result.extend(["user-backup-cheats", "https://github.com/MiSTer-devel/Distribution_MiSTer/archive/refs/heads/main.zip"]) # Uncomment if user-cheats breaks (and comment user-cheats instead) result.extend(["user-content-fonts", "https://github.com/MiSTer-devel/Fonts_MiSTer"]) result.extend(["user-content-folders"]) + result.extend(["https://github.com/MiSTer-devel/Cheats_MiSTer"]) result.extend(["https://github.com/MiSTer-devel/Filters_MiSTer"]) result.extend(["https://github.com/MiSTer-devel/ShadowMasks_MiSTer"]) result.extend(["https://github.com/MiSTer-devel/Presets_MiSTer"]) @@ -217,8 +202,6 @@ def classify_extra_content(extra_content_urls): if url == "user-content-linux-binary": current_category = url elif url == "user-content-zip-release": current_category = url elif url == "user-content-scripts": current_category = url - elif url == "user-cheats": current_category = url - elif url == "user-backup-cheats": current_category = url elif url == "user-content-empty-folder": current_category = url elif url == "user-content-gamecontrollerdb": current_category = url elif url == "user-content-folders": current_category = url @@ -265,8 +248,11 @@ def process_core(core, delme, target, metadata_props): print(f'Warning! Ignored {category}: {url}') return + metadata = Metadata(metadata_props) + metadata.set_ctx(core) + if category in core_installers: - return core_installers[category](path, target, core, Metadata(metadata_props)) + return core_installers[category](path, target, core, metadata) raise SystemError(f'Ignored core: {url} {category}') @@ -294,7 +280,7 @@ def install_arcade_core(path, target_dir, core, metadata): releases_dir = f'{path}/releases' arcade_installed = False - for bin in uniq_files_with_stripped_date(releases_dir, 'Arcade-'): + for bin in try_filter_list(uniq_files_with_stripped_date(releases_dir), 'Arcade-'): latest_release = get_latest_release(releases_dir, bin) if not is_rbf(latest_release): print(f'{core["url"]}: {latest_release} is NOT a RBF file') @@ -320,7 +306,7 @@ def impl_install_generic_core(path, target_dir, core, metadata, touch_games_fold releases_dir = f'{path}/releases' binaries = [] - for bin in uniq_files_with_stripped_date(releases_dir, core["home"]): + for bin in try_filter_list(uniq_files_with_stripped_date(releases_dir), core["home"]): if is_arcade_core(bin): continue @@ -334,6 +320,7 @@ def impl_install_generic_core(path, target_dir, core, metadata, touch_games_fold binaries.append(bin) metadata.add_home(core['home'], core['category']) + metadata.add_core_aliases([core['home'], *binaries]) home_folders = [core['home']] for mgl in mgl_files(releases_dir): @@ -347,6 +334,7 @@ def impl_install_generic_core(path, target_dir, core, metadata, touch_games_fold home_folders.append(setname) metadata.add_mgl_home(setname, core['category']) + metadata.add_core_aliases([setname, Path(mgl).stem]) for folder in home_folders: for readme in list_readmes(path): @@ -389,7 +377,7 @@ def install_main_binary(path, target_dir, category, url): print(f'Warning! Ignored {category}: {url}') return - for bin in uniq_files_with_stripped_date(releases_dir, None): + for bin in uniq_files_with_stripped_date(releases_dir): latest_release = get_latest_release(releases_dir, bin) if is_empty_release(latest_release): continue @@ -404,7 +392,7 @@ def install_linux_binary(path, target_dir, category, url): print(f'Warning! Ignored {category}: {url}') return - for bin in uniq_files_with_stripped_date(releases_dir, None): + for bin in uniq_files_with_stripped_date(releases_dir): latest_release = get_latest_release(releases_dir, bin) if is_empty_release(latest_release): continue @@ -419,7 +407,7 @@ def install_zip_release(path, target_dir, category, url): print(f'Warning! Ignored {category}: {url}') return - for zip in uniq_files_with_stripped_date(releases_dir, None): + for zip in uniq_files_with_stripped_date(releases_dir): latest_release = get_latest_release(releases_dir, zip) if is_empty_release(latest_release): continue @@ -464,35 +452,9 @@ def install_gamecontrollerdb(url, target_dir): print(f"SDL Game Controller DB: {url}") download_file(url, f'{target_dir}/linux/gamecontrollerdb/{Path(url).name}') -def install_cheats(mapping, target_dir): - page_url = "https://gamehacking.org/mister" - - parts = mapping.split('|') - cheat_key = parts[0].strip() - cheat_platform = parts[1].strip() - - cheat_zips = collect_cheat_zips(page_url) - - cheat_zip = next(cheat_zip for cheat_zip in cheat_zips if cheat_key in cheat_zip) - cheat_url = f'{page_url}/{cheat_zip}' - tmp_zip = f'/tmp/{cheat_key}{cheat_platform}.zip' - cheat_folder = f'{target_dir}/Cheats/{cheat_platform}' - - print(f'cheat_keys: {cheat_key}, cheat_platform: {cheat_platform}, cheat_zip: {cheat_zip}, cheat_url: {cheat_url}') - - download_file(cheat_url, tmp_zip) - unzip(tmp_zip, cheat_folder) - -def install_cheats_backup(url, target_dir): - tmp_zip = '/tmp/old_main.zip' - download_file(url, tmp_zip) - unzip(tmp_zip, f'{target_dir}/Cheats/') - extra_content_early_installers = { 'user-content-scripts': install_script, 'user-content-empty-folder': install_empty_folder, - 'user-cheats': install_cheats, - 'user-backup-cheats': install_cheats_backup, 'user-content-gamecontrollerdb': install_gamecontrollerdb, } @@ -501,10 +463,15 @@ extra_content_early_installers = { class Metadata: @staticmethod def new_props(): - return {'home': {}} + return {'home': {}, 'aliases': []} def __init__(self, props): self._props = props + self._terms = set() + self._ctx = None + + def set_ctx(self, ctx): + self._ctx = ctx def add_mgl_home(self, folder, category): lower = folder.lower() @@ -515,6 +482,15 @@ class Metadata: self._props['home'][lower] = self._props['home'].get(lower, {'is_mgl': True, 'category': category.lower()[1:]}) self._props['home'][lower]['is_mgl'] = False + def add_core_aliases(self, core_aliases): + terms = {to_filter_term(c) for c in core_aliases} + for t in terms: + if t in self._terms: + raise ValueError(f'{t} from {str(core_aliases)} was already present!', self._ctx) + self._terms.add(t) + if len(terms) > 1: + self._props['aliases'].append(list(terms)) + def mra_files(folder): return [without_folder(folder, f) for f in list_files(folder, recursive=False) if Path(f).suffix.lower() == '.mra'] @@ -529,7 +505,7 @@ def get_latest_release(folder, bin): releases = sorted([f for f in files if bin in f and remove_date(f) != f]) return releases[-1] -def uniq_files_with_stripped_date(folder, home): +def uniq_files_with_stripped_date(folder): result = [] for f in list_files(folder, recursive=False): f = without_folder(folder, str(Path(f).with_suffix(''))) @@ -539,14 +515,15 @@ def uniq_files_with_stripped_date(folder, home): continue result.append(no_date) - - if home is not None: - only_home = [f for f in result if home.lower() in f.lower()] - if len(only_home) > 0: - return only_home - return result +def try_filter_list(col, filter): + filtered = [el for el in col if filter.lower() in el.lower()] + if len(filtered) > 0: + return filtered + + return col + def clean_palettes(palette_folder): for file in list_files(palette_folder, recursive=True): path = Path(file) @@ -616,10 +593,6 @@ def is_empty_release(bin): def list_fonts(path): return [Path(f).name for f in list_files(path, recursive=True) if Path(f).suffix.lower() == '.pf'] -def collect_cheat_zips(url): - text = fetch_text(url, cookies={'challenge': 'BitMitigate.com'}) - return [f[f.find('mister_'):f.find('.zip') + 4] for f in text.splitlines() if 'mister_' in f and '.zip' in f] - def download_mister_devel_repository(input_url, delme, category): name = get_repository_name(input_url) branch = get_branch(input_url) @@ -645,6 +618,11 @@ def get_branch(url): return "" return url[pos + len('/tree/'):] +filter_term_char_regex = re.compile("[-_a-z0-9.]$", ) +def to_filter_term(name: str): + result = ''.join(filter(lambda chr: filter_term_char_regex.match(chr), name.lower().replace(' ', ''))) + return result.replace('-', '').replace('_', '') + # file system utilities def list_files(directory, recursive): diff --git a/.github/tests/test_calculate_db.py b/.github/tests/test_calculate_db.py index 69e7e8819..97a5c3b68 100755 --- a/.github/tests/test_calculate_db.py +++ b/.github/tests/test_calculate_db.py @@ -7,7 +7,7 @@ spec = importlib.util.spec_from_file_location("calculate_db", "../calculate_db.p calculate_db = importlib.util.module_from_spec(spec) spec.loader.exec_module(calculate_db) -tags = calculate_db.Tags({}) +tags = calculate_db.Tags(None) db = calculate_db.create_db('../..', { 'sha': 3, 'latest_zip_url': 'w/',