diff --git a/.github/calculate_db.py b/.github/calculate_db.py index 08a9b3b9d..02d5326c3 100755 --- a/.github/calculate_db.py +++ b/.github/calculate_db.py @@ -12,6 +12,43 @@ import sys from datetime import datetime import os import tempfile +import shutil +from typing import Protocol, Any + + +class Finder: + def __init__(self, dir): + self._dir = dir + self._not_in_directory = [] + + @property + def dir(self): + return self._dir + + def ignore_folder(self, folder): + directory = str(Path(folder)) + print('ignore_folder: %s' % directory) + self._not_in_directory.append(directory) + + def find_all(self): + return sorted(self._scan(self._dir), key=lambda file: str(file).lower()) + + def _scan(self, directory): + for entry in os.scandir(directory): + if entry.is_dir(follow_symlinks=False): + if str(Path(entry.path)) not in self._not_in_directory: + yield from self._scan(entry.path) + else: + yield Path(entry.path) + + +class EmptyFinder(Finder): + def __init__(self): + Finder.__init__(self, None) + + def find_all(self): + return [] + def benchtime(func): def benchfn(*args, **kwargs): @@ -23,8 +60,9 @@ def benchtime(func): return benchfn + @benchtime -def main(): +def main(dryrun): sha = run_stdout('git rev-parse --verify HEAD').strip() print('sha: %s' % sha) @@ -33,39 +71,201 @@ def main(): db_file_json = Path(db_url).stem db = create_db('.', { - 'base_files_url': envvar('BASE_FILES_URL') % sha, + 'sha': sha, + 'base_files_url': envvar('BASE_FILES_URL'), 'db_url': db_url, 'db_files': [db_file_zip], 'db_id': envvar('DB_ID'), + 'dryrun': dryrun, 'latest_zip_url': envvar('LATEST_ZIP_URL'), - 'linux_github_repository': os.getenv('LINUX_GITHUB_REPOSITORY', '').strip() + 'linux_github_repository': os.getenv('LINUX_GITHUB_REPOSITORY', '').strip(), + 'zips_config': os.getenv('ZIPS_CONFIG', '').strip() }) save_data_to_compressed_json(db, db_file_json, db_file_zip) - force_push_file(db_file_zip, 'main') + if not dryrun: + force_push_file(db_file_zip, 'main') + def envvar(var): result = os.getenv(var) print("{} = {}".format(var, result)) return result + def create_db(folder, options): - - finder = Finder(folder) - folders = dict() - + db_finder = Finder(folder) + db_finder.ignore_folder('./.git') + db_finder.ignore_folder('./.github') db = { "db_id": options['db_id'], "db_url": options['db_url'], "db_files": options['db_files'], "latest_zip_url": options['latest_zip_url'], "files": dict(), - "zips": dict(), - "base_files_url": "" + "base_files_url": options['base_files_url'] % options['sha'] } + zips = dict() + + if options['zips_config'] != '': + print('reading zips_config: ' + options['zips_config']) + with open(options['zips_config']) as zips_config_file: + zips_config = json.load(zips_config_file) + for zip_id in zips_config: + zip_description = zips_config[zip_id] + zip_creator = make_zip_creator(zip_description) + zip_creator.create_zip(db_finder, zips, zip_id, zip_description, options) + + if len(zips) > 0: + + current_branch = run_stdout('git rev-parse --abbrev-ref HEAD').strip() + if current_branch == 'zips': + raise Exception('Should not start on branch "zip"') + + run_succesfully('git branch -D zips || true') + run_succesfully('git checkout --orphan zips') + run_succesfully('git reset') + + for zip_id in zips: + run_succesfully('git add %s.zip' % zip_id) + run_succesfully('git add %s_summary.json.zip' % zip_id) + + run_succesfully('git commit -m "-"') + zip_sha = run_stdout('git rev-parse --verify HEAD').strip() + + if not options['dryrun']: + run_succesfully('git fetch origin main || true') + if not run_conditional('git diff --quiet main origin/zip'): + print('zip branch has changes') + run_succesfully('git push --force origin zips') + else: + print('Using old zip branch from origin') + zip_sha = run_stdout('git rev-parse --verify origin/zip').strip() + + print('zip_sha: ' + zip_sha) + + run_succesfully('git checkout --force ' + current_branch) + + for zip_id in zips: + zips[zip_id]['contents_file']['url'] = (options['base_files_url'] % zip_sha) + '%s.zip' % zip_id + zips[zip_id]['summary_file']['url'] = (options['base_files_url'] % zip_sha) + '%s_summary.json.zip' % zip_id + + db['zips'] = zips + + db_summary = create_summary(db_finder, options) + db['files'] = db_summary['files'] + db['folders'] = db_summary['folders'] + db['files_count'] = db_summary['files_count'] + db['folders_count'] = db_summary['folders_count'] + + if options['linux_github_repository'] != '': + db["linux"] = create_linux_description(options['linux_github_repository']) + + return db + + +class ZipCreator(Protocol): + def create_zip(self, db_finder: Finder, zips: [str, Any], zip_id: str, zip_description: [str, Any], options: [str, Any]) -> None: + pass + + +def make_zip_creator(zip_description: [str, Any]) -> ZipCreator: + mode = zip_description['mode'] if 'mode' in zip_description else 'simple' + + if mode == 'simple': + return SimpleZipCreator() + elif mode == 'subfolders': + return SubfoldersZipCreator() + elif mode == 'multi': + return MultiSourcesZipCreator() + else: + raise NotImplementedError('No ZipCreator for mode: ' + mode) + + +class SimpleZipCreator: + def create_zip(self, db_finder: Finder, zips: [str, Any], zip_id: str, zip_description: [str, Any], options: [str, Any]) -> None: + source_path = Path(zip_description['source']) + zip_description['sources'] = [source_path.name] + zip_description['path'] = str(source_path.parent) + multi = MultiSourcesZipCreator() + multi.create_zip(db_finder, zips, zip_id, zip_description, options) + return + + +class SubfoldersZipCreator: + def create_zip(self, db_finder: Finder, zips: [str, Any], zip_id: str, zip_description: [str, Any], options: [str, Any]) -> None: + simple = SimpleZipCreator() + for folder in [entry.path for entry in os.scandir(zip_description['source']) if entry.is_dir(follow_symlinks=False)]: + if len(Finder(folder).find_all()) < 60: + continue + simple.create_zip(db_finder, zips, zip_id + Path(folder).name.lower(), {"source": folder}, options) + + +class MultiSourcesZipCreator: + def create_zip(self, db_finder: Finder, zips: [str, Any], zip_id: str, zip_description: [str, Any], options: [str, Any]) -> None: + print('Processing zip_id: %s' % zip_id) + + source_parent = zip_description['path'] + summary_name = '%s_summary.json' % zip_id + + multi_summary = create_summary(EmptyFinder(), options) + + source_name_list = [] + for source in zip_description['sources']: + db_finder.ignore_folder('./' + source_parent + '/' + source) + zip_finder = Finder(source_parent + '/' + source) + zip_summary = create_summary(zip_finder, options) + zip_summary['folders'].append(str(Path(source_parent + '/' + source))) + multi_summary['files_count'] += zip_summary['files_count'] + multi_summary['folders_count'] += zip_summary['folders_count'] + multi_summary['files'].update(zip_summary['files']) + multi_summary['folders'].extend(zip_summary['folders']) + source_name_list.append(Path(source).name) + + multi_summary['folders'] = list(set(multi_summary['folders'])) + + zip_description['raw_files_size'] = 0 + zip_description['path'] = source_parent + '/' + zip_description['contents'] = zip_description['sources'] + zip_description['files_count'] = multi_summary['files_count'] + zip_description['folders_count'] = multi_summary['folders_count'] + zip_description['base_files_url'] = options['base_files_url'] % options['sha'] + zip_description.pop('sources') + + for file in multi_summary['files']: + multi_summary['files'][file]['zip_id'] = zip_id + zip_description['raw_files_size'] += multi_summary['files'][file]['size'] + + summary_zip = summary_name + '.zip' + + save_data_to_compressed_json(multi_summary, summary_name, summary_zip) + zip_description['summary_file'] = { + 'size': size(summary_zip), + 'hash': hash(summary_zip) + } + Path(summary_name).unlink() + + zip_name = zip_id + '.zip' + + run_succesfully('cur=$(pwd) && cd %s && zip -q -D -X -A -r $cur/%s %s' % (source_parent, zip_name, " ".join(source_name_list))) + zip_description['contents_file'] = { + 'size': size(zip_name), + 'hash': hash(zip_name) + } + + zips[zip_id] = zip_description + print('Created zip: ' + zip_name) + + +def create_summary(finder: Finder, options): + folders = dict() delete_list_regex = re.compile("^(.*_)[0-9]{8}(\.[a-zA-Z0-9]+)*$", ) + summary = { + 'files': dict() + } + for file in finder.find_all(): strfile = str(file) folders[str(file.parent)] = True @@ -73,30 +273,29 @@ def create_db(folder, options): if file.name in ['.delme'] or strfile in ['README.md', 'LICENSE', 'latest_linux.txt']: continue - db["files"][strfile] = { - "url": options['base_files_url'] + strfile, - "delete": delete_list(strfile, delete_list_regex), + summary["files"][strfile] = { "size": size(file), "hash": hash(file) } + delete_list = create_delete_list(strfile, delete_list_regex) + if len(delete_list) > 0: + summary["files"][strfile]["delete"] = delete_list + if file.name.lower() == "boot.rom": - db["files"][strfile]['overwrite'] = False + summary["files"][strfile]['overwrite'] = False if strfile in ['MiSTer', 'menu.rbf']: - db["files"][strfile]['path'] = 'system' - db["files"][strfile]['reboot'] = True + summary["files"][strfile]['path'] = 'system' + summary["files"][strfile]['reboot'] = True - folders.pop(folder, None) + folders.pop(finder.dir, None) - db["folders"] = sorted(list(folders.keys())) - db["files_count"] = len(db["files"]) - db["folders_count"] = len(db["folders"]) + summary['folders'] = sorted(list(folders.keys())) + summary['files_count'] = len(summary['files']) + summary['folders_count'] = len(summary['folders']) + return summary - if options['linux_github_repository'] != '': - db["linux"] = create_linux_description(options['linux_github_repository']) - - return db def create_linux_description(repository): sd_installer_output = run_stdout('curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/%s/git/trees/HEAD' % repository) @@ -111,12 +310,12 @@ def create_linux_description(repository): return { "url": url_linux, - "delete": [], "size": size(tmp_file.name), "hash": hash(tmp_file.name), "version": Path(latest_release).stem[-6:] } + def save_data_to_compressed_json(db, json_name, zip_name): with open(json_name, 'w') as f: @@ -125,6 +324,7 @@ def save_data_to_compressed_json(db, json_name, zip_name): run_succesfully('touch -a -m -t 202108231405 %s' % json_name) run_succesfully('zip -q -D -X -9 %s %s' % (zip_name, json_name)) + def force_push_file(file_name, branch): run_succesfully('git add %s' % file_name) run_succesfully('git commit -m "-"') @@ -132,6 +332,7 @@ def force_push_file(file_name, branch): print() print("New %s ready to be used." % file_name) + def run_conditional(command): result = subprocess.run(command, shell=True, stderr=subprocess.DEVNULL, stdout=subprocess.PIPE) @@ -141,6 +342,7 @@ def run_conditional(command): return result.returncode == 0 + def run_succesfully(command): result = subprocess.run(command, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) @@ -155,6 +357,7 @@ def run_succesfully(command): if result.returncode != 0: raise Exception("subprocess.run Return Code was '%d'" % result.returncode) + def run_stdout(command): result = subprocess.run(command, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) @@ -163,13 +366,15 @@ def run_stdout(command): return result.stdout.decode() -def delete_list(strfile, regex): + +def create_delete_list(strfile, regex): matches = regex.match(strfile) if matches: return [matches.group(1) + "*"] return [] + def hash(file): with open(file, "rb") as f: file_hash = hashlib.md5() @@ -179,24 +384,11 @@ def hash(file): chunk = f.read(8192) return file_hash.hexdigest() + def size(file): return os.path.getsize(file) -class Finder: - def __init__(self, dir): - self._dir = dir - self._not_in_directory = [dir + '/.git', dir + '/.github'] - - def find_all(self): - return sorted(self._scan(self._dir), key=lambda file: file.name.lower()) - - def _scan(self, directory): - for entry in os.scandir(directory): - if entry.is_dir(follow_symlinks=False): - if entry.path not in self._not_in_directory: - yield from self._scan(entry.path) - else: - yield Path(entry.path) if __name__ == '__main__': - main() + dryrun = len(sys.argv) == 2 and sys.argv[1] == '-d' + main(dryrun) diff --git a/.github/tests/test_calculate_db.py b/.github/tests/test_calculate_db.py index 3e1592889..e0be72fe8 100755 --- a/.github/tests/test_calculate_db.py +++ b/.github/tests/test_calculate_db.py @@ -6,12 +6,14 @@ calculate_db = importlib.util.module_from_spec(spec) spec.loader.exec_module(calculate_db) db = calculate_db.create_db('../..', { + 'sha': 3, 'latest_zip_url': 'w/', - 'base_files_url': 'x/', + 'base_files_url': 'x/%s/a', 'db_url': 'y/lala.json.zip', 'db_files': 'lala.json.zip', 'db_id': 'z/', - 'linux_github_repository': '' + 'linux_github_repository': '', + 'zips_config': '' }) calculate_db.save_data_to_compressed_json(db, 'db.json', 'db1.json.zip') diff --git a/.github/update_distribution.sh b/.github/update_distribution.sh index 709fc025b..64cff5b51 100755 --- a/.github/update_distribution.sh +++ b/.github/update_distribution.sh @@ -23,7 +23,10 @@ update_distribution() { done if [[ "${PUSH_COMMAND}" == "--push" ]] ; then - git checkout -f develop -b main + git checkout -f develop -b main + echo "Running detox" + detox -v -s utf_8-only -r * + echo "Detox done" git add "${OUTPUT_FOLDER}" git commit -m "-" git fetch origin main || true @@ -54,6 +57,7 @@ fetch_core_urls() { CORE_URLS=${CORE_URLS}$'\n'"https://raw.githubusercontent.com/MiSTer-devel/Scripts_MiSTer/master/rtc.sh" CORE_URLS=${CORE_URLS}$'\n'"https://raw.githubusercontent.com/MiSTer-devel/Scripts_MiSTer/master/timezone.sh" CORE_URLS=${CORE_URLS}$'\n'"user-content-folders"$'\n'"games/TGFX16-CD" + CORE_URLS=${CORE_URLS}$'\n'"user-cheats"$'\n'"https://gamehacking.org/mister/" } cat_local_core_urls() { @@ -72,6 +76,7 @@ classify_core_categories() { "user-content-service-cores") CURRENT_CORE_CATEGORY="_Utility" ;; "user-content-zip-release") ;& "user-content-scripts") ;& + "user-cheats") ;& "user-content-folders") ;& "user-content-fonts") CURRENT_CORE_CATEGORY="${url}" ;; "user-content-fpga-cores") ;& @@ -108,17 +113,18 @@ process_url() { local CATEGORY="${2}" local TARGET_DIR="${3}" + local EARLY_INSTALLER= case "${CATEGORY}" in - "user-content-scripts") - install_script "${URL}" "${TARGET_DIR}" - return - ;; - "user-content-folders") - install_folder "${URL}" "${TARGET_DIR}" - return - ;; + "user-content-scripts") EARLY_INSTALLER=install_script ;; + "user-content-folders") EARLY_INSTALLER=install_folder ;; + "user-cheats") EARLY_INSTALLER=install_cheats ;; *) ;; esac + + if [[ "${EARLY_INSTALLER}" != "" ]] ; then + ${EARLY_INSTALLER} "${URL}" "${TARGET_DIR}" + return + fi if ! [[ ${URL} =~ ^([a-zA-Z]+://)?github.com(:[0-9]+)?/([a-zA-Z0-9_-]*)/([a-zA-Z0-9_-]*)(/tree/([a-zA-Z0-9_-]+))?$ ]] ; then >&2 echo "WARNING! Wrong repository url '${URL}'." @@ -215,15 +221,12 @@ install_console_core() { continue fi - local PALETTES_TMP=$(mktemp -d) - unzip -o "${TMP_FOLDER}/palettes/${LAST_PALETTE_FILE}" -d "${PALETTES_TMP}/" - pushd "${PALETTES_TMP}" > /dev/null 2>&1 + local PALETTES_FOLDER="${TARGET_DIR}/games/${folder}/Palettes/" + mkdir -p "${PALETTES_FOLDER}" + unzip -q -o "${TMP_FOLDER}/palettes/${LAST_PALETTE_FILE}" -d "${PALETTES_FOLDER}" + pushd "${PALETTES_FOLDER}" > /dev/null 2>&1 find . -type f -not -iname '*.pal' -and -not -iname '*.gbp' -delete - find . -type f -print0 | while IFS= read -r -d '' file ; do touch -a -m -t 202108231405 "${file}" ; done - zip -q -0 -D -X -A -r "Palettes.zip" * popd > /dev/null 2>&1 - mv "${PALETTES_TMP}/Palettes.zip" "${TARGET_DIR}/games/${folder}/Palettes.zip" - rm -rf "${PALETTES_TMP}" done touch_folder "${TARGET_DIR}/games/${folder}" @@ -367,7 +370,8 @@ install_zip_release() { continue fi - unzip -o "${TMP_FOLDER}/releases/${GET_LATEST_RELEASE_RET}" -d "${TARGET_DIR}/" + echo "unzip ${TMP_FOLDER}/releases/${GET_LATEST_RELEASE_RET} to ${TARGET_DIR}/" + unzip -q -o "${TMP_FOLDER}/releases/${GET_LATEST_RELEASE_RET}" -d "${TARGET_DIR}/" done } @@ -398,6 +402,42 @@ install_folder() { touch_folder "${TARGET_DIR}/${URL}" } + +declare -A CHEAT_MAPPINGS=( \ + ["fds"]="NES" \ + ["gb"]="GameBoy" \ + ["gba"]="GBA" \ + ["gbc"]="GameBoy" \ + ["gen"]="Genesis" \ + ["gg"]="SMS" \ + ["lnx"]="AtariLynx" \ + ["nes"]="NES" \ + ["pce"]="TGFX16" \ + ["pcd"]="TGFX16" \ + ["scd"]="MegaCD" \ + ["sms"]="SMS" \ + ["snes"]="SNES" \ +) + +install_cheats() { + local URL="${1}" + local TARGET_DIR="${2}" + + mkdir -p "${TARGET_DIR}/Cheats/" + + local CHEAT_URLS=$(curl -sSLf --cookie "challenge=BitMitigate.com" "${URL}" | grep -oE '"mister_[^_]+_[0-9]{8}.zip"' | sed 's/"//g') + for cheat_key in ${!CHEAT_MAPPINGS[@]} ; do + local cheat_platform=${CHEAT_MAPPINGS[${cheat_key}]} + local cheat_zip=$(echo "${CHEAT_URLS}" | grep "mister_${cheat_key}_") + local cheat_url="${URL}${cheat_zip}" + echo "cheat_key: ${cheat_key}, cheat_platform: ${cheat_platform}, cheat_zip: ${cheat_zip}, cheat_url: ${cheat_url}" + + mkdir -p "${TARGET_DIR}/Cheats/${cheat_platform}" + curl --silent --show-error --fail --location -o "/tmp/${cheat_platform}.zip" "${cheat_url}" + unzip -q -o "/tmp/${cheat_platform}.zip" -d "${TARGET_DIR}/Cheats/${cheat_platform}" + done +} + GET_LATEST_RELEASE_RET= get_latest_release() { echo "BINARY_NAME: ${2}" diff --git a/.github/workflows/update_distribution.yml b/.github/workflows/update_distribution.yml index a0eb6ac4d..26b78699b 100644 --- a/.github/workflows/update_distribution.yml +++ b/.github/workflows/update_distribution.yml @@ -10,6 +10,9 @@ jobs: runs-on: ubuntu-20.04 steps: + - name: Install detox + run: sudo apt-get install detox + - uses: actions/checkout@v2 with: ref: develop @@ -19,10 +22,14 @@ jobs: set -euo pipefail git config --global user.email "theypsilon@gmail.com" git config --global user.name "The CI/CD Bot" + + export DB_ID=distribution_mister + export DB_URL=https://raw.githubusercontent.com/MiSTer-devel/Distribution_MiSTer/main/db.json.zip + export BASE_FILES_URL=https://raw.githubusercontent.com/MiSTer-devel/Distribution_MiSTer/%s/ + export LATEST_ZIP_URL=https://github.com/MiSTer-devel/Distribution_MiSTer/archive/refs/heads/main.zip + export ZIPS_CONFIG=./.github/zips_config.json + export LINUX_GITHUB_REPOSITORY=MiSTer-devel/SD-Installer-Win64_MiSTer + + ./.github/update_distribution.sh . --push - env: - DB_ID: distribution_mister - DB_URL: https://raw.githubusercontent.com/MiSTer-devel/Distribution_MiSTer/main/db.json.zip - BASE_FILES_URL: https://raw.githubusercontent.com/MiSTer-devel/Distribution_MiSTer/%s/ - LATEST_ZIP_URL: https://github.com/MiSTer-devel/Distribution_MiSTer/archive/refs/heads/main.zip - LINUX_GITHUB_REPOSITORY: MiSTer-devel/SD-Installer-Win64_MiSTer + diff --git a/.github/zips_config.json b/.github/zips_config.json new file mode 100644 index 000000000..d6fd1ad8c --- /dev/null +++ b/.github/zips_config.json @@ -0,0 +1,26 @@ +{ + "cheats_folder_": { + "source": "Cheats", + "mode": "subfolders" + }, + "nes_palettes": { + "source": "games/NES/Palettes" + }, + "gameboy_palettes": { + "source": "games/GAMEBOY/Palettes" + }, + "gameboy2p_palettes": { + "source": "games/GAMEBOY2P/Palettes" + }, + "global_filters": { + "path": ".", + "sources": ["Filters", "Filters_Audio", "Gamma"], + "mode": "multi" + }, + "global_fonts": { + "source": "font" + }, + "mra_alternatives": { + "source": "_Arcade/_alternatives" + } +} \ No newline at end of file