Downloader Release 1.1, +Cheats, +Palettes, +Zips

2026-05-24 03:03:20 +00:00 · 2021-10-16 17:25:32 +02:00
parent 09f77c761b
commit 0ccf0ce03f
5 changed files with 334 additions and 67 deletions
--- a/.github/calculate_db.py
+++ b/.github/calculate_db.py
@@ -12,6 +12,43 @@ import sys
 from datetime import datetime
 import os
 import tempfile
+import shutil
+from typing import Protocol, Any
+
+
+class Finder:
+    def __init__(self, dir):
+        self._dir = dir
+        self._not_in_directory = []
+
+    @property
+    def dir(self):
+        return self._dir
+
+    def ignore_folder(self, folder):
+        directory = str(Path(folder))
+        print('ignore_folder: %s' % directory)
+        self._not_in_directory.append(directory)
+
+    def find_all(self):
+        return sorted(self._scan(self._dir), key=lambda file: str(file).lower())
+
+    def _scan(self, directory):
+        for entry in os.scandir(directory):
+            if entry.is_dir(follow_symlinks=False):
+                if str(Path(entry.path)) not in self._not_in_directory:
+                    yield from self._scan(entry.path)
+            else:
+                yield Path(entry.path)
+
+
+class EmptyFinder(Finder):
+    def __init__(self):
+        Finder.__init__(self, None)
+
+    def find_all(self):
+        return []
+

 def benchtime(func):
    def benchfn(*args, **kwargs):
@@ -23,8 +60,9 @@ def benchtime(func):

    return benchfn

+
@benchtime
-def main():
+def main(dryrun):
    sha = run_stdout('git rev-parse --verify HEAD').strip()
    print('sha: %s' % sha)

@@ -33,39 +71,201 @@ def main():
    db_file_json = Path(db_url).stem

    db = create_db('.', {
-        'base_files_url': envvar('BASE_FILES_URL') % sha,
+        'sha': sha,
+        'base_files_url': envvar('BASE_FILES_URL'),
        'db_url': db_url,
        'db_files': [db_file_zip],
        'db_id': envvar('DB_ID'),
+        'dryrun': dryrun,
        'latest_zip_url': envvar('LATEST_ZIP_URL'),
-        'linux_github_repository': os.getenv('LINUX_GITHUB_REPOSITORY', '').strip()
+        'linux_github_repository': os.getenv('LINUX_GITHUB_REPOSITORY', '').strip(),
+        'zips_config': os.getenv('ZIPS_CONFIG', '').strip()
    })

    save_data_to_compressed_json(db, db_file_json, db_file_zip)
-    force_push_file(db_file_zip, 'main')
+    if not dryrun:
+        force_push_file(db_file_zip, 'main')
+

 def envvar(var):
    result = os.getenv(var)
    print("{} = {}".format(var, result))
    return result

+
 def create_db(folder, options):
-
-    finder = Finder(folder)
-    folders = dict()
-
+    db_finder = Finder(folder)
+    db_finder.ignore_folder('./.git')
+    db_finder.ignore_folder('./.github')
    db = {
        "db_id": options['db_id'],
        "db_url": options['db_url'],
        "db_files": options['db_files'],
        "latest_zip_url": options['latest_zip_url'],
        "files": dict(),
-        "zips": dict(),
-        "base_files_url": ""
+        "base_files_url": options['base_files_url'] % options['sha']
    }

+    zips = dict()
+
+    if options['zips_config'] != '':
+        print('reading zips_config: ' + options['zips_config'])
+        with open(options['zips_config']) as zips_config_file:
+            zips_config = json.load(zips_config_file)
+            for zip_id in zips_config:
+                zip_description = zips_config[zip_id]
+                zip_creator = make_zip_creator(zip_description)
+                zip_creator.create_zip(db_finder, zips, zip_id, zip_description, options)
+
+    if len(zips) > 0:
+
+        current_branch = run_stdout('git rev-parse --abbrev-ref HEAD').strip()
+        if current_branch == 'zips':
+            raise Exception('Should not start on branch "zip"')
+
+        run_succesfully('git branch -D zips || true')
+        run_succesfully('git checkout --orphan zips')
+        run_succesfully('git reset')
+
+        for zip_id in zips:
+            run_succesfully('git add %s.zip' % zip_id)
+            run_succesfully('git add %s_summary.json.zip' % zip_id)
+
+        run_succesfully('git commit -m "-"')
+        zip_sha = run_stdout('git rev-parse --verify HEAD').strip()
+
+        if not options['dryrun']:
+            run_succesfully('git fetch origin main || true')
+            if not run_conditional('git diff --quiet main origin/zip'):
+                print('zip branch has changes')
+                run_succesfully('git push --force origin zips')
+            else:
+                print('Using old zip branch from origin')
+                zip_sha = run_stdout('git rev-parse --verify origin/zip').strip()
+
+        print('zip_sha: ' + zip_sha)
+
+        run_succesfully('git checkout --force ' + current_branch)
+
+        for zip_id in zips:
+            zips[zip_id]['contents_file']['url'] = (options['base_files_url'] % zip_sha) + '%s.zip' % zip_id
+            zips[zip_id]['summary_file']['url'] = (options['base_files_url'] % zip_sha) + '%s_summary.json.zip' % zip_id
+
+        db['zips'] = zips
+
+    db_summary = create_summary(db_finder, options)
+    db['files'] = db_summary['files']
+    db['folders'] = db_summary['folders']
+    db['files_count'] = db_summary['files_count']
+    db['folders_count'] = db_summary['folders_count']
+
+    if options['linux_github_repository'] != '':
+        db["linux"] = create_linux_description(options['linux_github_repository'])
+
+    return db
+
+
+class ZipCreator(Protocol):
+    def create_zip(self, db_finder: Finder, zips: [str, Any], zip_id: str, zip_description: [str, Any], options: [str, Any]) -> None:
+        pass
+
+
+def make_zip_creator(zip_description: [str, Any]) -> ZipCreator:
+    mode = zip_description['mode'] if 'mode' in zip_description else 'simple'
+
+    if mode == 'simple':
+        return SimpleZipCreator()
+    elif mode == 'subfolders':
+        return SubfoldersZipCreator()
+    elif mode == 'multi':
+        return MultiSourcesZipCreator()
+    else:
+        raise NotImplementedError('No ZipCreator for mode: ' + mode)
+
+
+class SimpleZipCreator:
+    def create_zip(self, db_finder: Finder, zips: [str, Any], zip_id: str, zip_description: [str, Any], options: [str, Any]) -> None:
+        source_path = Path(zip_description['source'])
+        zip_description['sources'] = [source_path.name]
+        zip_description['path'] = str(source_path.parent)
+        multi = MultiSourcesZipCreator()
+        multi.create_zip(db_finder, zips, zip_id, zip_description, options)
+        return
+
+
+class SubfoldersZipCreator:
+    def create_zip(self, db_finder: Finder, zips: [str, Any], zip_id: str, zip_description: [str, Any], options: [str, Any]) -> None:
+        simple = SimpleZipCreator()
+        for folder in [entry.path for entry in os.scandir(zip_description['source']) if entry.is_dir(follow_symlinks=False)]:
+            if len(Finder(folder).find_all()) < 60:
+                continue
+            simple.create_zip(db_finder, zips, zip_id + Path(folder).name.lower(), {"source": folder}, options)
+
+
+class MultiSourcesZipCreator:
+    def create_zip(self, db_finder: Finder, zips: [str, Any], zip_id: str, zip_description: [str, Any], options: [str, Any]) -> None:
+        print('Processing zip_id: %s' % zip_id)
+
+        source_parent = zip_description['path']
+        summary_name = '%s_summary.json' % zip_id
+
+        multi_summary = create_summary(EmptyFinder(), options)
+
+        source_name_list = []
+        for source in zip_description['sources']:
+            db_finder.ignore_folder('./' + source_parent + '/' + source)
+            zip_finder = Finder(source_parent + '/' + source)
+            zip_summary = create_summary(zip_finder, options)
+            zip_summary['folders'].append(str(Path(source_parent + '/' + source)))
+            multi_summary['files_count'] += zip_summary['files_count']
+            multi_summary['folders_count'] += zip_summary['folders_count']
+            multi_summary['files'].update(zip_summary['files'])
+            multi_summary['folders'].extend(zip_summary['folders'])
+            source_name_list.append(Path(source).name)
+
+        multi_summary['folders'] = list(set(multi_summary['folders']))
+
+        zip_description['raw_files_size'] = 0
+        zip_description['path'] = source_parent + '/'
+        zip_description['contents'] = zip_description['sources']
+        zip_description['files_count'] = multi_summary['files_count']
+        zip_description['folders_count'] = multi_summary['folders_count']
+        zip_description['base_files_url'] = options['base_files_url'] % options['sha']
+        zip_description.pop('sources')
+
+        for file in multi_summary['files']:
+            multi_summary['files'][file]['zip_id'] = zip_id
+            zip_description['raw_files_size'] += multi_summary['files'][file]['size']
+
+        summary_zip = summary_name + '.zip'
+
+        save_data_to_compressed_json(multi_summary, summary_name, summary_zip)
+        zip_description['summary_file'] = {
+            'size': size(summary_zip),
+            'hash': hash(summary_zip)
+        }
+        Path(summary_name).unlink()
+
+        zip_name = zip_id + '.zip'
+
+        run_succesfully('cur=$(pwd) && cd %s && zip -q -D -X -A -r $cur/%s %s' % (source_parent, zip_name, " ".join(source_name_list)))
+        zip_description['contents_file'] = {
+            'size': size(zip_name),
+            'hash': hash(zip_name)
+        }
+
+        zips[zip_id] = zip_description
+        print('Created zip: ' + zip_name)
+
+
+def create_summary(finder: Finder, options):
+    folders = dict()
    delete_list_regex = re.compile("^(.*_)[0-9]{8}(\.[a-zA-Z0-9]+)*$", )

+    summary = {
+        'files': dict()
+    }
+
    for file in finder.find_all():
        strfile = str(file)
        folders[str(file.parent)] = True
@@ -73,30 +273,29 @@ def create_db(folder, options):
        if file.name in ['.delme'] or strfile in ['README.md', 'LICENSE', 'latest_linux.txt']:
            continue

-        db["files"][strfile] = {
-            "url": options['base_files_url'] + strfile,
-            "delete": delete_list(strfile, delete_list_regex),
+        summary["files"][strfile] = {
            "size": size(file),
            "hash": hash(file)
        }

+        delete_list = create_delete_list(strfile, delete_list_regex)
+        if len(delete_list) > 0:
+            summary["files"][strfile]["delete"] = delete_list
+
        if file.name.lower() == "boot.rom":
-            db["files"][strfile]['overwrite'] = False
+            summary["files"][strfile]['overwrite'] = False

        if strfile in ['MiSTer', 'menu.rbf']:
-            db["files"][strfile]['path'] = 'system'
-            db["files"][strfile]['reboot'] = True
+            summary["files"][strfile]['path'] = 'system'
+            summary["files"][strfile]['reboot'] = True

-    folders.pop(folder, None)
+    folders.pop(finder.dir, None)

-    db["folders"] = sorted(list(folders.keys()))
-    db["files_count"] = len(db["files"])
-    db["folders_count"] = len(db["folders"])
+    summary['folders'] = sorted(list(folders.keys()))
+    summary['files_count'] = len(summary['files'])
+    summary['folders_count'] = len(summary['folders'])
+    return summary

-    if options['linux_github_repository'] != '':
-        db["linux"] = create_linux_description(options['linux_github_repository'])
-
-    return db

 def create_linux_description(repository):
    sd_installer_output = run_stdout('curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/%s/git/trees/HEAD' % repository)
@@ -111,12 +310,12 @@ def create_linux_description(repository):

        return {
            "url": url_linux,
-            "delete": [],
            "size": size(tmp_file.name),
            "hash": hash(tmp_file.name),
            "version": Path(latest_release).stem[-6:]
        }

+
 def save_data_to_compressed_json(db, json_name, zip_name):

    with open(json_name, 'w') as f:
@@ -125,6 +324,7 @@ def save_data_to_compressed_json(db, json_name, zip_name):
    run_succesfully('touch -a -m -t 202108231405 %s' % json_name)
    run_succesfully('zip -q -D -X -9  %s %s' % (zip_name, json_name))

+
 def force_push_file(file_name, branch):
    run_succesfully('git add %s' % file_name)
    run_succesfully('git commit -m "-"')
@@ -132,6 +332,7 @@ def force_push_file(file_name, branch):
    print()
    print("New %s ready to be used." % file_name)

+
 def run_conditional(command):
    result = subprocess.run(command, shell=True, stderr=subprocess.DEVNULL, stdout=subprocess.PIPE)

@@ -141,6 +342,7 @@ def run_conditional(command):
        
    return result.returncode == 0

+
 def run_succesfully(command):
    result = subprocess.run(command, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)

@@ -155,6 +357,7 @@ def run_succesfully(command):
    if result.returncode != 0:
        raise Exception("subprocess.run Return Code was '%d'" % result.returncode)

+
 def run_stdout(command):
    result = subprocess.run(command, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)

@@ -163,13 +366,15 @@ def run_stdout(command):

    return result.stdout.decode()

-def delete_list(strfile, regex):
+
+def create_delete_list(strfile, regex):
    matches = regex.match(strfile)
    if matches:
        return [matches.group(1) + "*"]

    return []

+
 def hash(file):
    with open(file, "rb") as f:
        file_hash = hashlib.md5()
@@ -179,24 +384,11 @@ def hash(file):
            chunk = f.read(8192)
        return file_hash.hexdigest()

+
 def size(file):
    return os.path.getsize(file)

-class Finder:
-    def __init__(self, dir):
-        self._dir = dir
-        self._not_in_directory = [dir + '/.git', dir + '/.github']
-
-    def find_all(self):
-        return sorted(self._scan(self._dir), key=lambda file: file.name.lower())
-
-    def _scan(self, directory):
-        for entry in os.scandir(directory):
-            if entry.is_dir(follow_symlinks=False):
-                if entry.path not in self._not_in_directory:
-                    yield from self._scan(entry.path)
-            else:
-                yield Path(entry.path)

 if __name__ == '__main__':
-    main()
+    dryrun = len(sys.argv) == 2 and sys.argv[1] == '-d'
+    main(dryrun)
--- a/.github/tests/test_calculate_db.py
+++ b/.github/tests/test_calculate_db.py
@@ -6,12 +6,14 @@ calculate_db = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(calculate_db)

 db = calculate_db.create_db('../..', {
+    'sha': 3,
    'latest_zip_url': 'w/',
-    'base_files_url': 'x/',
+    'base_files_url': 'x/%s/a',
    'db_url': 'y/lala.json.zip',
    'db_files': 'lala.json.zip',
    'db_id': 'z/',
-    'linux_github_repository': ''
+    'linux_github_repository': '',
+    'zips_config': ''
 })

 calculate_db.save_data_to_compressed_json(db, 'db.json', 'db1.json.zip')
--- a/.github/update_distribution.sh
+++ b/.github/update_distribution.sh
@@ -23,7 +23,10 @@ update_distribution() {
    done

    if [[ "${PUSH_COMMAND}" == "--push" ]] ; then
-        git checkout -f develop -b main 
+        git checkout -f develop -b main
+        echo "Running detox"
+        detox -v -s utf_8-only -r *
+        echo "Detox done"
        git add "${OUTPUT_FOLDER}"
        git commit -m "-"
        git fetch origin main || true
@@ -54,6 +57,7 @@ fetch_core_urls() {
    CORE_URLS=${CORE_URLS}$'\n'"https://raw.githubusercontent.com/MiSTer-devel/Scripts_MiSTer/master/rtc.sh"
    CORE_URLS=${CORE_URLS}$'\n'"https://raw.githubusercontent.com/MiSTer-devel/Scripts_MiSTer/master/timezone.sh"
    CORE_URLS=${CORE_URLS}$'\n'"user-content-folders"$'\n'"games/TGFX16-CD"
+    CORE_URLS=${CORE_URLS}$'\n'"user-cheats"$'\n'"https://gamehacking.org/mister/"
 }

 cat_local_core_urls() {
@@ -72,6 +76,7 @@ classify_core_categories() {
            "user-content-service-cores") CURRENT_CORE_CATEGORY="_Utility" ;;
            "user-content-zip-release") ;&
            "user-content-scripts") ;&
+            "user-cheats") ;&
            "user-content-folders") ;&
            "user-content-fonts") CURRENT_CORE_CATEGORY="${url}" ;;
            "user-content-fpga-cores") ;&
@@ -108,17 +113,18 @@ process_url() {
    local CATEGORY="${2}"
    local TARGET_DIR="${3}"

+    local EARLY_INSTALLER=
    case "${CATEGORY}" in
-        "user-content-scripts")
-            install_script "${URL}" "${TARGET_DIR}"
-            return
-            ;;
-        "user-content-folders")
-            install_folder "${URL}" "${TARGET_DIR}"
-            return
-            ;;
+        "user-content-scripts") EARLY_INSTALLER=install_script ;;
+        "user-content-folders") EARLY_INSTALLER=install_folder ;;
+        "user-cheats") EARLY_INSTALLER=install_cheats ;;
        *) ;;
    esac
+    
+    if [[ "${EARLY_INSTALLER}" != "" ]] ; then
+        ${EARLY_INSTALLER} "${URL}" "${TARGET_DIR}"
+        return
+    fi

    if ! [[ ${URL} =~ ^([a-zA-Z]+://)?github.com(:[0-9]+)?/([a-zA-Z0-9_-]*)/([a-zA-Z0-9_-]*)(/tree/([a-zA-Z0-9_-]+))?$ ]] ; then
        >&2 echo "WARNING! Wrong repository url '${URL}'."
@@ -215,15 +221,12 @@ install_console_core() {
                continue
            fi

-            local PALETTES_TMP=$(mktemp -d)
-            unzip -o "${TMP_FOLDER}/palettes/${LAST_PALETTE_FILE}" -d "${PALETTES_TMP}/"
-            pushd "${PALETTES_TMP}" > /dev/null 2>&1
+            local PALETTES_FOLDER="${TARGET_DIR}/games/${folder}/Palettes/"
+            mkdir -p "${PALETTES_FOLDER}"
+            unzip -q -o "${TMP_FOLDER}/palettes/${LAST_PALETTE_FILE}" -d "${PALETTES_FOLDER}"
+            pushd "${PALETTES_FOLDER}" > /dev/null 2>&1
            find . -type f -not -iname '*.pal' -and -not -iname '*.gbp' -delete
-            find . -type f -print0 | while IFS= read -r -d '' file ; do touch -a -m -t 202108231405 "${file}" ; done
-            zip -q -0 -D -X -A -r "Palettes.zip" *
            popd > /dev/null 2>&1
-            mv "${PALETTES_TMP}/Palettes.zip" "${TARGET_DIR}/games/${folder}/Palettes.zip"
-            rm -rf "${PALETTES_TMP}"
        done

        touch_folder "${TARGET_DIR}/games/${folder}"
@@ -367,7 +370,8 @@ install_zip_release() {
            continue
        fi

-        unzip -o "${TMP_FOLDER}/releases/${GET_LATEST_RELEASE_RET}" -d "${TARGET_DIR}/"
+        echo "unzip ${TMP_FOLDER}/releases/${GET_LATEST_RELEASE_RET} to ${TARGET_DIR}/"
+        unzip -q -o "${TMP_FOLDER}/releases/${GET_LATEST_RELEASE_RET}" -d "${TARGET_DIR}/"
    done
 }

@@ -398,6 +402,42 @@ install_folder() {
    touch_folder "${TARGET_DIR}/${URL}"
 }

+
+declare -A CHEAT_MAPPINGS=( \
+    ["fds"]="NES" \
+    ["gb"]="GameBoy" \
+    ["gba"]="GBA" \
+    ["gbc"]="GameBoy" \
+    ["gen"]="Genesis" \
+    ["gg"]="SMS" \
+    ["lnx"]="AtariLynx" \
+    ["nes"]="NES" \
+    ["pce"]="TGFX16" \
+    ["pcd"]="TGFX16" \
+    ["scd"]="MegaCD" \
+    ["sms"]="SMS" \
+    ["snes"]="SNES" \
+)
+
+install_cheats() {
+    local URL="${1}"
+    local TARGET_DIR="${2}"
+
+    mkdir -p "${TARGET_DIR}/Cheats/"
+
+    local CHEAT_URLS=$(curl -sSLf --cookie "challenge=BitMitigate.com" "${URL}" | grep -oE '"mister_[^_]+_[0-9]{8}.zip"' | sed 's/"//g')
+    for cheat_key in ${!CHEAT_MAPPINGS[@]} ; do
+        local cheat_platform=${CHEAT_MAPPINGS[${cheat_key}]}
+        local cheat_zip=$(echo "${CHEAT_URLS}" | grep "mister_${cheat_key}_")
+        local cheat_url="${URL}${cheat_zip}"
+        echo "cheat_key: ${cheat_key}, cheat_platform: ${cheat_platform}, cheat_zip: ${cheat_zip}, cheat_url: ${cheat_url}"
+
+        mkdir -p "${TARGET_DIR}/Cheats/${cheat_platform}"
+        curl --silent --show-error --fail --location -o "/tmp/${cheat_platform}.zip" "${cheat_url}"
+        unzip -q -o "/tmp/${cheat_platform}.zip" -d "${TARGET_DIR}/Cheats/${cheat_platform}"
+    done
+}
+
 GET_LATEST_RELEASE_RET=
 get_latest_release() {
    echo "BINARY_NAME: ${2}"
--- a/.github/workflows/update_distribution.yml
+++ b/.github/workflows/update_distribution.yml
@@ -10,6 +10,9 @@ jobs:
    runs-on: ubuntu-20.04

    steps:
+    - name: Install detox
+      run: sudo apt-get install detox
+
    - uses: actions/checkout@v2
      with:
        ref: develop
@@ -19,10 +22,14 @@ jobs:
        set -euo pipefail
        git config --global user.email "theypsilon@gmail.com"
        git config --global user.name "The CI/CD Bot"
+
+        export DB_ID=distribution_mister
+        export DB_URL=https://raw.githubusercontent.com/MiSTer-devel/Distribution_MiSTer/main/db.json.zip
+        export BASE_FILES_URL=https://raw.githubusercontent.com/MiSTer-devel/Distribution_MiSTer/%s/
+        export LATEST_ZIP_URL=https://github.com/MiSTer-devel/Distribution_MiSTer/archive/refs/heads/main.zip
+        export ZIPS_CONFIG=./.github/zips_config.json
+        export LINUX_GITHUB_REPOSITORY=MiSTer-devel/SD-Installer-Win64_MiSTer
+
+
        ./.github/update_distribution.sh . --push
-      env:
-        DB_ID: distribution_mister
-        DB_URL: https://raw.githubusercontent.com/MiSTer-devel/Distribution_MiSTer/main/db.json.zip
-        BASE_FILES_URL: https://raw.githubusercontent.com/MiSTer-devel/Distribution_MiSTer/%s/
-        LATEST_ZIP_URL: https://github.com/MiSTer-devel/Distribution_MiSTer/archive/refs/heads/main.zip
-        LINUX_GITHUB_REPOSITORY: MiSTer-devel/SD-Installer-Win64_MiSTer
+
--- a/.github/zips_config.json
+++ b/.github/zips_config.json
@@ -0,0 +1,26 @@
+{
+    "cheats_folder_": {
+        "source": "Cheats",
+        "mode": "subfolders"
+    },
+    "nes_palettes": {
+        "source": "games/NES/Palettes"
+    },
+    "gameboy_palettes": {
+        "source": "games/GAMEBOY/Palettes"
+    },
+    "gameboy2p_palettes": {
+        "source": "games/GAMEBOY2P/Palettes"
+    },
+    "global_filters": {
+        "path": ".",
+        "sources": ["Filters", "Filters_Audio", "Gamma"],
+        "mode": "multi"
+    },
+    "global_fonts": {
+        "source": "font"
+    },
+    "mra_alternatives": {
+        "source": "_Arcade/_alternatives"
+    }
+}