From c9994f9dd2bd1ff0108193ca185041bb8ba123d6 Mon Sep 17 00:00:00 2001 From: Rikard Bengtsson <36540711+yxkalle@users.noreply.github.com> Date: Fri, 6 Oct 2023 20:36:57 +0200 Subject: [PATCH] Cleanup of n64.cpp (#833) * Squashed commit of the following: commit 40c58abf1d87e870d20ab77bf2934df6ed6d7b94 Merge: 1fc9c37 7c65b22 Author: Rikard Bengtsson Date: Fri Oct 6 09:00:07 2023 +0200 Merge branch 'master' of https://github.com/yxkalle/Main_MiSTer commit 1fc9c37c070b54719aa6ae14890db5fc6bcec3e1 Author: Rikard Bengtsson Date: Fri Oct 6 08:57:57 2023 +0200 N64: Just some cleanup. Made code simpler. Added some comments where needed. commit 7c65b22cab1a33f599b3a7ecb2faaaa27613046b Author: Rikard Bengtsson Date: Fri Oct 6 04:09:51 2023 +0200 Delete MiSTer_noface commit fe2169544f5ce5ace125109bf65df16dc65594a2 Merge: c163956 ee5659a Author: Rikard Bengtsson Date: Fri Oct 6 03:28:02 2023 +0200 Merge branch 'master' into add-id-to-db commit c1639563da1ea714575f51f27a21df9c2f493a6e Author: Rikard Bengtsson Date: Fri Oct 6 03:15:05 2023 +0200 Update n64.cpp commit f4a6b98b5dd650bc13e08db8bb21e3da31fea64b Author: Rikard Bengtsson Date: Fri Oct 6 03:12:51 2023 +0200 cleanup commit 401b9dc6a9b2d0dd06c18be1e48014b3dc81e946 Author: Rikard Bengtsson Date: Tue Oct 3 22:48:03 2023 +0200 Clean-up Clean-up the code, added some comments. Allow uppercase md5s in the database. commit e0e2affdfe277577820ba73a202b06311b645ec7 Author: Rikard Bengtsson Date: Mon Oct 2 20:05:28 2023 +0200 Cleaner code? commit 554666832bd55d924b531f469a45cb7996f13e03 Author: Rikard Bengtsson Date: Sun Oct 1 02:13:00 2023 +0200 Delete N64-database.txt Delete before PR commit 9e8fc3f30c36ea9de59d4f1801a4341edbf73eb8 Author: Rikard Bengtsson Date: Sun Oct 1 02:09:07 2023 +0200 Some commenting and clean-up commit be19bb6e679cd76d32f9b0382c08052ba1e68944 Author: Rikard Bengtsson Date: Sun Oct 1 01:19:05 2023 +0200 Autodetect settings by ID Settings are saved in the database text files as usual, but you can match by both MD5 (first 4096), MD5 (full) as well as ID + region + revision. * Added support for hacked boot codes --- support/n64/n64.cpp | 210 +++++++++++++++++++++++++++----------------- 1 file changed, 127 insertions(+), 83 deletions(-) diff --git a/support/n64/n64.cpp b/support/n64/n64.cpp index 6107928..9058bbb 100644 --- a/support/n64/n64.cpp +++ b/support/n64/n64.cpp @@ -13,6 +13,8 @@ static constexpr uint64_t FNV_PRIME = 0x100000001b3; static constexpr uint64_t FNV_OFFSET_BASIS = 0xcbf29ce484222325; +static constexpr uint8_t CARTID_LENGTH = 6; +static constexpr auto CARTID_PREFIX = "ID:"; static constexpr uint64_t fnv_hash(const char *s, uint64_t h = FNV_OFFSET_BASIS) { if (s) while (uint8_t a = *(s++)) h = (h ^ a) * FNV_PRIME; @@ -29,7 +31,8 @@ enum class MemoryType { }; enum class CIC { - CIC_NUS_6101 = 0, + UNKNOWN = -1, + CIC_NUS_6101, CIC_NUS_6102, CIC_NUS_7101, CIC_NUS_7102, @@ -46,7 +49,8 @@ enum class CIC { }; enum class SystemType { - NTSC = 0, + UNKNOWN = -1, + NTSC, PAL }; @@ -62,22 +66,21 @@ enum class AutoDetect { OFF = 1, }; -static RomFormat detectRomFormat(const uint8_t* data) -{ +static RomFormat detectRomFormat(const uint8_t* data) { // data should be aligned const uint32_t val = *(uint32_t*)data; // the following checks assume we're on a little-endian platform // for each check, the first value is for regular roms, the 2nd is for 64DD images - if (val == 0x40123780 || val == 0x40072780) return RomFormat::BIG_ENDIAN; - if (val == 0x12408037 || val == 0x07408027) return RomFormat::BYTE_SWAPPED; - if (val == 0x80371240 || val == 0x80270740) return RomFormat::LITTLE_ENDIAN; + // third is a malformed magic word used in homebrew (mostly pointless) + if (val == UINT32_C(0x40123780) || val == UINT32_C(0x40072780) || val == UINT32_C(0x41123780)) return RomFormat::BIG_ENDIAN; + if (val == UINT32_C(0x12408037) || val == UINT32_C(0x07408027) || val == UINT32_C(0x12418037)) return RomFormat::BYTE_SWAPPED; + if (val == UINT32_C(0x80371240) || val == UINT32_C(0x80270740) || val == UINT32_C(0x80371241)) return RomFormat::LITTLE_ENDIAN; return RomFormat::UNKNOWN; } -static void normalizeData(uint8_t* data, size_t size, RomFormat format) -{ +static void normalizeData(uint8_t* data, size_t size, RomFormat format) { switch(format) { case RomFormat::BYTE_SWAPPED: for (size_t i = 0; i < size; i += 2) { @@ -115,14 +118,12 @@ static void normalizeString(char* s) { // return true if cic and system region is detected static bool parse_and_apply_db_tags(char* tags) { MemoryType save_type = MemoryType::NONE; - SystemType system_type = SystemType::NTSC; - CIC cic = CIC::CIC_NUS_6102; + SystemType system_type = SystemType::UNKNOWN; + CIC cic = CIC::UNKNOWN; bool cpak = false; bool rpak = false; bool tpak = false; bool rtc = false; - bool system_type_known = false; - bool cic_known = false; const char separator[] = "|"; @@ -136,27 +137,27 @@ static bool parse_and_apply_db_tags(char* tags) { case fnv_hash("sram32k"): save_type = MemoryType::SRAM_32k; break; case fnv_hash("sram96k"): save_type = MemoryType::SRAM_96k; break; case fnv_hash("flash128k"): save_type = MemoryType::FLASH_128k; break; - case fnv_hash("ntsc"): system_type = SystemType::NTSC; system_type_known = true; break; - case fnv_hash("pal"): system_type = SystemType::PAL; system_type_known = true; break; + case fnv_hash("ntsc"): system_type = SystemType::NTSC; break; + case fnv_hash("pal"): system_type = SystemType::PAL; break; case fnv_hash("cpak"): cpak = true; break; case fnv_hash("rpak"): rpak = true; break; case fnv_hash("tpak"): tpak = true; break; case fnv_hash("rtc"): rtc = true; break; - case fnv_hash("cic6101"): cic = CIC::CIC_NUS_6101; cic_known = true; break; - case fnv_hash("cic6102"): cic = CIC::CIC_NUS_6102; cic_known = true; break; - case fnv_hash("cic6103"): cic = CIC::CIC_NUS_6103; cic_known = true; break; - case fnv_hash("cic6105"): cic = CIC::CIC_NUS_6105; cic_known = true; break; - case fnv_hash("cic6106"): cic = CIC::CIC_NUS_6106; cic_known = true; break; - case fnv_hash("cic7101"): cic = CIC::CIC_NUS_7101; cic_known = true; break; - case fnv_hash("cic7102"): cic = CIC::CIC_NUS_7102; cic_known = true; break; - case fnv_hash("cic7103"): cic = CIC::CIC_NUS_7103; cic_known = true; break; - case fnv_hash("cic7105"): cic = CIC::CIC_NUS_7105; cic_known = true; break; - case fnv_hash("cic7106"): cic = CIC::CIC_NUS_7106; cic_known = true; break; - case fnv_hash("cic8303"): cic = CIC::CIC_NUS_8303; cic_known = true; break; - case fnv_hash("cic8401"): cic = CIC::CIC_NUS_8401; cic_known = true; break; - case fnv_hash("cic5167"): cic = CIC::CIC_NUS_5167; cic_known = true; break; - case fnv_hash("cicDDUS"): cic = CIC::CIC_NUS_DDUS; cic_known = true; break; - default: printf("Unknown tag: %s\n", tag); + case fnv_hash("cic6101"): cic = CIC::CIC_NUS_6101; break; + case fnv_hash("cic6102"): cic = CIC::CIC_NUS_6102; break; + case fnv_hash("cic6103"): cic = CIC::CIC_NUS_6103; break; + case fnv_hash("cic6105"): cic = CIC::CIC_NUS_6105; break; + case fnv_hash("cic6106"): cic = CIC::CIC_NUS_6106; break; + case fnv_hash("cic7101"): cic = CIC::CIC_NUS_7101; break; + case fnv_hash("cic7102"): cic = CIC::CIC_NUS_7102; break; + case fnv_hash("cic7103"): cic = CIC::CIC_NUS_7103; break; + case fnv_hash("cic7105"): cic = CIC::CIC_NUS_7105; break; + case fnv_hash("cic7106"): cic = CIC::CIC_NUS_7106; break; + case fnv_hash("cic8303"): cic = CIC::CIC_NUS_8303; break; + case fnv_hash("cic8401"): cic = CIC::CIC_NUS_8401; break; + case fnv_hash("cic5167"): cic = CIC::CIC_NUS_5167; break; + case fnv_hash("cicDDUS"): cic = CIC::CIC_NUS_DDUS; break; + default: printf("Unknown tag: %s\n", tag); break; } } printf("System: %d, Save Type: %d, CIC: %d, CPak: %d, RPak: %d, TPak %d, RTC: %d\n", (int)system_type, (int)save_type, (int)cic, cpak, rpak, tpak, rtc); @@ -166,8 +167,8 @@ static bool parse_and_apply_db_tags(char* tags) { if (auto_detect == AutoDetect::ON) { printf("Auto-detect is on, updating OSD settings\n"); - if (system_type_known) user_io_status_set("[80:79]", (uint32_t)system_type); - if (cic_known) user_io_status_set("[68:65]", (uint32_t)cic); + if (system_type != SystemType::UNKNOWN) user_io_status_set("[80:79]", (uint32_t)system_type); + if (cic != CIC::UNKNOWN) user_io_status_set("[68:65]", (uint32_t)cic); user_io_status_set("[71]", (uint32_t)cpak); user_io_status_set("[72]", (uint32_t)rpak); user_io_status_set("[73]", (uint32_t)tpak); @@ -178,21 +179,30 @@ static bool parse_and_apply_db_tags(char* tags) { printf("Auto-detect is off, not updating OSD settings\n"); } - return (auto_detect != AutoDetect::ON) || (system_type_known && cic_known); + return (auto_detect != AutoDetect::ON) || (system_type != SystemType::UNKNOWN && cic != CIC::UNKNOWN); } -bool id_matches(const char* line, const char* cart_id) { +bool md5_matches(const char* line, const char* md5) { + for (auto i = 0; i < 32; i++) { + if (line[i] == '\0' || md5[i] != tolower(line[i])) + return false; + } + + return true; +} + +bool cart_id_matches(const char* line, const char* cart_id) { // A valid ID line should start with "ID:" - if (strlen(line) < 9 || strncmp(line, "ID:", 3) != 0) + if (strncmp(line, CARTID_PREFIX, strlen(CARTID_PREFIX)) != 0) return false; // Skip the line if it doesn't match our cart_id, '_' = don't care - // Cart IDs should always be 6 characters long - for (size_t i = 0; i < 6; i++) { - if (line[i + 3] != '_' && line[i + 3] != cart_id[i]) + auto lp = (char*)line + strlen(CARTID_PREFIX); + for (auto i = 0; i < CARTID_LENGTH; i++, lp++) { + if (*lp != '_' && *lp != cart_id[i]) return false; // character didn't match - if (line[i + 3] == ' ') // early end of ID + if (*lp == ' ') // early termination return true; } @@ -214,11 +224,8 @@ static uint8_t detect_rom_settings_in_db(const char* lookup_hash, const char* db const char* line; while ((line = FileReadLine(&reader))) { - if (strlen(line) < 32 || !((line[0] >= '0' && line[0] <= '9') || (line[0] >= 'a' && line[0] <= 'f'))) - continue; - - // skip the line if it doesn't start with our hash - if (strncmp(lookup_hash, line, 32) != 0) + // Skip the line if it doesn't start with our hash + if (!md5_matches(line, lookup_hash)) continue; printf("Found ROM entry: %s\n", line); @@ -250,8 +257,8 @@ static uint8_t detect_rom_settings_in_db_with_cartid(const char* cart_id, const const char* line; while ((line = FileReadLine(&reader))) { - // skip the line if it doesn't start with our id - if (!id_matches(line, cart_id)) + // Skip the line if it doesn't start with our ID + if (!cart_id_matches(line, cart_id)) continue; printf("Found ROM entry: %s\n", line); @@ -274,7 +281,7 @@ static const char* DB_FILE_NAMES[] = "N64-database_user.txt", }; -static uint8_t detect_rom_settings_in_dbs(const char* lookup) { +static uint8_t detect_rom_settings_in_dbs_with_md5(const char* lookup) { for (const char* db_file_name : DB_FILE_NAMES) { const auto detected = detect_rom_settings_in_db(lookup, db_file_name); if (detected != 0) return detected; @@ -283,12 +290,22 @@ static uint8_t detect_rom_settings_in_dbs(const char* lookup) { return 0; } -static uint8_t detect_rom_settings_in_dbs_with_cartid(const char* id) { - for (const char* db_file_name : DB_FILE_NAMES) { - const auto detected = detect_rom_settings_in_db_with_cartid(id, db_file_name); - if (detected != 0) return detected; +static uint8_t detect_rom_settings_in_dbs_with_cartid(const char* lookup) { + if (strlen(lookup) < CARTID_LENGTH) + return 0; + + for (auto i = 0; i < CARTID_LENGTH; i++) { + if ((lookup[i] >= '0' && lookup[i] <= '9') || (lookup[i] >= 'A' && lookup[i] <= 'Z')) + continue; + + return 0; } + for (const char* db_file_name : DB_FILE_NAMES) { + const auto detected = detect_rom_settings_in_db_with_cartid(lookup, db_file_name); + if (detected != 0) return detected; + } + return 0; } @@ -320,28 +337,35 @@ static bool detect_rom_settings_from_first_chunk(char region_code, uint64_t crc) // the following checks assume we're on a little-endian platform switch (crc) { default: + printf("Unknown CIC (0x%016llx), uses default\n", crc); is_known_cic = false; // fall through case UINT64_C(0x000000a316adc55a): + case UINT64_C(0x000000a30dacd530): // NOP:ed out CRC check case UINT64_C(0x000000039c981107): // hcs64's CIC-6102 IPL3 replacement - case UINT64_C(0x000000a30dacd530): // Unknown. Used in SM64 hacks case UINT64_C(0x000000d2828281b0): // Unknown. Used in some homebrew case UINT64_C(0x0000009acc31e644): // Unknown. Used in some betas and homebrew. Dev boot code? - cic = system_type == SystemType::NTSC + cic = system_type == SystemType::NTSC ? CIC::CIC_NUS_6102 : CIC::CIC_NUS_7101; break; - case UINT64_C(0x000000a405397b05): cic = CIC::CIC_NUS_7102; system_type = SystemType::PAL; break; - case UINT64_C(0x000000a0f26f62fe): cic = CIC::CIC_NUS_6101; system_type = SystemType::NTSC; break; + case UINT64_C(0x000000a405397b05): + case UINT64_C(0x000000a3fc388adb): // NOP:ed out CRC check + system_type = SystemType::PAL; cic = CIC::CIC_NUS_7102; break; + case UINT64_C(0x000000a0f26f62fe): + case UINT64_C(0x000000a0e96e72d4): // NOP:ed out CRC check + system_type = SystemType::NTSC; cic = CIC::CIC_NUS_6101; break; case UINT64_C(0x000000a9229d7c45): - cic = system_type == SystemType::NTSC + case UINT64_C(0x000000a9199c8c1b): // NOP:ed out CRC check + cic = system_type == SystemType::NTSC ? CIC::CIC_NUS_6103 : CIC::CIC_NUS_7103; break; case UINT64_C(0x000000f8b860ed00): - cic = system_type == SystemType::NTSC + case UINT64_C(0x000000f8af5ffcd6): // NOP:ed out CRC check + cic = system_type == SystemType::NTSC ? CIC::CIC_NUS_6105 : CIC::CIC_NUS_7105; break; case UINT64_C(0x000000ba5ba4b8cd): - cic = system_type == SystemType::NTSC + cic = system_type == SystemType::NTSC ? CIC::CIC_NUS_6106 : CIC::CIC_NUS_7106; break; case UINT64_C(0x0000012daafc8aab): cic = CIC::CIC_NUS_5167; break; @@ -360,12 +384,10 @@ static bool detect_rom_settings_from_first_chunk(char region_code, uint64_t crc) } static void md5_to_hex(uint8_t* in, char* out) { - char *p = out; - for (int i = 0; i < 16; i++) { - sprintf(p, "%02x", in[i]); - p += 2; + for (auto i = 0; i < 16; i++) { + sprintf(out, "%02x", in[i]); + out += 2; } - *p = '\0'; } int n64_rom_tx(const char* name, unsigned char index) { @@ -398,13 +420,13 @@ int n64_rom_tx(const char* name, unsigned char index) { // 2 = Found some ROM info in DB (Save type etc.), but System region and CIC has not been determined // 3 = Has detected everything, System type, CIC, Save type etc. uint8_t rom_settings_detected = 0; - RomFormat rom_format = RomFormat::UNKNOWN; + RomFormat rom_format; MD5Context ctx; MD5Init(&ctx); uint8_t md5[16]; char md5_hex[40]; - uint64_t ipl3_crc = 0; + uint64_t bootcode_sum = 0; char cart_id[8]; while (bytes2send) @@ -419,7 +441,10 @@ int n64_rom_tx(const char* name, unsigned char index) { printf("Failed to load ROM: must be at least 4096 bytes\n"); return 0; } + rom_format = detectRomFormat(buf); + if (rom_format == RomFormat::UNKNOWN) + printf("Unknown ROM format\n"); } // normalize data to big-endian format @@ -428,7 +453,7 @@ int n64_rom_tx(const char* name, unsigned char index) { MD5Update(&ctx, buf, chunk); if (is_first_chunk) { - // try to detect ROM settings based on header MD5 hash + // Try to detect ROM settings based on header MD5 hash // For calculating the MD5 hash of the header, we need to make a // copy of the context before calling MD5Final, otherwise the file @@ -439,16 +464,28 @@ int n64_rom_tx(const char* name, unsigned char index) { md5_to_hex(md5, md5_hex); printf("Header MD5: %s\n", md5_hex); - rom_settings_detected = detect_rom_settings_in_dbs(md5_hex); - if (rom_settings_detected == 0) { - // ROM settings wasn't found in DB, System region and/or CIC wasn't detected + rom_settings_detected = detect_rom_settings_in_dbs_with_md5(md5_hex); + if (rom_settings_detected == 0) printf("No ROM information found for header hash: %s\n", md5_hex); - for (size_t i = 0x40 / sizeof(uint32_t); i < 0x1000 / sizeof(uint32_t); i++) ipl3_crc += ((uint32_t*)buf)[i]; - strncpy(cart_id, (char*)(buf + 0x3b), 4); - sprintf((char*)(cart_id + 4), "%02X", buf[0x3f]); - cart_id[6] = '\0'; - printf("Cartridge ID: %s\n", cart_id); + + // Calculate boot ROM checksum + for (uint32_t i = 0x40 / sizeof(uint32_t); i < 0x1000 / sizeof(uint32_t); i++) { + bootcode_sum += ((uint32_t*)buf)[i]; } + + /* The first byte (starting at 0x3b) indicates the type of ROM + * 'N' = cart + * 'D' = 64DD disk + * 'C' = cartridge part of expandable game + * 'E' = 64DD expansion for cart + * 'Z' = Aleck64 cart + * The second and third byte form a 2-letter ID for the game + * The fourth byte indicates the region and language for the game + * The fifth byte indicates the revision of the game */ + + strncpy(cart_id, (char*)(buf + 0x3b), 4); + sprintf((char*)(cart_id + 4), "%02X", buf[0x3f]); + printf("Cartridge ID: %s\n", cart_id); } user_io_file_tx_data(buf, chunk); @@ -462,30 +499,37 @@ int n64_rom_tx(const char* name, unsigned char index) { md5_to_hex(md5, md5_hex); printf("File MD5: %s\n", md5_hex); - // Try to detect ROM settings from full file MD5 if they are not detected yet + // Try to detect ROM settings from full file MD5 if they're are not detected yet if (rom_settings_detected == 0) - rom_settings_detected = detect_rom_settings_in_dbs(md5_hex); + rom_settings_detected = detect_rom_settings_in_dbs_with_md5(md5_hex); + // Try to detect ROM settings by cart ID if they're are not detected yet if (rom_settings_detected == 0) { + printf("No ROM information found for file hash: %s\n", md5_hex); rom_settings_detected = detect_rom_settings_in_dbs_with_cartid(cart_id); - // Try detect (partial) ROM settings by analyzing the ROM itself. (region, cic and save type) - if ((rom_settings_detected == 0 || rom_settings_detected == 2) && detect_rom_settings_from_first_chunk(cart_id[3], ipl3_crc)) - rom_settings_detected += 1; + if (rom_settings_detected == 0) + printf("No ROM information found for cart ID: %s\n", cart_id); + // Try detect (partial) ROM settings by analyzing the ROM itself. (System region and CIC) + if ((rom_settings_detected == 0 || rom_settings_detected == 2) && + detect_rom_settings_from_first_chunk(cart_id[3], bootcode_sum)) { + rom_settings_detected |= 1; + } } - else if (rom_settings_detected == 2 && detect_rom_settings_from_first_chunk(cart_id[3], ipl3_crc)) { - // Complement info found in DB with System region and CIC + // Complement info found in DB with System region and CIC + else if (rom_settings_detected == 2 && + detect_rom_settings_from_first_chunk(cart_id[3], bootcode_sum)) { rom_settings_detected = 3; } printf("Done.\n"); FileClose(&f); - // mount save state + // Mount save state char file_path[1024]; FileGenerateSavePath(name, file_path); user_io_file_mount(file_path, 0, 1); - // signal end of transmission + // Signal end of transmission user_io_set_download(0); ProgressMessage(0, 0, 0, 0);