N64: Find ROM settings in database by ID, region and revision (#830)

ID, region and revision are grabbed from the games' header and are looked up in the database text file, the same file where MD5 hashes are now. Both MD5 and ID will be valid ways of determining ROM settings, so old database files will still work. The benefit of this is that the database can be shrunk quite considerably, as many, many duplicates can be removed. It will be (much) easier to maintain. I'm hoping to see fewer pull request made to the database repository as a consequence of this change.
This commit is contained in:
Rikard Bengtsson
2023-10-01 13:56:12 +02:00
committed by GitHub
parent d53a998eac
commit ee5659ad9c

View File

@@ -14,14 +14,12 @@
static constexpr uint64_t FNV_PRIME = 0x100000001b3;
static constexpr uint64_t FNV_OFFSET_BASIS = 0xcbf29ce484222325;
static constexpr uint64_t fnv_hash(const char *s, uint64_t h = FNV_OFFSET_BASIS)
{
static constexpr uint64_t fnv_hash(const char *s, uint64_t h = FNV_OFFSET_BASIS) {
if (s) while (uint8_t a = *(s++)) h = (h ^ a) * FNV_PRIME;
return h;
}
enum class MemoryType
{
enum class MemoryType {
NONE = 0,
EEPROM_512,
EEPROM_2k,
@@ -30,8 +28,7 @@ enum class MemoryType
FLASH_128k
};
enum class CIC
{
enum class CIC {
CIC_NUS_6101 = 0,
CIC_NUS_6102,
CIC_NUS_7101,
@@ -48,22 +45,19 @@ enum class CIC
CIC_NUS_DDUS
};
enum class SystemType
{
enum class SystemType {
NTSC = 0,
PAL
};
enum class RomFormat
{
enum class RomFormat {
UNKNOWN = 0,
BIG_ENDIAN,
BYTE_SWAPPED,
LITTLE_ENDIAN,
};
enum class AutoDetect
{
enum class AutoDetect {
ON = 0,
OFF = 1,
};
@@ -84,54 +78,134 @@ static RomFormat detectRomFormat(const uint8_t* data)
static void normalizeData(uint8_t* data, size_t size, RomFormat format)
{
switch(format)
{
switch(format) {
case RomFormat::BYTE_SWAPPED:
for (size_t i = 0; i < size; i += 2)
{
auto c0 = data[0];
auto c1 = data[1];
data[0] = c1;
data[1] = c0;
data += 2;
}
break;
for (size_t i = 0; i < size; i += 2) {
auto c0 = data[0];
auto c1 = data[1];
data[0] = c1;
data[1] = c0;
data += 2;
}
break;
case RomFormat::LITTLE_ENDIAN:
for (size_t i = 0; i < size; i += 4)
{
auto c0 = data[0];
auto c1 = data[1];
auto c2 = data[2];
auto c3 = data[3];
data[0] = c3;
data[1] = c2;
data[2] = c1;
data[3] = c0;
data += 4;
}
break;
for (size_t i = 0; i < size; i += 4) {
auto c0 = data[0];
auto c1 = data[1];
auto c2 = data[2];
auto c3 = data[3];
data[0] = c3;
data[1] = c2;
data[2] = c1;
data[3] = c0;
data += 4;
}
break;
default:
{
// nothing to do
}
break;
}
}
static void normalizeString(char* s)
{
static void normalizeString(char* s) {
// change the string to lower-case
while (*s) { *s = tolower(*s); ++s; }
}
static bool detect_rom_settings_in_db(const char* lookup_hash, const char* db_file_name)
{
// return true if cic and system region is detected
static bool parse_and_apply_db_tags(char* tags) {
MemoryType save_type = MemoryType::NONE;
SystemType system_type = SystemType::NTSC;
CIC cic = CIC::CIC_NUS_6102;
bool cpak = false;
bool rpak = false;
bool tpak = false;
bool rtc = false;
bool system_type_known = false;
bool cic_known = false;
const char separator[] = "|";
for (char* tag = strtok(tags, separator); tag; tag = strtok(nullptr, separator)) {
printf("Tag: %s\n", tag);
normalizeString(tag);
switch (fnv_hash(tag)) {
case fnv_hash("eeprom512"): save_type = MemoryType::EEPROM_512; break;
case fnv_hash("eeprom2k"): save_type = MemoryType::EEPROM_2k; break;
case fnv_hash("sram32k"): save_type = MemoryType::SRAM_32k; break;
case fnv_hash("sram96k"): save_type = MemoryType::SRAM_96k; break;
case fnv_hash("flash128k"): save_type = MemoryType::FLASH_128k; break;
case fnv_hash("ntsc"): system_type = SystemType::NTSC; system_type_known = true; break;
case fnv_hash("pal"): system_type = SystemType::PAL; system_type_known = true; break;
case fnv_hash("cpak"): cpak = true; break;
case fnv_hash("rpak"): rpak = true; break;
case fnv_hash("tpak"): tpak = true; break;
case fnv_hash("rtc"): rtc = true; break;
case fnv_hash("cic6101"): cic = CIC::CIC_NUS_6101; cic_known = true; break;
case fnv_hash("cic6102"): cic = CIC::CIC_NUS_6102; cic_known = true; break;
case fnv_hash("cic6103"): cic = CIC::CIC_NUS_6103; cic_known = true; break;
case fnv_hash("cic6105"): cic = CIC::CIC_NUS_6105; cic_known = true; break;
case fnv_hash("cic6106"): cic = CIC::CIC_NUS_6106; cic_known = true; break;
case fnv_hash("cic7101"): cic = CIC::CIC_NUS_7101; cic_known = true; break;
case fnv_hash("cic7102"): cic = CIC::CIC_NUS_7102; cic_known = true; break;
case fnv_hash("cic7103"): cic = CIC::CIC_NUS_7103; cic_known = true; break;
case fnv_hash("cic7105"): cic = CIC::CIC_NUS_7105; cic_known = true; break;
case fnv_hash("cic7106"): cic = CIC::CIC_NUS_7106; cic_known = true; break;
case fnv_hash("cic8303"): cic = CIC::CIC_NUS_8303; cic_known = true; break;
case fnv_hash("cic8401"): cic = CIC::CIC_NUS_8401; cic_known = true; break;
case fnv_hash("cic5167"): cic = CIC::CIC_NUS_5167; cic_known = true; break;
case fnv_hash("cicDDUS"): cic = CIC::CIC_NUS_DDUS; cic_known = true; break;
default: printf("Unknown tag: %s\n", tag);
}
}
printf("System: %d, Save Type: %d, CIC: %d, CPak: %d, RPak: %d, TPak %d, RTC: %d\n", (int)system_type, (int)save_type, (int)cic, cpak, rpak, tpak, rtc);
const auto auto_detect = (AutoDetect)user_io_status_get("[64]");
if (auto_detect == AutoDetect::ON) {
printf("Auto-detect is on, updating OSD settings\n");
if (system_type_known) user_io_status_set("[80:79]", (uint32_t)system_type);
if (cic_known) user_io_status_set("[68:65]", (uint32_t)cic);
user_io_status_set("[71]", (uint32_t)cpak);
user_io_status_set("[72]", (uint32_t)rpak);
user_io_status_set("[73]", (uint32_t)tpak);
user_io_status_set("[74]", (uint32_t)rtc);
user_io_status_set("[77:75]", (uint32_t)save_type);
}
else {
printf("Auto-detect is off, not updating OSD settings\n");
}
return (auto_detect != AutoDetect::ON) || (system_type_known && cic_known);
}
bool id_matches(const char* line, const char* cart_id) {
// A valid ID line should start with "ID:"
if (strlen(line) < 9 || strncmp(line, "ID:", 3) != 0)
return false;
// Skip the line if it doesn't match our cart_id, '_' = don't care
// Cart IDs should always be 6 characters long
for (size_t i = 0; i < 6; i++) {
if (line[i + 3] != '_' && line[i + 3] != cart_id[i])
return false; // character didn't match
if (line[i + 3] == ' ') // early end of ID
return true;
}
return true;
}
static uint8_t detect_rom_settings_in_db(const char* lookup_hash, const char* db_file_name) {
fileTextReader reader = {};
char file_path[1024];
sprintf(file_path, "%s/%s", HomeDir(), db_file_name);
if (!FileOpenTextReader(&reader, file_path))
{
if (!FileOpenTextReader(&reader, file_path)) {
printf("Failed to open N64 data file %s\n", file_path);
return false;
}
@@ -139,90 +213,59 @@ static bool detect_rom_settings_in_db(const char* lookup_hash, const char* db_fi
char tags[128];
const char* line;
while ((line = FileReadLine(&reader)))
{
while ((line = FileReadLine(&reader))) {
if (strlen(line) < 32 || !((line[0] >= '0' && line[0] <= '9') || (line[0] >= 'a' && line[0] <= 'f')))
continue;
// skip the line if it doesn't start with our hash
if (strncmp(lookup_hash, line, 32) != 0)
continue;
if (sscanf(line, "%*s %s", tags) != 1)
{
printf("Found ROM entry: %s\n", line);
if (sscanf(line, "%*s %s", tags) != 1) {
printf("No tags found.\n");
continue;
return 2;
}
// 2 = System region and/or CIC wasn't in DB, will need detection
return parse_and_apply_db_tags(tags) ? 3 : 2;
}
return 0;
}
static uint8_t detect_rom_settings_in_db_with_cartid(const char* cart_id, const char* db_file_name) {
fileTextReader reader = {};
char file_path[1024];
sprintf(file_path, "%s/%s", HomeDir(), db_file_name);
if (!FileOpenTextReader(&reader, file_path)) {
printf("Failed to open N64 data file %s\n", file_path);
return false;
}
char tags[128];
const char* line;
while ((line = FileReadLine(&reader))) {
// skip the line if it doesn't start with our id
if (!id_matches(line, cart_id))
continue;
printf("Found ROM entry: %s\n", line);
MemoryType save_type = MemoryType::NONE;
SystemType system_type = SystemType::NTSC;
CIC cic = CIC::CIC_NUS_6102;
bool cpak = false;
bool rpak = false;
bool tpak = false;
bool rtc = false;
const char separator[] = "|";
for (char* tag = strtok(tags, separator); tag; tag = strtok(nullptr, separator))
{
printf("Tag: %s\n", tag);
normalizeString(tag);
switch (fnv_hash(tag))
{
case fnv_hash("eeprom512"): save_type = MemoryType::EEPROM_512; break;
case fnv_hash("eeprom2k"): save_type = MemoryType::EEPROM_2k; break;
case fnv_hash("sram32k"): save_type = MemoryType::SRAM_32k; break;
case fnv_hash("sram96k"): save_type = MemoryType::SRAM_96k; break;
case fnv_hash("flash128k"): save_type = MemoryType::FLASH_128k; break;
case fnv_hash("ntsc"): system_type = SystemType::NTSC; break;
case fnv_hash("pal"): system_type = SystemType::PAL; break;
case fnv_hash("cpak"): cpak = true; break;
case fnv_hash("rpak"): rpak = true; break;
case fnv_hash("tpak"): tpak = true; break;
case fnv_hash("rtc"): rtc = true; break;
case fnv_hash("cic6101"): cic = CIC::CIC_NUS_6101; break;
case fnv_hash("cic6102"): cic = CIC::CIC_NUS_6102; break;
case fnv_hash("cic6103"): cic = CIC::CIC_NUS_6103; break;
case fnv_hash("cic6105"): cic = CIC::CIC_NUS_6105; break;
case fnv_hash("cic6106"): cic = CIC::CIC_NUS_6106; break;
case fnv_hash("cic7101"): cic = CIC::CIC_NUS_7101; break;
case fnv_hash("cic7102"): cic = CIC::CIC_NUS_7102; break;
case fnv_hash("cic7103"): cic = CIC::CIC_NUS_7103; break;
case fnv_hash("cic7105"): cic = CIC::CIC_NUS_7105; break;
case fnv_hash("cic7106"): cic = CIC::CIC_NUS_7106; break;
case fnv_hash("cic8303"): cic = CIC::CIC_NUS_8303; break;
case fnv_hash("cic8401"): cic = CIC::CIC_NUS_8401; break;
case fnv_hash("cic5167"): cic = CIC::CIC_NUS_5167; break;
case fnv_hash("cicDDUS"): cic = CIC::CIC_NUS_DDUS; break;
default: printf("Unknown tag: %s\n", tag);
}
}
printf("System: %d, Save Type: %d, CIC: %d, CPak: %d, RPak: %d, TPak %d, RTC: %d\n", (int)system_type, (int)save_type, (int)cic, cpak, rpak, tpak, rtc);
const auto auto_detect = (AutoDetect)user_io_status_get("[64]");
if (auto_detect == AutoDetect::ON)
{
printf("Auto-detect is on, updating OSD settings\n");
user_io_status_set("[80:79]", (uint32_t)system_type);
user_io_status_set("[68:65]", (uint32_t)cic);
user_io_status_set("[71]", (uint32_t)cpak);
user_io_status_set("[72]", (uint32_t)rpak);
user_io_status_set("[73]", (uint32_t)tpak);
user_io_status_set("[74]", (uint32_t)rtc);
user_io_status_set("[77:75]", (uint32_t)save_type);
}
else
{
printf("Auto-detect is off, not updating OSD settings\n");
if (sscanf(line, "%*s %s", tags) != 1) {
printf("No tags found.\n");
return 2;
}
return true;
// 2 = System region and/or CIC wasn't in DB, will need detection
return parse_and_apply_db_tags(tags) ? 3 : 2;
}
return false;
return 0;
}
static const char* DB_FILE_NAMES[] =
@@ -231,50 +274,54 @@ static const char* DB_FILE_NAMES[] =
"N64-database_user.txt",
};
static bool detect_rom_settings_in_dbs(const char* lookup_hash)
{
for (const char* db_file_name: DB_FILE_NAMES)
{
if (detect_rom_settings_in_db(lookup_hash, db_file_name))
return true;
static uint8_t detect_rom_settings_in_dbs(const char* lookup) {
for (const char* db_file_name : DB_FILE_NAMES) {
const auto detected = detect_rom_settings_in_db(lookup, db_file_name);
if (detected != 0) return detected;
}
return false;
return 0;
}
static bool detect_rom_settings_from_first_chunk(char region_code, uint64_t crc)
{
SystemType system_type;
static uint8_t detect_rom_settings_in_dbs_with_cartid(const char* id) {
for (const char* db_file_name : DB_FILE_NAMES) {
const auto detected = detect_rom_settings_in_db_with_cartid(id, db_file_name);
if (detected != 0) return detected;
}
return 0;
}
static bool detect_rom_settings_from_first_chunk(char region_code, uint64_t crc) {
SystemType system_type = SystemType::NTSC;
CIC cic;
bool is_known_cic = true;
const auto auto_detect = (AutoDetect)user_io_status_get("[64]");
if (auto_detect != AutoDetect::ON)
{
if ((AutoDetect)user_io_status_get("[64]") != AutoDetect::ON) {
printf("Auto-detect is off, not updating OSD settings\n");
return true;
}
switch (region_code)
{
case 'D': //Germany
case 'F': //France
case 'H': //Netherlands (Dutch)
case 'I': //Italy
case 'L': //Gateway 64
case 'P': //Europe
case 'S': //Spain
case 'U': //Australia
case 'W': //Scandinavia
case 'X': //Europe
case 'Y': //Europe
switch (region_code) {
case 'D': // Germany
case 'F': // France
case 'H': // Netherlands (Dutch)
case 'I': // Italy
case 'L': // Gateway 64
case 'P': // Europe
case 'S': // Spain
case 'U': // Australia
case 'W': // Scandinavia
case 'X': // Europe
case 'Y': // Europe
system_type = SystemType::PAL; break;
default:
system_type = SystemType::NTSC; break;
}
// the following checks assume we're on a little-endian platform
switch (crc)
{
switch (crc) {
default:
is_known_cic = false;
// fall through
case UINT64_C(0x000000a316adc55a):
case UINT64_C(0x000000039c981107): // hcs64's CIC-6102 IPL3 replacement
case UINT64_C(0x000000a30dacd530): // Unknown. Used in SM64 hacks
@@ -301,7 +348,6 @@ static bool detect_rom_settings_from_first_chunk(char region_code, uint64_t crc)
case UINT64_C(0x000000a9df4b39e1): cic = CIC::CIC_NUS_8303; break;
case UINT64_C(0x000000aa764e39e1): cic = CIC::CIC_NUS_8401; break;
case UINT64_C(0x000000abb0b739e1): cic = CIC::CIC_NUS_DDUS; break;
default: return false;
}
printf("System: %d, CIC: %d\n", (int)system_type, (int)cic);
@@ -310,22 +356,19 @@ static bool detect_rom_settings_from_first_chunk(char region_code, uint64_t crc)
user_io_status_set("[80:79]", (uint32_t)system_type);
user_io_status_set("[68:65]", (uint32_t)cic);
return true;
return is_known_cic;
}
static void md5_to_hex(uint8_t* in, char* out)
{
static void md5_to_hex(uint8_t* in, char* out) {
char *p = out;
for (int i = 0; i < 16; i++)
{
for (int i = 0; i < 16; i++) {
sprintf(p, "%02x", in[i]);
p += 2;
}
*p = '\0';
}
int n64_rom_tx(const char* name, unsigned char index)
{
int n64_rom_tx(const char* name, unsigned char index) {
static uint8_t buf[4096];
fileTYPE f = {};
@@ -349,8 +392,12 @@ int n64_rom_tx(const char* name, unsigned char index)
process_ss(name);
bool is_first_chunk = true;
bool rom_found_in_db = false;
bool cic_detected = false;
// 0 = Nothing detected
// 1 = System region and CIC detected
// 2 = Found some ROM info in DB (Save type etc.), but System region and CIC has not been determined
// 3 = Has detected everything, System type, CIC, Save type etc.
uint8_t rom_settings_detected = 0;
RomFormat rom_format = RomFormat::UNKNOWN;
MD5Context ctx;
@@ -358,7 +405,7 @@ int n64_rom_tx(const char* name, unsigned char index)
uint8_t md5[16];
char md5_hex[40];
uint64_t ipl3_crc = 0;
char region_code = '\0';
char cart_id[8];
while (bytes2send)
{
@@ -367,10 +414,8 @@ int n64_rom_tx(const char* name, unsigned char index)
FileReadAdv(&f, buf, chunk);
// perform sanity checks and detect ROM format
if (is_first_chunk)
{
if (chunk < 4096)
{
if (is_first_chunk) {
if (chunk < 4096) {
printf("Failed to load ROM: must be at least 4096 bytes\n");
return 0;
}
@@ -382,8 +427,7 @@ int n64_rom_tx(const char* name, unsigned char index)
MD5Update(&ctx, buf, chunk);
if (is_first_chunk)
{
if (is_first_chunk) {
// try to detect ROM settings based on header MD5 hash
// For calculating the MD5 hash of the header, we need to make a
@@ -395,12 +439,15 @@ int n64_rom_tx(const char* name, unsigned char index)
md5_to_hex(md5, md5_hex);
printf("Header MD5: %s\n", md5_hex);
rom_found_in_db = detect_rom_settings_in_dbs(md5_hex);
if (!rom_found_in_db)
{
rom_settings_detected = detect_rom_settings_in_dbs(md5_hex);
if (rom_settings_detected == 0) {
// ROM settings wasn't found in DB, System region and/or CIC wasn't detected
printf("No ROM information found for header hash: %s\n", md5_hex);
for (size_t i = 0x40 / sizeof(uint32_t); i < 0x1000 / sizeof(uint32_t); i++) ipl3_crc += ((uint32_t*)buf)[i];
region_code = buf[0x3e];
strncpy(cart_id, (char*)(buf + 0x3b), 4);
sprintf((char*)(cart_id + 4), "%02X", buf[0x3f]);
cart_id[6] = '\0';
printf("Cartridge ID: %s\n", cart_id);
}
}
@@ -415,19 +462,19 @@ int n64_rom_tx(const char* name, unsigned char index)
md5_to_hex(md5, md5_hex);
printf("File MD5: %s\n", md5_hex);
// Try to detect ROM settings from file MD5 if they are not available yet
if (!rom_found_in_db)
{
rom_found_in_db = detect_rom_settings_in_dbs(md5_hex);
if (!rom_found_in_db) printf("No ROM information found for file hash: %s\n", md5_hex);
}
// Try to detect ROM settings from full file MD5 if they are not detected yet
if (rom_settings_detected == 0)
rom_settings_detected = detect_rom_settings_in_dbs(md5_hex);
// Try detect (partial) ROM settings by analyzing the ROM itself. (region, cic and save type)
// Fallback for missing db entries.
if (!rom_found_in_db)
{
cic_detected = detect_rom_settings_from_first_chunk(region_code, ipl3_crc);
if (!cic_detected) printf("Unknown CIC type: %016" PRIX64 "\n", ipl3_crc);
if (rom_settings_detected == 0) {
rom_settings_detected = detect_rom_settings_in_dbs_with_cartid(cart_id);
// Try detect (partial) ROM settings by analyzing the ROM itself. (region, cic and save type)
if ((rom_settings_detected == 0 || rom_settings_detected == 2) && detect_rom_settings_from_first_chunk(cart_id[3], ipl3_crc))
rom_settings_detected += 1;
}
else if (rom_settings_detected == 2 && detect_rom_settings_from_first_chunk(cart_id[3], ipl3_crc)) {
// Complement info found in DB with System region and CIC
rom_settings_detected = 3;
}
printf("Done.\n");
@@ -442,11 +489,10 @@ int n64_rom_tx(const char* name, unsigned char index)
user_io_set_download(0);
ProgressMessage(0, 0, 0, 0);
if (!rom_found_in_db)
{
if (!cic_detected) Info("auto-detect failed");
else Info("use database if save is needed");
}
if (rom_settings_detected == 0 || rom_settings_detected == 2)
Info("Auto-detect failed:\nUnknown CIC type.");
else if (rom_settings_detected == 1)
Info("Auto-detect failed:\nROM missing from database,\nyou might not be able to save.", 5000);
return 1;
}