Function DetectFileKeyBySectorSize

3rdParty/StormLib/src/SBaseCommon.cpp:458–505

 */

DWORD DetectFileKeyBySectorSize(LPDWORD EncryptedData, DWORD dwSectorSize, DWORD dwDecrypted0)
{
    DWORD dwDecrypted1Max = dwSectorSize + dwDecrypted0;
    DWORD dwKey1PlusKey2;
    DWORD DataBlock[2];

    // We must have at least 2 DWORDs there to be able to decrypt something
    if(dwSectorSize < 0x08)
        return 0;

    // Get the value of the combined encryption key
    dwKey1PlusKey2 = (EncryptedData[0] ^ dwDecrypted0) - 0xEEEEEEEE;

    // Try all 256 combinations of dwKey1
    for(DWORD i = 0; i < 0x100; i++)
    {
        DWORD dwSaveKey1;
        DWORD dwKey1 = dwKey1PlusKey2 - StormBuffer[MPQ_HASH_KEY2_MIX + i];
        DWORD dwKey2 = 0xEEEEEEEE;

        // Modify the second key and decrypt the first DWORD
        dwKey2 += StormBuffer[MPQ_HASH_KEY2_MIX + (dwKey1 & 0xFF)];
        DataBlock[0] = EncryptedData[0] ^ (dwKey1 + dwKey2);

        // Did we obtain the same value like dwDecrypted0?
        if(DataBlock[0] == dwDecrypted0)
        {
            // Save this key value. Increment by one because
            // we are decrypting sector offset table
            dwSaveKey1 = dwKey1 + 1;

            // Rotate both keys
            dwKey1 = ((~dwKey1 << 0x15) + 0x11111111) | (dwKey1 >> 0x0B);
            dwKey2 = DataBlock[0] + dwKey2 + (dwKey2 << 5) + 3;

            // Modify the second key again and decrypt the second DWORD
            dwKey2 += StormBuffer[MPQ_HASH_KEY2_MIX + (dwKey1 & 0xFF)];
            DataBlock[1] = EncryptedData[1] ^ (dwKey1 + dwKey2);

            // Now compare the results
            if(DataBlock[1] <= dwDecrypted1Max)
                return dwSaveKey1;
        }
    }

    // Key not found
    return 0;
}

// Function tries to detect file encryption key based on expected file content
// It is the same function like before, except that we know the value of the second DWORD