Function decodeOpcode

src/e9patch/e9x86_64.cpp:103–204

* Decode an instruction opcode. */

 * Decode an instruction opcode.
 */
static int decodeOpcode(const uint8_t *bytes, unsigned size, int i,
    Encoding &encoding, uint8_t &opcode)
{
    if (i < 0 || i >= (int)size)
        return i;

    encoding = ENCODING_SINGLE_BYTE;

    // Extended encodings:
    if (bytes[i] == 0x62)
    {
        if (size - i > 4 && (~bytes[i+1] & 0x0c) == 0x0c &&
            (bytes[i+2] & 0x04) == 0x04)
        {
            // 4-byte EVEX encoding:
            switch (bytes[i+1] & 0x03)
            {
                case 0x01:
                    encoding = ENCODING_TWO_BYTES_0F;
                    break;
                case 0x02:
                    encoding = ENCODING_THREE_BYTES_0F38;
                    break;
                case 0x03:
                    encoding = ENCODING_THREE_BYTES_0F3A;
                    break;
                default:
                    return -1;
            }
            i += 4;
            opcode = bytes[i++];
            return i;
        }
        else
            return -1;
    }
    else if (bytes[i] == 0xc4 && (i > 0? (bytes[i-1] & 0xf0) != 0x40: true))
    {
        if (size - i > 3)
        {
            // 3-byte VEX encoding:
            switch (bytes[i+1] & 0x1f)
            {
                case 0x1:
                    encoding = ENCODING_TWO_BYTES_0F;
                    break;
                case 0x02:
                    encoding = ENCODING_THREE_BYTES_0F38;
                    break;
                case 0x03:
                    encoding = ENCODING_THREE_BYTES_0F3A;
                    break;
                default:
                    return -1;
            }
            i += 3;
            opcode = bytes[i++];
            return i;