vmprofiler/src/vminstrs.cpp

#include <vmprofiler.hpp>

namespace vm
{
    namespace instrs
    {
        std::pair< std::uint64_t, std::uint64_t > decrypt_operand( transform::map_t &transforms, std::uint64_t operand,
                                                                   std::uint64_t rolling_key )
        {
            const auto& generic_decrypt_0 = transforms[ transform::type::generic0 ];
            const auto& key_decrypt = transforms[ transform::type::rolling_key ];
            const auto& generic_decrypt_1 = transforms[ transform::type::generic1 ];
            const auto& generic_decrypt_2 = transforms[ transform::type::generic2 ];
            const auto& generic_decrypt_3 = transforms[ transform::type::generic3 ];
            const auto& update_key = transforms[ transform::type::update_key ];

            if ( generic_decrypt_0.mnemonic != ZYDIS_MNEMONIC_INVALID )
            {
                operand = transform::apply(
                    generic_decrypt_0.operands[ 0 ].size, generic_decrypt_0.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_0 ) ? generic_decrypt_0.operands[ 1 ].imm.value.u : 0 );
            }

            // apply transformation with rolling decrypt key...
            operand = transform::apply( key_decrypt.operands[ 0 ].size, key_decrypt.mnemonic, operand, rolling_key );

            // apply three generic transformations...
            {
                operand = transform::apply(
                    generic_decrypt_1.operands[ 0 ].size, generic_decrypt_1.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_1 ) ? generic_decrypt_1.operands[ 1 ].imm.value.u : 0 );

                operand = transform::apply(
                    generic_decrypt_2.operands[ 0 ].size, generic_decrypt_2.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_2 ) ? generic_decrypt_2.operands[ 1 ].imm.value.u : 0 );

                operand = transform::apply(
                    generic_decrypt_3.operands[ 0 ].size, generic_decrypt_3.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_3 ) ? generic_decrypt_3.operands[ 1 ].imm.value.u : 0 );
            }

            // update rolling key...
            auto result =
                transform::apply( update_key.operands[ 0 ].size, update_key.mnemonic, rolling_key, operand );

            // update decryption key correctly...
            switch ( update_key.operands[ 0 ].size )
            {
            case 8:
                rolling_key = ( rolling_key & ~0xFFull ) + result;
                break;
            case 16:
                rolling_key = ( rolling_key & ~0xFFFFull ) + result;
                break;
            default:
                rolling_key = result;
                break;
            }

            return { operand, rolling_key };
        }

        std::pair< std::uint64_t, std::uint64_t > encrypt_operand( transform::map_t &transforms, std::uint64_t operand,
                                                                   std::uint64_t rolling_key )
        {
            transform::map_t inverse;
            inverse_transforms( transforms, inverse );

            const auto& generic_decrypt_0 = inverse[ transform::type::generic0 ];
            const auto& key_decrypt = inverse[ transform::type::rolling_key ];
            const auto& generic_decrypt_1 = inverse[ transform::type::generic1 ];
            const auto& generic_decrypt_2 = inverse[ transform::type::generic2 ];
            const auto& generic_decrypt_3 = inverse[ transform::type::generic3 ];
            const auto& update_key = inverse[ transform::type::update_key ];

            auto result =
                transform::apply( update_key.operands[ 0 ].size, update_key.mnemonic, rolling_key, operand );

            // make sure we update the rolling decryption key correctly...
            switch ( update_key.operands[ 0 ].size )
            {
            case 8:
                rolling_key = ( rolling_key & ~0xFFull ) + result;
                break;
            case 16:
                rolling_key = ( rolling_key & ~0xFFFFull ) + result;
                break;
            default:
                rolling_key = result;
                break;
            }

            {
                operand = transform::apply(
                    generic_decrypt_3.operands[ 0 ].size, generic_decrypt_3.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_3 ) ? generic_decrypt_3.operands[ 1 ].imm.value.u : 0 );

                operand = transform::apply(
                    generic_decrypt_2.operands[ 0 ].size, generic_decrypt_2.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_2 ) ? generic_decrypt_2.operands[ 1 ].imm.value.u : 0 );

                operand = transform::apply(
                    generic_decrypt_1.operands[ 0 ].size, generic_decrypt_1.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_1 ) ? generic_decrypt_1.operands[ 1 ].imm.value.u : 0 );
            }

            operand = transform::apply( key_decrypt.operands[ 0 ].size, key_decrypt.mnemonic, operand, rolling_key );

            if ( generic_decrypt_0.mnemonic != ZYDIS_MNEMONIC_INVALID )
            {
                operand = transform::apply(
                    generic_decrypt_0.operands[ 0 ].size, generic_decrypt_0.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_0 ) ? generic_decrypt_0.operands[ 1 ].imm.value.u : 0 );
            }

            return { operand, rolling_key };
        }

        bool get_rva_decrypt( const zydis_routine_t &vm_entry, std::vector< zydis_decoded_instr_t > &transform_instrs )
        {
            //
            // find mov esi, [rsp+0xA0]
            //

            auto result =
                std::find_if( vm_entry.begin(), vm_entry.end(), []( const zydis_instr_t &instr_data ) -> bool {
                    if ( instr_data.instr.mnemonic == ZYDIS_MNEMONIC_MOV && instr_data.instr.operand_count == 2 &&
                         instr_data.instr.operands[ 0 ].reg.value == ZYDIS_REGISTER_ESI &&
                         instr_data.instr.operands[ 1 ].mem.base == ZYDIS_REGISTER_RSP &&
                         instr_data.instr.operands[ 1 ].mem.disp.has_displacement &&
                         instr_data.instr.operands[ 1 ].mem.disp.value == 0xA0 )
                        return true;
                    return false;
                } );

            if ( result == vm_entry.end() )
                return false;

            //
            // find the next three instruction with ESI as the dest...
            //

            for ( auto idx = 0u; idx < 3; ++idx )
            {
                result = std::find_if( ++result, vm_entry.end(), []( const zydis_instr_t &instr_data ) -> bool {
                    return instr_data.instr.operands[ 0 ].reg.value == ZYDIS_REGISTER_ESI;
                } );

                if ( result == vm_entry.end() )
                    return false;

                transform_instrs.push_back( result->instr );
            }

            return true;
        }
    } // namespace instrs
} // namespace vm