vmprofiler/src/vminstrs.cpp

#include <vmprofiler.hpp>

namespace vm
{
    namespace instrs
    {
        std::pair< std::uint64_t, std::uint64_t > decrypt_operand( transform::map_t &transforms, std::uint64_t operand,
                                                                   std::uint64_t rolling_key )
        {
            const auto &generic_decrypt_0 = transforms[ transform::type::generic0 ];
            const auto &key_decrypt = transforms[ transform::type::rolling_key ];
            const auto &generic_decrypt_1 = transforms[ transform::type::generic1 ];
            const auto &generic_decrypt_2 = transforms[ transform::type::generic2 ];
            const auto &generic_decrypt_3 = transforms[ transform::type::generic3 ];
            const auto &update_key = transforms[ transform::type::update_key ];

            if ( generic_decrypt_0.mnemonic != ZYDIS_MNEMONIC_INVALID )
            {
                operand = transform::apply(
                    generic_decrypt_0.operands[ 0 ].size, generic_decrypt_0.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_0 ) ? generic_decrypt_0.operands[ 1 ].imm.value.u : 0 );
            }

            // apply transformation with rolling decrypt key...
            operand = transform::apply( key_decrypt.operands[ 0 ].size, key_decrypt.mnemonic, operand, rolling_key );

            // apply three generic transformations...
            {
                operand = transform::apply(
                    generic_decrypt_1.operands[ 0 ].size, generic_decrypt_1.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_1 ) ? generic_decrypt_1.operands[ 1 ].imm.value.u : 0 );

                operand = transform::apply(
                    generic_decrypt_2.operands[ 0 ].size, generic_decrypt_2.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_2 ) ? generic_decrypt_2.operands[ 1 ].imm.value.u : 0 );

                operand = transform::apply(
                    generic_decrypt_3.operands[ 0 ].size, generic_decrypt_3.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_3 ) ? generic_decrypt_3.operands[ 1 ].imm.value.u : 0 );
            }

            // update rolling key...
            auto result = transform::apply( update_key.operands[ 0 ].size, update_key.mnemonic, rolling_key, operand );

            // update decryption key correctly...
            switch ( update_key.operands[ 0 ].size )
            {
            case 8:
                rolling_key = ( rolling_key & ~0xFFull ) + result;
                break;
            case 16:
                rolling_key = ( rolling_key & ~0xFFFFull ) + result;
                break;
            default:
                rolling_key = result;
                break;
            }

            return { operand, rolling_key };
        }

        std::pair< std::uint64_t, std::uint64_t > encrypt_operand( transform::map_t &transforms, std::uint64_t operand,
                                                                   std::uint64_t rolling_key )
        {
            transform::map_t inverse;
            inverse_transforms( transforms, inverse );

            const auto &generic_decrypt_0 = inverse[ transform::type::generic0 ];
            const auto &key_decrypt = inverse[ transform::type::rolling_key ];
            const auto &generic_decrypt_1 = inverse[ transform::type::generic1 ];
            const auto &generic_decrypt_2 = inverse[ transform::type::generic2 ];
            const auto &generic_decrypt_3 = inverse[ transform::type::generic3 ];
            const auto &update_key = inverse[ transform::type::update_key ];

            auto result = transform::apply( update_key.operands[ 0 ].size, update_key.mnemonic, rolling_key, operand );

            // make sure we update the rolling decryption key correctly...
            switch ( update_key.operands[ 0 ].size )
            {
            case 8:
                rolling_key = ( rolling_key & ~0xFFull ) + result;
                break;
            case 16:
                rolling_key = ( rolling_key & ~0xFFFFull ) + result;
                break;
            default:
                rolling_key = result;
                break;
            }

            {
                operand = transform::apply(
                    generic_decrypt_3.operands[ 0 ].size, generic_decrypt_3.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_3 ) ? generic_decrypt_3.operands[ 1 ].imm.value.u : 0 );

                operand = transform::apply(
                    generic_decrypt_2.operands[ 0 ].size, generic_decrypt_2.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_2 ) ? generic_decrypt_2.operands[ 1 ].imm.value.u : 0 );

                operand = transform::apply(
                    generic_decrypt_1.operands[ 0 ].size, generic_decrypt_1.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_1 ) ? generic_decrypt_1.operands[ 1 ].imm.value.u : 0 );
            }

            operand = transform::apply( key_decrypt.operands[ 0 ].size, key_decrypt.mnemonic, operand, rolling_key );

            if ( generic_decrypt_0.mnemonic != ZYDIS_MNEMONIC_INVALID )
            {
                operand = transform::apply(
                    generic_decrypt_0.operands[ 0 ].size, generic_decrypt_0.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_0 ) ? generic_decrypt_0.operands[ 1 ].imm.value.u : 0 );
            }

            return { operand, rolling_key };
        }

        bool get_rva_decrypt( const zydis_routine_t &vm_entry, std::vector< zydis_decoded_instr_t > &transform_instrs )
        {
            //
            // find mov esi, [rsp+0xA0]
            //

            auto result =
                std::find_if( vm_entry.begin(), vm_entry.end(), []( const zydis_instr_t &instr_data ) -> bool {
                    if ( instr_data.instr.mnemonic == ZYDIS_MNEMONIC_MOV && instr_data.instr.operand_count == 2 &&
                         instr_data.instr.operands[ 0 ].reg.value == ZYDIS_REGISTER_ESI &&
                         instr_data.instr.operands[ 1 ].mem.base == ZYDIS_REGISTER_RSP &&
                         instr_data.instr.operands[ 1 ].mem.disp.has_displacement &&
                         instr_data.instr.operands[ 1 ].mem.disp.value == 0xA0 )
                        return true;
                    return false;
                } );

            if ( result == vm_entry.end() )
                return false;

            //
            // find the next three instruction with ESI as the dest...
            //

            for ( auto idx = 0u; idx < 3; ++idx )
            {
                result = std::find_if( ++result, vm_entry.end(), []( const zydis_instr_t &instr_data ) -> bool {
                    return instr_data.instr.operands[ 0 ].reg.value == ZYDIS_REGISTER_ESI;
                } );

                if ( result == vm_entry.end() )
                    return false;

                transform_instrs.push_back( result->instr );
            }

            return true;
        }

        std::optional< std::uint64_t > get_imm( vm::ctx_t &ctx, std::uint8_t imm_size, std::uintptr_t vip )
        {
            if ( !imm_size )
                return {};

            return ctx.exec_type == vmp2::exec_type_t::forward
                       ? *reinterpret_cast< std::uintptr_t * >( vip + ( imm_size / 8 ) )
                       : *reinterpret_cast< std::uintptr_t * >( vip - ( imm_size / 8 ) );
        }

        std::optional< virt_instr_t > get( vm::ctx_t &ctx, vmp2::v2::entry_t &entry )
        {
            virt_instr_t result;
            const auto &vm_handler = ctx.vm_handlers[ entry.handler_idx ];
            const auto profile = vm_handler.profile;

            result.mnemonic_t = profile ? profile->mnemonic : vm::handler::mnemonic_t::INVALID;
            result.opcode = entry.handler_idx;
            result.trace_data = entry;

            if ( vm_handler.imm_size )
            {
                result.operand.has_imm = true;
                const auto imm_val = get_imm( ctx, vm_handler.imm_size, entry.vip );

                if ( !imm_val.has_value() )
                    return {};

                result.operand.imm.u = imm_val.value();
            }
            else
                result.operand.has_imm = false;

            return result;
        }
    } // namespace instrs
} // namespace vm