vmprofiler/src/vminstrs.cpp

#include <vmprofiler.hpp>

namespace vm
{
    namespace instrs
    {
        std::pair< std::uint64_t, std::uint64_t > decrypt_operand( transform::map_t &transforms, std::uint64_t operand,
                                                                   std::uint64_t rolling_key )
        {
            const auto &generic_decrypt_0 = transforms[ transform::type::generic0 ];
            const auto &key_decrypt = transforms[ transform::type::rolling_key ];
            const auto &generic_decrypt_1 = transforms[ transform::type::generic1 ];
            const auto &generic_decrypt_2 = transforms[ transform::type::generic2 ];
            const auto &generic_decrypt_3 = transforms[ transform::type::generic3 ];
            const auto &update_key = transforms[ transform::type::update_key ];

            if ( generic_decrypt_0.mnemonic != ZYDIS_MNEMONIC_INVALID )
            {
                operand = transform::apply(
                    generic_decrypt_0.operands[ 0 ].size, generic_decrypt_0.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_0 ) ? generic_decrypt_0.operands[ 1 ].imm.value.u : 0 );
            }

            // apply transformation with rolling decrypt key...
            operand = transform::apply( key_decrypt.operands[ 0 ].size, key_decrypt.mnemonic, operand, rolling_key );

            // apply three generic transformations...
            {
                operand = transform::apply(
                    generic_decrypt_1.operands[ 0 ].size, generic_decrypt_1.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_1 ) ? generic_decrypt_1.operands[ 1 ].imm.value.u : 0 );

                operand = transform::apply(
                    generic_decrypt_2.operands[ 0 ].size, generic_decrypt_2.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_2 ) ? generic_decrypt_2.operands[ 1 ].imm.value.u : 0 );

                operand = transform::apply(
                    generic_decrypt_3.operands[ 0 ].size, generic_decrypt_3.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_3 ) ? generic_decrypt_3.operands[ 1 ].imm.value.u : 0 );
            }

            // update rolling key...
            auto result = transform::apply( update_key.operands[ 0 ].size, update_key.mnemonic, rolling_key, operand );

            // update decryption key correctly...
            switch ( update_key.operands[ 0 ].size )
            {
            case 8:
                rolling_key = ( rolling_key & ~0xFFull ) + result;
                break;
            case 16:
                rolling_key = ( rolling_key & ~0xFFFFull ) + result;
                break;
            default:
                rolling_key = result;
                break;
            }

            return { operand, rolling_key };
        }

        std::pair< std::uint64_t, std::uint64_t > encrypt_operand( transform::map_t &transforms, std::uint64_t operand,
                                                                   std::uint64_t rolling_key )
        {
            transform::map_t inverse;
            inverse_transforms( transforms, inverse );

            const auto &generic_decrypt_0 = inverse[ transform::type::generic0 ];
            const auto &key_decrypt = inverse[ transform::type::rolling_key ];
            const auto &generic_decrypt_1 = inverse[ transform::type::generic1 ];
            const auto &generic_decrypt_2 = inverse[ transform::type::generic2 ];
            const auto &generic_decrypt_3 = inverse[ transform::type::generic3 ];
            const auto &update_key = inverse[ transform::type::update_key ];

            auto result = transform::apply( update_key.operands[ 0 ].size, update_key.mnemonic, rolling_key, operand );

            // make sure we update the rolling decryption key correctly...
            switch ( update_key.operands[ 0 ].size )
            {
            case 8:
                rolling_key = ( rolling_key & ~0xFFull ) + result;
                break;
            case 16:
                rolling_key = ( rolling_key & ~0xFFFFull ) + result;
                break;
            default:
                rolling_key = result;
                break;
            }

            {
                operand = transform::apply(
                    generic_decrypt_3.operands[ 0 ].size, generic_decrypt_3.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_3 ) ? generic_decrypt_3.operands[ 1 ].imm.value.u : 0 );

                operand = transform::apply(
                    generic_decrypt_2.operands[ 0 ].size, generic_decrypt_2.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_2 ) ? generic_decrypt_2.operands[ 1 ].imm.value.u : 0 );

                operand = transform::apply(
                    generic_decrypt_1.operands[ 0 ].size, generic_decrypt_1.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_1 ) ? generic_decrypt_1.operands[ 1 ].imm.value.u : 0 );
            }

            operand = transform::apply( key_decrypt.operands[ 0 ].size, key_decrypt.mnemonic, operand, rolling_key );

            if ( generic_decrypt_0.mnemonic != ZYDIS_MNEMONIC_INVALID )
            {
                operand = transform::apply(
                    generic_decrypt_0.operands[ 0 ].size, generic_decrypt_0.mnemonic, operand,
                    // check to see if this instruction has an IMM...
                    transform::has_imm( &generic_decrypt_0 ) ? generic_decrypt_0.operands[ 1 ].imm.value.u : 0 );
            }

            return { operand, rolling_key };
        }

        bool get_rva_decrypt( const zydis_routine_t &vm_entry, std::vector< zydis_decoded_instr_t > &transform_instrs )
        {
            //
            // find mov esi, [rsp+0xA0]
            //

            auto result =
                std::find_if( vm_entry.begin(), vm_entry.end(), []( const zydis_instr_t &instr_data ) -> bool {
                    if ( instr_data.instr.mnemonic == ZYDIS_MNEMONIC_MOV && instr_data.instr.operand_count == 2 &&
                         instr_data.instr.operands[ 0 ].reg.value == ZYDIS_REGISTER_ESI &&
                         instr_data.instr.operands[ 1 ].mem.base == ZYDIS_REGISTER_RSP &&
                         instr_data.instr.operands[ 1 ].mem.disp.has_displacement &&
                         instr_data.instr.operands[ 1 ].mem.disp.value == 0xA0 )
                        return true;
                    return false;
                } );

            if ( result == vm_entry.end() )
                return false;

            //
            // find the next three instruction with ESI as the dest...
            //

            for ( auto idx = 0u; idx < 3; ++idx )
            {
                result = std::find_if( ++result, vm_entry.end(), []( const zydis_instr_t &instr_data ) -> bool {
                    return instr_data.instr.operands[ 0 ].reg.value == ZYDIS_REGISTER_ESI;
                } );

                if ( result == vm_entry.end() )
                    return false;

                transform_instrs.push_back( result->instr );
            }

            return true;
        }

        std::optional< std::uint64_t > get_imm( vm::ctx_t &ctx, std::uint8_t imm_size, std::uintptr_t vip )
        {
            if ( !imm_size )
                return {};

            std::uint64_t result = 0u;
            if ( ctx.exec_type == vmp2::exec_type_t::forward )
                std::memcpy( &result, reinterpret_cast< void * >( vip ), imm_size / 8 );
            else // else the second operand is below vip...
                std::memcpy( &result, reinterpret_cast< void * >( vip - ( imm_size / 8 ) ), imm_size / 8 );

            return result;
        }

        std::optional< virt_instr_t > get( vm::ctx_t &ctx, vmp2::v2::entry_t &entry )
        {
            virt_instr_t result;
            auto &vm_handler = ctx.vm_handlers[ entry.handler_idx ];
            const auto profile = vm_handler.profile;

            result.mnemonic_t = profile ? profile->mnemonic : vm::handler::INVALID;
            result.opcode = entry.handler_idx;
            result.trace_data = entry;

            if ( vm_handler.imm_size )
            {
                result.operand.has_imm = true;
                result.operand.imm.imm_size = vm_handler.imm_size;
                const auto imm_val = get_imm( ctx, vm_handler.imm_size, entry.vip );

                if ( !imm_val.has_value() )
                    return {};

                result.operand.imm.u =
                    vm::instrs::decrypt_operand( vm_handler.transforms, imm_val.value(), entry.decrypt_key ).first;
            }
            else
                result.operand.has_imm = false;

            return result;
        }

        std::optional< jcc_data > get_jcc_data( vm::ctx_t &vmctx, code_block_t &code_block )
        {
            // there is no branch for this as this is a vmexit...
            if ( code_block.vinstrs.back().mnemonic_t == vm::handler::VMEXIT )
                return {};

            // find the last LCONSTDW... the imm value is the JMP xor decrypt key...
            // we loop backwards here (using rbegin and rend)...
            auto result = std::find_if( code_block.vinstrs.rbegin(), code_block.vinstrs.rend(),
                                        []( const vm::instrs::virt_instr_t &vinstr ) -> bool {
                                            auto profile = vm::handler::get_profile( vinstr.mnemonic_t );
                                            return profile && profile->mnemonic == vm::handler::LCONSTDW;
                                        } );

            jcc_data jcc;
            const auto xor_key = static_cast< std::uint32_t >( result->operand.imm.u );
            const auto &last_trace = code_block.vinstrs.back().trace_data;

            // since result is already a variable and is a reverse itr
            // im going to be using rbegin and rend here again...
            //
            // look for PUSHVSP virtual instructions with two encrypted virtual
            // instruction rva's ontop of the virtual stack...
            result = std::find_if( code_block.vinstrs.rbegin(), code_block.vinstrs.rend(),
                                   [ & ]( const vm::instrs::virt_instr_t &vinstr ) -> bool {
                                       auto profile = vm::handler::get_profile( vinstr.mnemonic_t );
                                       if ( profile && profile->mnemonic == vm::handler::PUSHVSP )
                                       {
                                           const auto possible_block_1 =
                                               code_block_addr( vmctx, vinstr.trace_data.vsp.qword[ 0 ] ^ xor_key );

                                           const auto possible_block_2 =
                                               code_block_addr( vmctx, vinstr.trace_data.vsp.qword[ 1 ] ^ xor_key );

                                           // if this returns too many false positives we might have to get
                                           // our hands dirty and look into trying to emulate each branch
                                           // to see if the first instruction is an SREGQ...
                                           return possible_block_1 > vmctx.module_base &&
                                                  possible_block_1 < vmctx.module_base + vmctx.image_size &&
                                                  possible_block_2 > vmctx.module_base &&
                                                  possible_block_2 < vmctx.module_base + vmctx.image_size;
                                       }

                                       return false;
                                   } );

            // if there is not two branches...
            if ( result == code_block.vinstrs.rend() )
            {
                jcc.block_addr[ 0 ] = code_block_addr( vmctx, last_trace );

                jcc.has_jcc = false;
                jcc.type = jcc_type::absolute;
            }
            // else there are two branches...
            else
            {
                jcc.block_addr[ 0 ] = code_block_addr( vmctx, result->trace_data.vsp.qword[ 0 ] ^ xor_key );
                jcc.block_addr[ 1 ] = code_block_addr( vmctx, result->trace_data.vsp.qword[ 1 ] ^ xor_key );

                jcc.has_jcc = true;
                jcc.type = jcc_type::branching;
            }

            return jcc;
        }

        std::uintptr_t code_block_addr( const vm::ctx_t &ctx, const vmp2::v2::entry_t &entry )
        {
            return ( ( entry.vsp.qword[ 0 ] + ( ctx.image_base & ~0xFFFFFFFFull ) ) - ctx.image_base ) +
                   ctx.module_base;
        }

        std::uintptr_t code_block_addr( const vm::ctx_t &ctx, const std::uint32_t lower_32bits )
        {
            return ( ( lower_32bits + ( ctx.image_base & ~0xFFFFFFFFull ) ) - ctx.image_base ) + ctx.module_base;
        }
    } // namespace instrs
} // namespace vm