In the fluctuating and frequently competitive world of cyber security, the ability to reverse engineer is an essential skill, vital for comprehending and countering security risks. This article contributes an additional element to the knowledge of reverse engineering, aiming to demonstrate the identification of standard function calls in C/C++ based on the assembly of an x64 Windows application. The calling convention shown in this post is called Microsoft x64.
The reference platform is x64 (said also called x86_64 or amd64); the C/C++ compiler is the Microsoft cl.exe for x64 version 19.38.33133 for Windows 64 bit. The disassembler tool is IDA Free Version 8.3.230608 for Windows x64. The compilation is executed with default options of correspondent compilers, namely no optimization is invoked.
In this post the focus is only on functions calls on Windows operating system on 64 bit platform, so the analysis of the disassembled code will cover only way to pass parameters and retrieve the result value and nothing else; the syntax of assembly code is the Intel syntax (so for two operand instruction, the first operand is the destination, the second operand is the source). In the previous post we showed the implementation of the calling convention of System V X86_64 used by the C/C++ and Rust compilers on Linux x64.
c++ code and related assembly CODE
The following C++ code show 6 examples of functions calls that takes long long parameters and return an long long:
#include <iostream>
long long f1(long long a)
{
std::cout << "f1(" << a << ")" << std::endl;
return a;
}
long long f2(long long a, long long b)
{
std::cout << "f2(" << a << ", " << b << ")" << std::endl;
return a + b;
}
long long f3(long long a, long long b, long long c)
{
std::cout << "f3("
<< a << ", "
<< b << ", "
<< c << ")"
<< std::endl;
return a + b + c;
}
long long f4(long long a, long long b, long long c, long long d)
{
std::cout << "f4("
<< a << ", "
<< b << ", "
<< c << ", "
<< d << ")"
<< std::endl;
return a + b + c + d;
}
long long f5(long long a, long long b, long long c, long long d, long long e)
{
std::cout << "f5("
<< a << ", "
<< b << ", "
<< c << ", "
<< d << ", "
<< e << ")"
<< std::endl;
return a + b + c + d + e;
}
long long f6(long long a, long long b, long long c, long long d, long long e, long long f)
{
std::cout << "f6("
<< a << ", "
<< b << ", "
<< c << ", "
<< d << ", "
<< e << ", "
<< f << ")"
<< std::endl;
return a + b + c + d + e + f;
}
int main()
{
long long z;
z = f1(0x1000000000000001);
z = f2(0x1000000000000001, 0x1000000000000002);
z = f3(0x1000000000000001, 0x1000000000000002, 0x1000000000000003);
z = f4(0x1000000000000001, 0x1000000000000002, 0x1000000000000003, 0x1000000000000004);
z = f5(0x1000000000000001, 0x1000000000000002, 0x1000000000000003, 0x1000000000000004,
0x1000000000000005);
z = f6(0x1000000000000001, 0x1000000000000002, 0x1000000000000003, 0x1000000000000004,
0x1000000000000005, 0x1000000000000006);
return 0;
}
The first function f1 takes a long long type parameter as input, the second function f2 takes two long long type parameters, and so on until the last function f6, which takes 6 long long type parameters as input. To see the full code visit my space on GitHub at this address: https://github.com/ettoremessina/reverse-engineering/tree/main/windows/calls/mscpp/calls
The corresponding assembly code of the main function (that contains the calls to f1…f6 functions) generated by the compiler is as follows:
; Attributes: bp-based frame fpd=0F0h
; int __fastcall main()
main proc near
e= qword ptr -100h
f= qword ptr -0F8h
z= qword ptr -0E8h
push rbp
push rdi
sub rsp, 118h
lea rbp, [rsp+30h]
lea rcx, __6FB64B36_calls@cpp ; JMC_flag
call j___CheckForDebuggerJustMyCode
mov rcx, 1000000000000001h ; a
call j_?f1@@YA_J_J@Z ; f1(__int64)
mov [rbp+0F0h+z], rax
mov rdx, 1000000000000002h ; b
mov rcx, 1000000000000001h ; a
call j_?f2@@YA_J_J0@Z ; f2(__int64,__int64)
mov [rbp+0F0h+z], rax
mov r8, 1000000000000003h ; c
mov rdx, 1000000000000002h ; b
mov rcx, 1000000000000001h ; a
call j_?f3@@YA_J_J00@Z ; f3(__int64,__int64,__int64)
mov [rbp+0F0h+z], rax
mov r9, 1000000000000004h ; d
mov r8, 1000000000000003h ; c
mov rdx, 1000000000000002h ; b
mov rcx, 1000000000000001h ; a
call j_?f4@@YA_J_J000@Z ; f4(__int64,__int64,__int64,__int64)
mov [rbp+0F0h+z], rax
mov rax, 1000000000000005h
mov [rsp+120h+e], rax ; e
mov r9, 1000000000000004h ; d
mov r8, 1000000000000003h ; c
mov rdx, 1000000000000002h ; b
mov rcx, 1000000000000001h ; a
call j_?f5@@YA_J_J0000@Z ; f5(__int64,__int64,__int64,__int64,__int64)
mov [rbp+0F0h+z], rax
mov rax, 1000000000000006h
mov [rsp+120h+f], rax ; f
mov rax, 1000000000000005h
mov [rsp+120h+e], rax ; e
mov r9, 1000000000000004h ; d
mov r8, 1000000000000003h ; c
mov rdx, 1000000000000002h ; b
mov rcx, 1000000000000001h ; a
call j_?f6@@YA_J_J00000@Z ; f6(__int64,__int64,__int64,__int64,__int64,__int64)
mov [rbp+0F0h+z], rax
xor eax, eax
lea rsp, [rbp+0E8h]
pop rdi
pop rbp
retn
main endpBy disassembling we can clearly see that the first into rcx, the second into rdx, the third into r8, and the fourth into r9. From the fifth parameter onward the stack is used from right to left, so the fifth is the last one to be pushed. The return value is always passed via rax register.
Note: Be aware that compilers may not always employ push instructions to transfer these arguments onto the stack. It’s a typical approach to reserve sufficient space on the stack for all of a function’s outgoing arguments during the function prologue, followed by utilizing mov instructions to position arguments on the stack when needed.
