c - Hooking usercall function? -


i have virtual machine, on vm_create passes address of function (systemcalls) virtual machine.

so hook vm_create , steal syscalls address, put backup function pointer, , address of modified systemcalls function pass original vm_create, can alter arguments, add or remove calls, , call backed-up syscalls function. worked well, until new release of game.

i believe have found problem:

this beginning of unmodified systemcalls function:

intptr_t cl_cgamesystemcalls(intptr_t *args) {      switch (args[0]) {          case cg_print:             com_printf( "%s", (const char*)vma(1));             return 0;          case cg_error:             com_error(err_drop, "%s", (const char*)vma(1));             return 0; 

this modifed syscall function:

intptr_t modified_cl_cgamesystemcalls (intptr_t *args) {     switch (*args)     {         case cg_getsnapshot:              mysnap = mysnap ;             mynextsnap = (snapshot_t*) (cg_qvm2native(args[2]));             mysnap = mynextsnap;              retval = original_cl_cgamesystemcalls(args);             break ; 

the problem calling original function modified one:

    intptr_t modified_cl_cgamesystemcalls(intptr_t *args)     {         retval = original_cl_cgamesystemcalls(args);         return retval;     } 

already fails.

as can see pseudocode of disassembly, new definition of cl_cgamesystemcalls seems be:

char __usercall sub_4017b0<al>(int a1<ebx>, int a2) 

which means changed function adding __usercall attribute , putting first argument register ebx, if interpret decompilation right.

now question:

how can retrieve *args (args[0]) variable?

and how can call unmodified function modified one, uses __usercall?

this disassembly of systemcalls usercall:

.text:004017b0 ; =============== s u b r o u t n e ======================================= .text:004017b0 .text:004017b0 .text:004017b0 sub_4017b0      proc near               ; data xref: sub_402670+5co .text:004017b0 .text:004017b0 var_18          = dword ptr -18h .text:004017b0 var_4           = dword ptr -4 .text:004017b0 arg_0           = dword ptr  4 .text:004017b0 .text:004017b0 ; function chunk @ .text:00401430 size 00000026 bytes .text:004017b0 ; function chunk @ .text:00401459 size 00000013 bytes .text:004017b0 ; function chunk @ .text:00410e90 size 00000006 bytes .text:004017b0 ; function chunk @ .text:00412ac0 size 00000006 bytes .text:004017b0 .text:004017b0                 push    esi .text:004017b1                 mov     esi, [esp+0ch+var_4] .text:004017b5                 mov     eax, [esi] .text:004017b7                 cmp     eax, 73h        ; switch 116 cases .text:004017ba                 push    edi .text:004017bb                 ja      loc_402486      ; default .text:004017bb                                         ; jumptable 004017c1 cases 21,90-99,109,110 .text:004017c1                 jmp     ds:off_40249c[eax*4] ; switch jump .text:004017c8 .text:004017c8 loc_4017c8:                             ; data xref: .text:off_40249co .text:004017c8                 mov     eax, [esi+4]    ; jumptable 004017c1 case 0 .text:004017cb                 push    eax .text:004017cc                 call    vm_argptr .text:004017d1                 push    eax             ; char .text:004017d2                 push    offset as_5     ; "%s" .text:004017d7                 call    com_printf .text:004017dc                 add     esp, 0ch .text:004017df                 pop     edi .text:004017e0                 xor     eax, eax .text:004017e2                 pop     esi .text:004017e3                 retn .text:004017e4 ; --------------------------------------------------------------------------- .text:004017e4 .text:004017e4 loc_4017e4:                             ; code xref: sub_4017b0+11j .text:004017e4                                         ; data xref: .text:off_40249co .text:004017e4                 mov     ecx, [esi+4]    ; jumptable 004017c1 case 1 .text:004017e7                 push    ecx .text:004017e8                 call    vm_argptr .text:004017ed                 push    eax             ; char .text:004017ee                 push    offset as_5     ; "%s" .text:004017f3                 push    1               ; int .text:004017f5                 call    com_error .text:004017f5 ; -------------------------------------------------------------- 

and pseudocode created hexrays decompiler:

char __usercall sub_4017b0<al>(int a1<ebx>, int a2) {     int v2;           // st34_4@1     char result;      // al@2     int v4;           // st34_4@2     int v5;           // eax@2     int v6;           // st34_4@3     int v7;           // eax@3     int v8;           // st34_4@5     int v9;           // st24_4@5     int v10;          // st20_4@5     int v11;          // st1c_4@5     int v12;          // eax@5     int v13;          // st34_4@6     int v14;          // eax@6     int v15;          // st34_4@7     int v16;          // st24_4@7     int v17;          // eax@7     int v18;          // st34_4@8     signed int v19;   // st24_4@8     int v20;          // st20_4@8     int v21;          // eax@8     int v22;          // st34_4@10     signed int v23;   // st24_4@10     int v24;          // eax@10     int v25;          // st34_4@11     signed int v26;   // st24_4@11     int v27;          // eax@11     int v28;          // st34_4@12     int v29;          // st24_4@12     int v30;          // st20_4@12     int v31;          // eax@12     int v32;          // st34_4@13     int v33;          // st24_4@13     int v34;          // st20_4@13     int v35;          // eax@13     int v36;          // st34_4@14     int v37;          // st24_4@14     size_t v38;       // st20_4@14     int v39;          // eax@14     int v40;          // st34_4@15     int v41;          // st34_4@16     int v42;          // st34_4@17     int v43;          // eax@17     int v44;          // st34_4@18     int v45;          // eax@18     int v46;          // st34_4@19     int v47;          // eax@19     int v48;          // st34_4@20     int v49;          // eax@20     int v50;          // st34_4@22     int v51;          // eax@22     void *v52;        // ecx@22     int v53;          // st34_4@24     int v54;          // st20_4@24     int v55;          // eax@24     int v56;          // st34_4@25     int v57;          // st20_4@25     int v58;          // eax@25     int v59;          // st34_4@26     signed int v60;   // st24_4@26     int v61;          // eax@26     int v62;          // st34_4@27     int v63;          // st24_4@27     int v64;          // st20_4@27     signed int v65;   // st1c_4@27     int v66;          // eax@27     int v67;          // st34_4@28     int v68;          // eax@29     int v69;          // st34_4@30     int v70;          // st34_4@32     int v71;          // st20_4@32     int v72;          // st1c_4@32     int v73;          // st18_4@32     signed int v74;   // st14_4@32     int v75;          // st10_4@32     int v76;          // st0c_4@32     int v77;          // st08_4@32     int v78;          // st04_4@32     int v79;          // eax@32     int v80;          // st34_4@34     int v81;          // st24_4@34     int v82;          // st20_4@34     int v83;          // st1c_4@34     int v84;          // st18_4@34     int v85;          // st14_4@34     int v86;          // eax@34     int v87;          // st34_4@35     int v88;          // st24_4@35     int v89;          // st20_4@35     int v90;          // st1c_4@35     int v91;          // eax@35     int v92;          // st34_4@36     int v93;          // st34_4@37     int v94;          // st34_4@38     int v95;          // st24_4@38     int v96;          // st20_4@38     int v97;          // eax@38     int v98;          // st34_4@39     int v99;          // st24_4@39     int v100;         // st20_4@39     int v101;         // eax@39     int v102;         // st34_4@40     int v103;         // st34_4@41     int v104;         // eax@41     int v105;         // st34_4@42     int v106;         // st20_4@42     int v107;         // eax@42     int v108;         // st34_4@43     int v109;         // eax@43     int v110;         // st34_4@44     int v111;         // st24_4@44     int v112;         // eax@44     int v113;         // st34_4@45     int v114;         // eax@45     int v115;         // st34_4@46     int v116;         // eax@46     int v117;         // st34_4@47     int v118;         // st34_4@48     int v119;         // st34_4@49     int v120;         // st34_4@50     int v121;         // st24_4@50     int v122;         // st20_4@50     int v123;         // eax@50     int v124;         // st34_4@52     int v125;         // eax@52     int v126;         // st34_4@53     int v127;         // eax@53     int v128;         // st34_4@54     int v129;         // st24_4@54     int v130;         // eax@54     int v131;         // st34_4@55     int v132;         // st24_4@55     int v133;         // st20_4@55     int v134;         // st1c_4@55     int v135;         // eax@55     int v136;         // st34_4@56     float v137;       // st24_4@56     float v138;       // st20_4@56     float v139;       // st1c_4@56     float v140;       // st18_4@56     int v141;         // eax@56     int v142;         // st34_4@57     float v143;       // st24_4@57     float v144;       // st20_4@57     float v145;       // st1c_4@57     float v146;       // st18_4@57     int v147;         // eax@57     int v148;         // st34_4@58     int v149;         // eax@58     int v150;         // st34_4@59     int v151;         // eax@59     int v152;         // st34_4@60     int v153;         // st34_4@61     int v154;         // st24_4@61     int v155;         // eax@61     int v156;         // st34_4@62     int v157;         // st24_4@62     float v158;       // st20_4@62     int v159;         // st1c_4@62     int v160;         // st18_4@62     int v161;         // st14_4@62     int v162;         // eax@62     int v163;         // st34_4@63     int v164;         // eax@63     int v165;         // st34_4@64     int v166;         // eax@64     int v167;         // st34_4@65     int v168;         // edi@65     int v169;         // st34_4@66     int v170;         // eax@66     int v171;         // st34_4@68     int v172;         // eax@68     int v173;         // edx@69     int v174;         // st34_4@69     float v175;       // st34_4@69     int v176;         // st34_4@72     int v177;         // st34_4@73     int v178;         // st34_4@74     unsigned int v179;// st24_4@74     int v180;         // st20_4@74     int v181;         // eax@74     int v182;         // st34_4@75     signed int v183;  // st24_4@75     int v184;         // st20_4@75     int v185;         // eax@75     int v186;         // st34_4@76     size_t v187;      // st24_4@76     int v188;         // st20_4@76     int v189;         // eax@76     int v190;         // st34_4@77     float v191;       // st34_4@77     int v192;         // st34_4@78     float v193;       // st34_4@78     double v194;      // st7@79     int v195;         // st34_4@79     float v196;       // st34_4@79     int v197;         // st34_4@80     float v198;       // st34_4@80     int v199;         // st34_4@81     float v200;       // st34_4@81     int v201;         // st34_4@82     float v202;       // st34_4@82     int v203;         // st34_4@83     float v204;       // st34_4@83     int v205;         // st34_4@84     int v206;         // edi@84     int v207;         // eax@84     int v208;         // st34_4@85     int v209;         // edi@85     int v210;         // eax@85     int v211;         // st34_4@87     int v212;         // edi@87     int v213;         // eax@87     int v214;         // st34_4@88     int v215;         // edi@88     int v216;         // st24_4@88     int v217;         // eax@88     int v218;         // st34_4@90     int v219;         // st34_4@91     int v220;         // eax@91     int v221;         // st34_4@92     unsigned int v222;// st24_4@92     int v223;         // st20_4@92     int v224;         // st1c_4@92     int v225;         // st18_4@92     int v226;         // st14_4@92     int v227;         // eax@92     int v228;         // st34_4@95     int v229;         // st34_4@96     int v230;         // st34_4@97     int v231;         // st24_4@97     int v232;         // st20_4@97     int v233;         // eax@97     int v234;         // st34_4@98     int v235;         // st24_4@98     int v236;         // eax@98     int v237;         // st34_4@99     int v238;         // st24_4@99     int v239;         // eax@99     int v240;         // st34_4@100     int v241;         // eax@100     int v242;         // st34_4@101     int v243;         // eax@101     int v244;         // st34_4@102     int v245;         // eax@102     int v246;         // st34_4@103     int v247;         // st20_4@103     int v248;         // eax@103     int v249;         // st34_4@104     int v250;         // [sp-18h] [bp-30h]@28     int v251;         // [sp-14h] [bp-2ch]@28     int v252;         // [sp-10h] [bp-28h]@28     int v253;         // [sp-ch] [bp-24h]@28     int v254;         // [sp-8h] [bp-20h]@28     signed int v255;  // [sp-4h] [bp-1ch]@28     int v256;         // [sp+0h] [bp-18h]@28     int v257;         // [sp+4h] [bp-14h]@28     int v258;         // [sp+4h] [bp-14h]@31      switch (*(_dword *)v2)     {         case 0:             v5 = vm_argptr(*(_dword *)(v4 + 4));             com_printf("%s", v5);             return 0;          case 1:             v7 = vm_argptr(*(_dword *)(v6 + 4));             com_error(1, "%s", v7);             return result;          case 2:             return sub_447700();          case 3:             v9 = *(_dword *)(v8 + 16);             v10 = vm_argptr(*(_dword *)(v8 + 12));             v11 = vm_argptr(*(_dword *)(v8 + 8));             v12 = vm_argptr(*(_dword *)(v8 + 4));             sub_4213c0(v12, (const char *)v11, v10, v9);             return 0;          case 4:          ...          default:             com_error(1, "bad cgame system trap: %i", *(_dword *)v249);             return result;     }     return result; } 

you can find full (last known official) source of cl_cgamesystemcalls function here (too copy-paste):

http://ioqsrc.vampireducks.com/da/d3b/cl__cgame_8c-source.html

and here disassembly of old version, in calling orig_syscall modified syscall worked:

.text:00402b40 ; =============== s u b r o u t n e ======================================= .text:00402b40 .text:00402b40 .text:00402b40 sub_402b40      proc near               ; code xref: sub_40b380+bp .text:00402b40                                         ; sub_40e3b0+bp ... .text:00402b40                 mov     edx, dword_bbe104 .text:00402b46                 mov     eax, dword_cb60ec .text:00402b4b                 ,     edx, 0fffffff7h .text:00402b4e                 test    eax, eax .text:00402b50                 mov     dword_bbe104, edx .text:00402b56                 mov     dword_bbe21c, 0 .text:00402b60                 jz      short locret_402b82 .text:00402b62                 push    1 .text:00402b64                 push    eax .text:00402b65                 call    sub_43e360 .text:00402b6a                 mov     eax, dword_cb60ec .text:00402b6f                 push    eax .text:00402b70                 call    sub_43e270 .text:00402b75                 add     esp, 0ch .text:00402b78                 mov     dword_cb60ec, 0 .text:00402b82 .text:00402b82 locret_402b82:                          ; code xref: sub_402b40+20j .text:00402b82                 retn .text:00402b82 sub_402b40      endp .text:00402b82 .text:00402b82 ; --------------------------------------------------------------------------- .text:00402b83                 align 10h .text:00402b90 .text:00402b90 loc_402b90:                             ; data xref: sub_403aa0+5co .text:00402b90                 push    ecx .text:00402b91                 push    ebx .text:00402b92                 push    esi .text:00402b93                 push    edi .text:00402b94                 mov     edi, [esp+14h] .text:00402b98                 mov     eax, [edi] .text:00402b9a                 cmp     eax, 6fh        ; switch 112 cases .text:00402b9d                 ja      loc_4038c7      ; default .text:00402b9d                                         ; jumptable 00402ba3 cases 21,90-99,109,110 .text:00402ba3                 jmp     ds:off_4038e0[eax*4] ; switch jump .text:00402baa .text:00402baa loc_402baa:                             ; data xref: .text:off_4038e0o .text:00402baa                 mov     eax, [edi+4]    ; jumptable 00402ba3 case 0 .text:00402bad                 push    eax .text:00402bae                 call    sub_43e300 .text:00402bb3                 push    eax .text:00402bb4                 push    offset as_7     ; "%s" .text:00402bb9                 call    sub_41bb90 .text:00402bbe                 add     esp, 0ch .text:00402bc1                 pop     edi .text:00402bc2                 pop     esi .text:00402bc3                 xor     eax, eax .text:00402bc5                 pop     ebx .text:00402bc6                 pop     ecx .text:00402bc7                 retn .text:00402bc8 ; --------------------------------------------------------------------------- .text:00402bc8 .text:00402bc8 loc_402bc8:                             ; code xref: .text:00402ba3j .text:00402bc8                                         ; data xref: .text:off_4038e0o .text:00402bc8                 mov     ecx, [edi+4]    ; jumptable 00402ba3 case 1 .text:00402bcb                 push    ecx .text:00402bcc                 call    sub_43e300 .text:00402bd1                 push    eax .text:00402bd2                 push    offset as_7     ; "%s" .text:00402bd7                 push    1 .text:00402bd9                 call    sub_41d850 .text:00402bde ; --------------------------------------------------------------------------- 

there no such thing __usercall. hex-rays uses represent non-standard calling convention. can happen in 3 ways:

  1. the programmer used assembly
  2. the programmer using open watcom's #pragma aux define custom convention
  3. the programmer using visual c++ 7+, optimizes register allocation prevent thrashing

in cases this, there 2 ways go, use assembly or use open watcom (which got update after years of abandonment). inline assembly works best imo, you'll need wrapper going in (to call original) , wrapper coming out (to hook original).


Comments