[PATCH v3] mscoree: Implement VTable fixup for x86_64 architecture.

[Paul Gofman]

Signed-off-by: Paul Gofman <gofmanp at gmail.com>
---
 dlls/mscoree/corruntimehost.c | 91 +++++++++++++++++++++++++++++++++++++------
 1 file changed, 79 insertions(+), 12 deletions(-)

diff --git a/dlls/mscoree/corruntimehost.c b/dlls/mscoree/corruntimehost.c
index e437cc2..05ac9c9 100644
--- a/dlls/mscoree/corruntimehost.c
+++ b/dlls/mscoree/corruntimehost.c
@@ -919,7 +919,68 @@ static const struct vtable_fixup_thunk thunk_template = {
 
 #include "poppack.h"
 
-#else /* !defined(__i386__) */
+#elif __x86_64__ /* !__i386__ */
+
+# define CAN_FIXUP_VTABLE 1
+
+#include "pshpack1.h"
+
+struct vtable_fixup_thunk
+{
+    /* push %rcx; push %rdx; push %r8; push %r9 */
+    BYTE i1[6];
+    /*  sub $0x68, %rsp ; 0x10*4 + 0x20 + 0x8 (align stack for subsequent func call)
+        movups %xmm0,0x28(%esp); ...; movups %xmm5,0x58(%esp)
+
+        ReallyFixupVTable will crash if we are called with unaligned stack
+    */
+    BYTE i2[28];
+    /* mov function,%eax */
+    BYTE i3[2];
+    void (CDECL *function)(struct dll_fixup *);
+    /* mov fixup,%rcx */
+    BYTE i4[2];
+    struct dll_fixup *fixup;
+    /* call *%eax */
+    BYTE i5[2];
+    /* 
+        movups 0x28(%esp),xmm0; ...; movups 0x58(%esp),xmm3
+        add $0x68, %rsp
+    */
+    BYTE i6[28];
+    /* pop %r9; pop %r8; pop %rdx; pop %rcx */
+    BYTE i7[6];
+    /* mov vtable_entry, %rax */
+    BYTE i8[2];
+    void *vtable_entry;
+    /* mov [%rax],%rax
+       jmp %rax */
+    BYTE i9[5];
+};
+
+static const struct vtable_fixup_thunk thunk_template = {
+    {0x51,0x52,0x41,0x50,0x41,0x51},
+    {0x48,0x83,0xEC,0x68,
+     0x67,0x0F,0x11,0x44,0x24,0x28, 0x67,0x0F,0x11,0x4C,0x24,0x38,
+     0x67,0x0F,0x11,0x54,0x24,0x48, 0x67,0x0F,0x11,0x5C,0x24,0x58,
+    },
+    {0x48,0xB8},
+    NULL,
+    {0x48,0xB9},
+    NULL,
+    {0xFF,0xD0},
+    {0x67,0x0F,0x10,0x44,0x24,0x28, 0x67,0x0F,0x10,0x4C,0x24,0x38,
+     0x67,0x0F,0x10,0x54,0x24,0x48, 0x67,0x0F,0x10,0x5C,0x24,0x58,
+     0x48,0x83,0xC4,0x68},
+    {0x41,0x59,0x41,0x58,0x5A,0x59},
+    {0x48,0xB8},
+    NULL,
+    {0x48,0x8B,0x00,0xFF,0xE0}
+};
+
+#include "poppack.h"
+
+#else /* !__i386__ && !__x86_64__ */
 
 # define CAN_FIXUP_VTABLE 0
 
@@ -982,15 +1043,19 @@ static void CDECL ReallyFixupVTable(struct dll_fixup *fixup)
         /* Mono needs an image that belongs to an assembly. */
         image = mono_assembly_get_image(assembly);
 
+#if __x86_64__
+        if (fixup->fixup->type & COR_VTABLE_64BIT)
+#else
         if (fixup->fixup->type & COR_VTABLE_32BIT)
+#endif
         {
-            DWORD *vtable = fixup->vtable;
-            DWORD *tokens = fixup->tokens;
+            void **vtable = fixup->vtable;
+            ULONG_PTR *tokens = fixup->tokens;
             for (i=0; i<fixup->fixup->count; i++)
             {
-                TRACE("%x\n", tokens[i]);
-                vtable[i] = PtrToUint(mono_marshal_get_vtfixup_ftnptr(
-                    image, tokens[i], fixup->fixup->type));
+                TRACE("%#lx\n", tokens[i]);
+                vtable[i] = mono_marshal_get_vtfixup_ftnptr(
+                    image, tokens[i], fixup->fixup->type);
             }
         }
 
@@ -1029,16 +1094,18 @@ static void FixupVTableEntry(HMODULE hmodule, VTableFixup *vtable_fixup)
     fixup->vtable = (BYTE*)hmodule + vtable_fixup->rva;
     fixup->done = FALSE;
 
+    TRACE("vtable_fixup->type=0x%x\n",vtable_fixup->type);
+#if __x86_64__
+    if (vtable_fixup->type & COR_VTABLE_64BIT)
+#else
     if (vtable_fixup->type & COR_VTABLE_32BIT)
+#endif
     {
-        DWORD *vtable = fixup->vtable;
-        DWORD *tokens;
+        void **vtable = fixup->vtable;
+        ULONG_PTR *tokens;
         int i;
         struct vtable_fixup_thunk *thunks = fixup->thunk_code;
 
-        if (sizeof(void*) > 4)
-            ERR("32-bit fixup in 64-bit mode; broken image?\n");
-
         tokens = fixup->tokens = HeapAlloc(GetProcessHeap(), 0, sizeof(*tokens) * vtable_fixup->count);
         memcpy(tokens, vtable, sizeof(*tokens) * vtable_fixup->count);
         for (i=0; i<vtable_fixup->count; i++)
@@ -1047,7 +1114,7 @@ static void FixupVTableEntry(HMODULE hmodule, VTableFixup *vtable_fixup)
             thunks[i].fixup = fixup;
             thunks[i].function = ReallyFixupVTable;
             thunks[i].vtable_entry = &vtable[i];
-            vtable[i] = PtrToUint(&thunks[i]);
+            vtable[i] = &thunks[i];
         }
     }
     else
-- 
2.5.0