[PATCH] mscoree: Implement VTable fixup for x86_64 architecture.

[Paul Gofman]

Loading .NET code from unmanaged app still does not work for 64bit due to
unsupported loading of pure .NET PE images in 64 bit process
(libmono64 fails to load mscorlib.dll).

Signed-off-by: Paul Gofman <gofmanp at gmail.com>
---
 dlls/mscoree/corruntimehost.c | 80 ++++++++++++++++++++++++++++++++++++-------
 1 file changed, 67 insertions(+), 13 deletions(-)

diff --git a/dlls/mscoree/corruntimehost.c b/dlls/mscoree/corruntimehost.c
index e437cc2..8dbd35a 100644
--- a/dlls/mscoree/corruntimehost.c
+++ b/dlls/mscoree/corruntimehost.c
@@ -919,7 +919,56 @@ static const struct vtable_fixup_thunk thunk_template = {
 
 #include "poppack.h"
 
-#else /* !defined(__i386__) */
+#elif __x86_64__ /* !defined(__i386__) */
+
+# define CAN_FIXUP_VTABLE 1
+
+#include "pshpack1.h"
+
+struct vtable_fixup_thunk
+{
+    /* push %rcx */
+    BYTE i1;
+    /* sub $0x20, %rsp */
+    BYTE i2[4];
+    /* mov function,%eax */
+    BYTE i3[2];
+    void (CDECL *function)(struct dll_fixup *);
+    /* mov fixup,%rcx */
+    BYTE i4[2];
+    struct dll_fixup *fixup;
+    /* call *%eax */
+    BYTE i5[2];
+    /* add $0x20, %rsp */
+    BYTE i6[4];
+    /* pop %rcx */
+    BYTE i7;
+    /* mov vtable_entry, %rax */
+    BYTE i8[2];
+    void *vtable_entry;
+    /* mov [%rax],%rax
+       jmp %rax */
+    BYTE i9[5];
+};
+
+static const struct vtable_fixup_thunk thunk_template = {
+    0x51,
+    {0x48,0x83,0xEC,0x20},
+    {0x48,0xB8},
+    NULL,
+    {0x48,0xB9},
+    NULL,
+    {0xFF,0xD0},
+    {0x48,0x83,0xC4,0x20},
+    0x59,
+    {0x48,0xB8},
+    NULL,
+    {0x48, 0x8B, 0x00, 0xFF, 0xE0}
+};
+
+#include "poppack.h"
+
+#else /* !defined(__i386__) && !defined(__x86_64__) */
 
 # define CAN_FIXUP_VTABLE 0
 
@@ -981,16 +1030,19 @@ static void CDECL ReallyFixupVTable(struct dll_fixup *fixup)
 
         /* Mono needs an image that belongs to an assembly. */
         image = mono_assembly_get_image(assembly);
-
+#if __x86_64__
+        if (fixup->fixup->type & COR_VTABLE_64BIT)
+#else
         if (fixup->fixup->type & COR_VTABLE_32BIT)
+#endif
         {
-            DWORD *vtable = fixup->vtable;
-            DWORD *tokens = fixup->tokens;
+            void **vtable = fixup->vtable;
+            ULONG_PTR *tokens = fixup->tokens;
             for (i=0; i<fixup->fixup->count; i++)
             {
-                TRACE("%x\n", tokens[i]);
-                vtable[i] = PtrToUint(mono_marshal_get_vtfixup_ftnptr(
-                    image, tokens[i], fixup->fixup->type));
+                TRACE("%#lx\n", tokens[i]);
+                vtable[i] = mono_marshal_get_vtfixup_ftnptr(
+                    image, tokens[i], fixup->fixup->type);
             }
         }
 
@@ -1029,16 +1081,18 @@ static void FixupVTableEntry(HMODULE hmodule, VTableFixup *vtable_fixup)
     fixup->vtable = (BYTE*)hmodule + vtable_fixup->rva;
     fixup->done = FALSE;
 
+    TRACE("vtable_fixup->type=0x%x\n",vtable_fixup->type);
+#if __x86_64__
+    if (vtable_fixup->type & COR_VTABLE_64BIT)
+#else
     if (vtable_fixup->type & COR_VTABLE_32BIT)
+#endif
     {
-        DWORD *vtable = fixup->vtable;
-        DWORD *tokens;
+        void **vtable = fixup->vtable;
+        ULONG_PTR *tokens;
         int i;
         struct vtable_fixup_thunk *thunks = fixup->thunk_code;
 
-        if (sizeof(void*) > 4)
-            ERR("32-bit fixup in 64-bit mode; broken image?\n");
-
         tokens = fixup->tokens = HeapAlloc(GetProcessHeap(), 0, sizeof(*tokens) * vtable_fixup->count);
         memcpy(tokens, vtable, sizeof(*tokens) * vtable_fixup->count);
         for (i=0; i<vtable_fixup->count; i++)
@@ -1047,7 +1101,7 @@ static void FixupVTableEntry(HMODULE hmodule, VTableFixup *vtable_fixup)
             thunks[i].fixup = fixup;
             thunks[i].function = ReallyFixupVTable;
             thunks[i].vtable_entry = &vtable[i];
-            vtable[i] = PtrToUint(&thunks[i]);
+            vtable[i] = &thunks[i];
         }
     }
     else
-- 
2.5.0