[PATCH 2/2] ntdll/tests: Add tests for RtlEncode(System)Pointer.

[Myah Caron]

Signed-off-by: Myah Caron <qsniyg at protonmail.com>
---
I'm sending this in without an implementation in case there's anything I need to change, as the implementation
would mostly be a copy+paste job I believe (with the exception of implementing both process/usd cookies).

It turns out that newer Windows versions rotate the cookie and the pointer right by an amount specified by the
first 31/63 bits before XORing them.

Unfortunately I couldn't find any mention of how this algorithm worked on google, so I had to spend some time
to figure it out.

If you're curious, here's the file I created to test how the the algorithm worked (sorry for the mess):
https://github.com/qsniyg/wine_dll_load_test/blob/master/encodeptr.c

 dlls/ntdll/tests/rtl.c | 83 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 83 insertions(+)

diff --git a/dlls/ntdll/tests/rtl.c b/dlls/ntdll/tests/rtl.c
index 7a62670ea05..416264bac0f 100644
--- a/dlls/ntdll/tests/rtl.c
+++ b/dlls/ntdll/tests/rtl.c
@@ -82,6 +82,11 @@ static NTSTATUS  (WINAPI *pRtlInitializeCriticalSectionEx)(CRITICAL_SECTION *, U
 static NTSTATUS  (WINAPI *pLdrEnumerateLoadedModules)(void *, void *, void *);
 static NTSTATUS  (WINAPI *pLdrRegisterDllNotification)(ULONG, PLDR_DLL_NOTIFICATION_FUNCTION, void *, void **);
 static NTSTATUS  (WINAPI *pLdrUnregisterDllNotification)(void *);
+static VOID*     (WINAPI *pRtlEncodePointer)(void *);
+static VOID*     (WINAPI *pRtlDecodePointer)(void *);
+static VOID*     (WINAPI *pRtlEncodeSystemPointer)(void *);
+static VOID*     (WINAPI *pRtlDecodeSystemPointer)(void *);
+static NTSTATUS  (WINAPI *pNtQueryInformationProcess)(HANDLE, PROCESSINFOCLASS, PVOID, ULONG, PULONG);

 static HMODULE hkernel32 = 0;
 static BOOL      (WINAPI *pIsWow64Process)(HANDLE, PBOOL);
@@ -122,6 +127,11 @@ static void InitFunctionPtrs(void)
         pLdrEnumerateLoadedModules = (void *)GetProcAddress(hntdll, "LdrEnumerateLoadedModules");
         pLdrRegisterDllNotification = (void *)GetProcAddress(hntdll, "LdrRegisterDllNotification");
         pLdrUnregisterDllNotification = (void *)GetProcAddress(hntdll, "LdrUnregisterDllNotification");
+        pRtlEncodePointer = (void *)GetProcAddress(hntdll, "RtlEncodePointer");
+        pRtlDecodePointer = (void *)GetProcAddress(hntdll, "RtlDecodePointer");
+        pRtlEncodeSystemPointer = (void *)GetProcAddress(hntdll, "RtlEncodeSystemPointer");
+        pRtlDecodeSystemPointer = (void *)GetProcAddress(hntdll, "RtlDecodeSystemPointer");
+        pNtQueryInformationProcess = (void *)GetProcAddress(hntdll, "NtQueryInformationProcess");
     }
     hkernel32 = LoadLibraryA("kernel32.dll");
     ok(hkernel32 != 0, "LoadLibrary failed\n");
@@ -3483,6 +3493,78 @@ static void test_LdrRegisterDllNotification(void)
     pLdrUnregisterDllNotification(cookie);
 }

+static void* our_encode_pointer(void* addr, ULONG cookie)
+{
+    int shift_amount;
+    uintptr_t rot_addr;
+    uintptr_t rot_cookie;
+
+#ifdef _WIN64
+    shift_amount = cookie & 63;
+#define ROTR _rotr64
+#else
+    shift_amount = cookie & 31;
+#define ROTR _rotr
+#endif
+
+    rot_addr = ROTR((uintptr_t)addr, shift_amount);
+    rot_cookie = ROTR((uintptr_t)cookie, shift_amount);
+
+    return (void*)(rot_addr ^ rot_cookie);
+}
+
+static void test_RtlEncodePointer(void)
+{
+    void *addr = (void*)0xdeadbeef0123abcd;
+    void *encoded, *encoded1, *decoded;
+    void *our_encoded;
+    ULONG process_cookie = 0xdeadbeef;
+    ULONG system_cookie = *(ULONG*)(0x7ffe0000 + 0x330); /* user_shared_data.Cookie */
+    NTSTATUS status;
+
+    if (!pRtlEncodePointer || !pRtlDecodePointer)
+    {
+        win_skip("RtlEn/DecodePointer not available\n");
+        return;
+    }
+
+    encoded = pRtlEncodePointer(addr);
+    ok(encoded != addr, "got %p\n", encoded);
+
+    decoded = pRtlDecodePointer(encoded);
+    ok(decoded == addr, "got %p\n", decoded);
+
+    if (!pNtQueryInformationProcess)
+    {
+        win_skip("NtQueryInformationProcess not available\n");
+    }
+    else
+    {
+        status = pNtQueryInformationProcess(GetCurrentProcess(), ProcessCookie, &process_cookie, sizeof(process_cookie), NULL);
+        ok(status == S_OK, "got %08x\n", status);
+        ok(process_cookie != 0xdeadbeef, "got %08x\n", process_cookie);
+
+        our_encoded = our_encode_pointer(addr, process_cookie);
+        todo_wine ok(encoded == our_encoded || broken(encoded == (void*)((uintptr_t)addr ^ (uintptr_t)process_cookie)), "got %p (ours %p)\n", encoded, our_encoded);
+    }
+
+    if (!pRtlEncodeSystemPointer || !pRtlDecodeSystemPointer)
+    {
+        win_skip("RtlEn/DecodeSystemPointer not available\n");
+        return;
+    }
+
+    encoded1 = pRtlEncodeSystemPointer(addr);
+    ok(encoded != addr, "got %p\n", encoded);
+    todo_wine ok(encoded1 != encoded, "got %p\n", encoded1);
+
+    decoded = pRtlDecodeSystemPointer(encoded1);
+    ok(decoded == addr, "got %p\n", decoded);
+
+    our_encoded = our_encode_pointer(addr, system_cookie);
+    todo_wine ok(encoded1 == our_encoded || broken(encoded1 == (void*)((uintptr_t)addr ^ (uintptr_t)system_cookie)), "got %p (ours %p)\n", encoded1, our_encoded);
+}
+
 START_TEST(rtl)
 {
     InitFunctionPtrs();
@@ -3523,4 +3605,5 @@ START_TEST(rtl)
     test_LdrEnumerateLoadedModules();
     test_RtlMakeSelfRelativeSD();
     test_LdrRegisterDllNotification();
+    test_RtlEncodePointer();
 }
--
2.28.0