[PATCH resend 1/3] ntdll: Use Vista's algorithm for RtlEncode/DecodePointer.

[Myah Caron]

The algorithm is described here:
https://web.archive.org/web/20100603042315/http://blogs.msdn.com/b/michael_howard/archive/2006/08/16/702707.aspx

It still appears to be the same algorithm used in Windows 10.

Signed-off-by: Myah Caron <qsniyg at protonmail.com>
---
 dlls/ntdll/rtl.c | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/dlls/ntdll/rtl.c b/dlls/ntdll/rtl.c
index cf21c980a45..c878035a044 100644
--- a/dlls/ntdll/rtl.c
+++ b/dlls/ntdll/rtl.c
@@ -1562,19 +1562,49 @@ static DWORD_PTR get_pointer_obfuscator( void )
     return pointer_obfuscator;
 }

+/***********************************************************************
+ * rotl_ptr (internal)
+ */
+#ifdef _WIN64
+#define ROT_BITS 64
+#else
+#define ROT_BITS 32
+#endif
+
+static DWORD_PTR rotl_ptr( DWORD_PTR num, int shift )
+{
+    shift &= ROT_BITS - 1;
+    return (num << shift) | (num >> (ROT_BITS-shift));
+}
+
+static DWORD_PTR rotr_ptr( DWORD_PTR num, int shift )
+{
+    shift &= ROT_BITS - 1;
+    return (num >> shift) | (num << (ROT_BITS-shift));
+}
+
+#undef ROT_BITS
+
 /*************************************************************************
  * RtlEncodePointer   [NTDLL.@]
  */
 PVOID WINAPI RtlEncodePointer( PVOID ptr )
 {
+
     DWORD_PTR ptrval = (DWORD_PTR) ptr;
-    return (PVOID)(ptrval ^ get_pointer_obfuscator());
+    DWORD_PTR cookie = get_pointer_obfuscator();
+
+    ptrval = (ptrval ^ cookie);
+    return (PVOID)rotr_ptr(ptrval, cookie);
 }

 PVOID WINAPI RtlDecodePointer( PVOID ptr )
 {
     DWORD_PTR ptrval = (DWORD_PTR) ptr;
-    return (PVOID)(ptrval ^ get_pointer_obfuscator());
+    DWORD_PTR cookie = get_pointer_obfuscator();
+
+    ptrval = rotl_ptr(ptrval, cookie);
+    return (PVOID)(ptrval ^ cookie);
 }

 /*************************************************************************
--
2.30.1