PATCH: better VT_PTR / NULL handling in typelib marshaller

[Marcus Meissner]

Hi,

The typelib marshaller was not handling NULL ptr arguments correctly.

when marshalling x([out] VARIANT*); the possibility of NULL as
argument was not taken care of.

This is solved by sending the ptr cookie back and forth between
the processes now.

InstallShield proven[tm].

Ciao, Marcus

Changelog:
	Handle VT_PTR / NULL marshalling correctly.

Index: dlls/oleaut32/tmarshal.c
===================================================================
RCS file: /home/wine/wine/dlls/oleaut32/tmarshal.c,v
retrieving revision 1.45
diff -u -r1.45 tmarshal.c
--- dlls/oleaut32/tmarshal.c	25 Mar 2005 16:38:37 -0000	1.45
+++ dlls/oleaut32/tmarshal.c	25 Mar 2005 23:25:37 -0000
@@ -583,12 +583,12 @@
 	DWORD cookie;
 
 	if (debugout) TRACE_(olerelay)("*");
-	if (writeit) {
-	    cookie = *arg ? 0x42424242 : 0;
-	    hres = xbuf_add(buf,(LPBYTE)&cookie,sizeof(cookie));
-	    if (hres)
-		return hres;
-	}
+	/* Write always, so the other side knows when it gets a NULL pointer.
+	 */
+	cookie = *arg ? 0x42424242 : 0;
+	hres = xbuf_add(buf,(LPBYTE)&cookie,sizeof(cookie));
+	if (hres)
+	    return hres;
 	if (!*arg) {
 	    if (debugout) TRACE_(olerelay)("NULL");
 	    return S_OK;
@@ -953,24 +953,26 @@
 	}
 	case VT_PTR: {
 	    DWORD	cookie;
-	    BOOL	derefhere = 0;
+	    BOOL        derefhere = 0;
 
 	    derefhere = (tdesc->u.lptdesc->vt != VT_USERDEFINED);
-
-	    if (readit) {
-		hres = xbuf_get(buf,(LPBYTE)&cookie,sizeof(cookie));
-		if (hres) {
-		    ERR("Failed to load pointer cookie.\n");
-		    return hres;
-		}
-		if (cookie != 0x42424242) {
-		    if (debugout) TRACE_(olerelay)("NULL");
-		    *arg = 0;
-		    return S_OK;
-		}
-		if (debugout) TRACE_(olerelay)("*");
+	    /* read it in all cases, we need to know if we have 
+	     * NULL pointer or not.
+	     */
+	    hres = xbuf_get(buf,(LPBYTE)&cookie,sizeof(cookie));
+	    if (hres) {
+		ERR("Failed to load pointer cookie.\n");
+		return hres;
+	    }
+	    if (cookie != 0x42424242) {
+		/* we read a NULL ptr from the remote side */
+		if (debugout) TRACE_(olerelay)("NULL");
+		*arg = 0;
+		return S_OK;
 	    }
+	    if (debugout) TRACE_(olerelay)("*");
 	    if (alloc) {
+		/* Allocate space for the referenced struct */
 		if (derefhere)
 		    *arg=(DWORD)HeapAlloc(GetProcessHeap(),HEAP_ZERO_MEMORY,_xsize(tdesc->u.lptdesc));
 	    }
@@ -1325,8 +1327,8 @@
 		if (i+1<nrofnames && names[i+1])
 		    TRACE_(olerelay)("%s=",relaystr(names[i+1]));
 	    }
-	    /* No need to marshal other data than FIN */
-	    if (!(elem->u.paramdesc.wParamFlags & PARAMFLAG_FIN)) {
+	    /* No need to marshal other data than FIN and any VT_PTR. */
+	    if (!(elem->u.paramdesc.wParamFlags & PARAMFLAG_FIN) && (elem->tdesc.vt != VT_PTR)) {
 		xargs+=_argsize(elem->tdesc.vt);
 		if (relaydeb) TRACE_(olerelay)("[out]");
 		continue;
@@ -1422,8 +1424,8 @@
 		if (i) TRACE_(olerelay)(",");
 		if (i+1<nrofnames && names[i+1]) TRACE_(olerelay)("%s=",relaystr(names[i+1]));
 	    }
-	    /* No need to marshal other data than FOUT I think */
-	    if (!(elem->u.paramdesc.wParamFlags & PARAMFLAG_FOUT)) {
+	    /* No need to marshal other data than FOUT and any VT_PTR */
+	    if (!(elem->u.paramdesc.wParamFlags & PARAMFLAG_FOUT) && (elem->tdesc.vt != VT_PTR)) {
 		xargs += _argsize(elem->tdesc.vt);
 		if (relaydeb) TRACE_(olerelay)("[in]");
 		continue;
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 189 bytes
Desc: not available
Url : http://www.winehq.org/pipermail/wine-patches/attachments/20050326/071960d9/attachment.pgp