[PATCH 1/5] [cmd] Add support for tokens= (for /f) (try2)
Ann and Jason Edmeades
jason at edmeades.me.uk
Mon Nov 19 18:36:32 CST 2012
This adds support for the last piece of functionality I am aware
is missing from the for loop processing.
Try2: Ignore NT4s broken parsing
[bug 21047]
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://www.winehq.org/pipermail/wine-patches/attachments/20121120/beca69d6/attachment-0001.html>
-------------- next part --------------
From 04e893536e621f3d31fc5a73806f534bb8e2cd76 Mon Sep 17 00:00:00 2001
From: Jason Edmeades <jason at edmeades.me.uk>
Date: Thu, 8 Nov 2012 21:53:08 +0000
Subject: [PATCH 1/5] [cmd] Add support for tokens= (for /f) (try2)
This adds support for the last piece of functionality I am aware
is missing from the for loop processing.
Try2: Ignore NT4s broken parsing
[bug 21047]
---
programs/cmd/builtins.c | 200 +++++++++++++++++++++++++++---
programs/cmd/tests/test_builtins.cmd | 31 +++++
programs/cmd/tests/test_builtins.cmd.exp | 21 ++++
3 files changed, 237 insertions(+), 15 deletions(-)
diff --git a/programs/cmd/builtins.c b/programs/cmd/builtins.c
index b323668..2e7fdd9 100644
--- a/programs/cmd/builtins.c
+++ b/programs/cmd/builtins.c
@@ -1609,7 +1609,7 @@ static BOOL WCMD_parse_forf_options(WCHAR *options, WCHAR *eol, int *skip,
pos++;
}
tokens[i++] = 0; /* Null terminate the tokens */
- WINE_FIXME("Found tokens as '%s'\n", wine_dbgstr_w(tokens));
+ WINE_TRACE("Found tokens as '%s'\n", wine_dbgstr_w(tokens));
} else {
WINE_WARN("Unexpected data in optionsroot: '%s'\n", wine_dbgstr_w(pos));
@@ -1670,6 +1670,111 @@ static void WCMD_add_dirstowalk(DIRECTORY_STACK *dirsToWalk) {
}
/**************************************************************************
+ * WCMD_for_nexttoken
+ *
+ * Parse the token= line, identifying the next highest number not processed
+ * so far. Count how many tokens are referred (including duplicates) and
+ * optionally return that, plus optionally indicate if the tokens= line
+ * ends in a star.
+ *
+ * Parameters:
+ * lasttoken [I] - Identifies the token index of the last one
+ * returned so far (-1 used for first loop)
+ * tokenstr [I] - The specified tokens= line
+ * firstCmd [O] - Optionally indicate how many tokens are listed
+ * doAll [O] - Optionally indicate if line ends with *
+ * duplicates [O] - Optionally indicate if there is any evidence of
+ * overlaying tokens in the string
+ * Note the caller should keep a running track of duplicates as the tokens
+ * are recursively passed. If any have duplicates, then the * token should
+ * not be honoured.
+ */
+static int WCMD_for_nexttoken(int lasttoken, WCHAR *tokenstr,
+ int *totalfound, BOOL *doall,
+ BOOL *duplicates)
+{
+ WCHAR *pos = tokenstr;
+ int nexttoken = -1;
+
+ if (totalfound) *totalfound = 0;
+ if (doall) *doall = FALSE;
+ if (duplicates) *duplicates = FALSE;
+
+ WINE_TRACE("Find next token after %d in %s was %d\n", lasttoken,
+ wine_dbgstr_w(tokenstr), nexttoken);
+
+ /* Loop through the token string, parsing it. Valid syntax is:
+ token=m or x-y with comma delimiter and optionally * to finish*/
+ while (*pos) {
+ int nextnumber1, nextnumber2 = -1;
+ WCHAR *nextchar;
+
+ /* Get the next number */
+ nextnumber1 = strtoulW(pos, &nextchar, 10);
+
+ /* If it is followed by a minus, its a range, so get the next one as well */
+ if (*nextchar == '-') {
+ nextnumber2 = strtoulW(nextchar+1, &nextchar, 10);
+
+ /* We want to return the lowest number that is higher than lasttoken
+ but only if range is positive */
+ if (nextnumber2 >= nextnumber1 &&
+ lasttoken < nextnumber2) {
+
+ int nextvalue;
+ if (nexttoken == -1) {
+ nextvalue = max(nextnumber1, (lasttoken+1));
+ } else {
+ nextvalue = min(nexttoken, max(nextnumber1, (lasttoken+1)));
+ }
+
+ /* Flag if duplicates identified */
+ if (nexttoken == nextvalue && duplicates) *duplicates = TRUE;
+
+ nexttoken = nextvalue;
+ }
+
+ /* Update the running total for the whole range */
+ if (nextnumber2 >= nextnumber1 && totalfound) {
+ *totalfound = *totalfound + 1 + (nextnumber2 - nextnumber1);
+ }
+
+ } else {
+ if (totalfound) (*totalfound)++;
+
+ /* See if the number found is one we have already seen */
+ if (nextnumber1 == nexttoken && duplicates) *duplicates = TRUE;
+
+ /* We want to return the lowest number that is higher than lasttoken */
+ if (lasttoken < nextnumber1 &&
+ ((nexttoken == -1) || (nextnumber1 < nexttoken))) {
+ nexttoken = nextnumber1;
+ }
+
+ }
+
+ /* Remember if it is followed by a star, and if it is indicate a need to
+ show all tokens, unless a duplicate has been found */
+ if (*nextchar == '*') {
+ if (doall) *doall = TRUE;
+ if (totalfound) (*totalfound)++;
+ }
+
+ /* Step on to the next character */
+ pos = nextchar;
+ if (*pos) pos++;
+ }
+
+ /* Return result */
+ if (nexttoken == -1) nexttoken = lasttoken;
+ WINE_TRACE("Found next token after %d was %d\n", lasttoken, nexttoken);
+ if (totalfound) WINE_TRACE("Found total tokens in total %d\n", *totalfound);
+ if (doall && *doall) WINE_TRACE("Request for all tokens found\n");
+ if (duplicates && *duplicates) WINE_TRACE("Duplicate numbers found\n");
+ return nexttoken;
+}
+
+/**************************************************************************
* WCMD_parse_line
*
* When parsing file or string contents (for /f), once the string to parse
@@ -1688,6 +1793,7 @@ static void WCMD_add_dirstowalk(DIRECTORY_STACK *dirsToWalk) {
* forf_skip [I/O] - How many lines to skip first
* forf_eol [I] - The 'end of line' (comment) character
* forf_delims [I] - The delimiters to use when breaking the string apart
+ * forf_tokens [I] - The tokens to use when breaking the string apart
*/
static void WCMD_parse_line(CMD_LIST *cmdStart,
const WCHAR *firstCmd,
@@ -1697,11 +1803,17 @@ static void WCMD_parse_line(CMD_LIST *cmdStart,
BOOL *doExecuted,
int *forf_skip,
WCHAR forf_eol,
- WCHAR *forf_delims) {
+ WCHAR *forf_delims,
+ WCHAR *forf_tokens) {
- WCHAR *parm, *where;
+ WCHAR *parm;
FOR_CONTEXT oldcontext;
- int varidx;
+ int varidx, varoffset;
+ int nexttoken, lasttoken = -1;
+ BOOL starfound = FALSE;
+ BOOL thisduplicate = FALSE;
+ BOOL anyduplicates = FALSE;
+ int totalfound;
/* Skip lines if requested */
if (*forf_skip) {
@@ -1712,23 +1824,81 @@ static void WCMD_parse_line(CMD_LIST *cmdStart,
/* Save away any existing for variable context (e.g. nested for loops) */
oldcontext = forloopcontext;
- /* Extract the parameter */
- parm = WCMD_parameter_with_delims(buffer, 0, &where, FALSE, FALSE, forf_delims);
- WINE_TRACE("Parsed parameter: %s from %s\n", wine_dbgstr_w(parm),
- wine_dbgstr_w(buffer));
-
- /* FIXME: Use tokens= line to populate forloopcontext */
+ /* Extract the parameters based on the tokens= value (There will always
+ be some value, as if it is not supplied, it defaults to tokens=1).
+ Rough logic:
+ Count how many tokens are named in the line, identify the lowest
+ Empty (set to null terminated string) that number of named variables
+ While lasttoken != nextlowest
+ %letter = parameter number 'nextlowest'
+ letter++ (if >26 or >52 abort)
+ Go through token= string finding next lowest number
+ If token ends in * set %letter = raw position of token(nextnumber+1)
+ */
+ lasttoken = -1;
+ nexttoken = WCMD_for_nexttoken(lasttoken, forf_tokens, &totalfound,
+ NULL, &thisduplicate);
varidx = FOR_VAR_IDX(variable);
- if (varidx >=0) forloopcontext.variable[varidx] = heap_strdupW(parm);
- if (where && where[0] != forf_eol) {
+ /* Empty out variables */
+ for (varoffset=0;
+ varidx >= 0 && varoffset<totalfound && ((varidx+varoffset)%26);
+ varoffset++) {
+ forloopcontext.variable[varidx + varoffset] = (WCHAR *)nullW;
+ /* Stop if we walk beyond z or Z */
+ if (((varidx+varoffset) % 26) == 0) break;
+ }
+
+ /* Loop extracting the tokens */
+ varoffset = 0;
+ WINE_TRACE("Parsing buffer into tokens: '%s'\n", wine_dbgstr_w(buffer));
+ while (varidx >= 0 && (nexttoken > lasttoken)) {
+ anyduplicates |= thisduplicate;
+
+ /* Extract the token number requested and set into the next variable context */
+ parm = WCMD_parameter_with_delims(buffer, (nexttoken-1), NULL, FALSE, FALSE, forf_delims);
+ WINE_TRACE("Parsed token %d(%d) as parameter %s\n", nexttoken,
+ varidx + varoffset, wine_dbgstr_w(parm));
+ if (varidx >=0) {
+ forloopcontext.variable[varidx + varoffset] = heap_strdupW(parm);
+ varoffset++;
+ if (((varidx + varoffset) %26) == 0) break;
+ }
+
+ /* Find the next token */
+ lasttoken = nexttoken;
+ nexttoken = WCMD_for_nexttoken(lasttoken, forf_tokens, NULL,
+ &starfound, &thisduplicate);
+ }
+
+ /* If all the rest of the tokens were requested, and there is still space in
+ the variable range, write them now */
+ if (!anyduplicates && starfound && varidx >= 0 && ((varidx+varoffset) % 26)) {
+ nexttoken++;
+ WCMD_parameter_with_delims(buffer, (nexttoken-1), &parm, FALSE, FALSE, forf_delims);
+ WINE_TRACE("Parsed allremaining tokens (%d) as parameter %s\n",
+ varidx + varoffset, wine_dbgstr_w(parm));
+ forloopcontext.variable[varidx + varoffset] = heap_strdupW(parm);
+ }
+
+ /* Execute the body of the foor loop with these values */
+ if (forloopcontext.variable[varidx] && forloopcontext.variable[varidx][0] != forf_eol) {
CMD_LIST *thisCmdStart = cmdStart;
*doExecuted = TRUE;
WCMD_part_execute(&thisCmdStart, firstCmd, FALSE, TRUE);
*cmdEnd = thisCmdStart;
}
- if (varidx >=0) heap_free(forloopcontext.variable[varidx]);
+ /* Free the duplicated strings, and restore the context */
+ if (varidx >=0) {
+ int i;
+ for (i=varidx; i<MAX_FOR_VARIABLES; i++) {
+ if ((forloopcontext.variable[i] != oldcontext.variable[i]) &&
+ (forloopcontext.variable[i] != nullW)) {
+ heap_free(forloopcontext.variable[i]);
+ }
+ }
+ }
/* Restore the original for variable contextx */
forloopcontext = oldcontext;
@@ -2096,7 +2266,7 @@ void WCMD_for (WCHAR *p, CMD_LIST **cmdList) {
/* Read line by line until end of file */
while (WCMD_fgets(buffer, sizeof(buffer)/sizeof(WCHAR), input)) {
WCMD_parse_line(cmdStart, firstCmd, &cmdEnd, variable[1], buffer, &doExecuted,
- &forf_skip, forf_eol, forf_delims);
+ &forf_skip, forf_eol, forf_delims, forf_tokens);
buffer[0] = 0;
}
CloseHandle (input);
@@ -2124,7 +2294,7 @@ void WCMD_for (WCHAR *p, CMD_LIST **cmdList) {
/* Copy the item away from the global buffer used by WCMD_parameter */
strcpyW(buffer, itemStart);
WCMD_parse_line(cmdStart, firstCmd, &cmdEnd, variable[1], buffer, &doExecuted,
- &forf_skip, forf_eol, forf_delims);
+ &forf_skip, forf_eol, forf_delims, forf_tokens);
/* Only one string can be supplied in the whole set, abort future set processing */
thisSet = NULL;
diff --git a/programs/cmd/tests/test_builtins.cmd b/programs/cmd/tests/test_builtins.cmd
index e7b8516..a1a3c4a 100644
--- a/programs/cmd/tests/test_builtins.cmd
+++ b/programs/cmd/tests/test_builtins.cmd
@@ -1194,6 +1194,37 @@ for /f "skip=02" %%i in (foo) do echo %%i
for /f "skip=0x2" %%i in (foo) do echo %%i
for /f "skip=1" %%i in ("skipme") do echo %%i > output_file
if not exist output_file (echo no output) else (del output_file)
+echo ------ tokens= option
+rem Basic
+for /f %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m o=%%o
+for /f "tokens=2" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m o=%%o
+for /f "tokens=1,3,5-7" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m o=%%o
+rem Show * means the rest
+for /f "tokens=1,5*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m o=%%o
+for /f "tokens=6,9*" %%i in ("a b c d e f g h i j k l m n o p q r s t u v w x y z") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m o=%%o
+rem Show * means the rest (not tokenized and rebuilt)
+for /f "tokens=6,9*" %%i in ("a b c d e f g h i j k l m n;;== o p q r s t u v w x y z") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m o=%%o
+rem Order is irrelevant
+for /f "tokens=1,2,3*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+for /f "tokens=3,2,1*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+rem Duplicates are ignored
+for /f "tokens=1,2,1*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+rem Large tokens are allowed
+for /f "tokens=25,1,5*" %%i in ("a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D E F G H I J K L M N O P Q R S T U V W X Y Z") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+rem Show tokens blanked in advance regardless of uniqueness of requested tokens
+for /f "tokens=1,1,1,2*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+for /f "tokens=1-2,1-2,1-2" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+rem Show No wrapping from z to A BUT wrapping sort of occurs Z to a occurs
+for /f "tokens=1-20" %%u in ("a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D E F G H I J K L M N O P Q R S T U V W X Y Z") do echo u=%%u v=%%v w=%%w x=%%x y=%%y z=%%z A=%%A a=%%a
+for /f "tokens=1-20" %%U in ("a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D E F G H I J K L M N O P Q R S T U V W X Y Z") do echo U=%%U V=%%V W=%%W X=%%X Y=%%Y Z=%%Z A=%%A a=%%a
+rem Show negative ranges have no effect
+for /f "tokens=1-3,5" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m o=%%o
+for /f "tokens=3-1,5" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m o=%%o
+rem Show duplicates stop * from working
+for /f "tokens=1,2,3*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+for /f "tokens=1,1,3*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+for /f "tokens=2,2,3*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+for /f "tokens=3,2,3*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
cd ..
rd /s/q foobar
diff --git a/programs/cmd/tests/test_builtins.cmd.exp b/programs/cmd/tests/test_builtins.cmd.exp
index 1d42a96..62506fc 100644
--- a/programs/cmd/tests/test_builtins.cmd.exp
+++ b/programs/cmd/tests/test_builtins.cmd.exp
@@ -842,6 +842,27 @@ no output
c
c
no output
+------ tokens= option
+h=%h i=a j=%j k=%k l=%l m=%m o=%o
+h=%h i=b j=%j k=%k l=%l m=%m o=%o
+h=%h i=a j=c k=e l=f m=g o=%o
+h=%h i=a j=e k=f g l=%l m=%m o=%o
+h=%h i=f j=i k=j k l m n o p q r s t u v w x y z l=%l m=%m o=%o
+h=%h i=f j=i k=j k l m n;;== o p q r s t u v w x y z l=%l m=%m o=%o
+h=%h i=a j=b k=c l=d e f g m=%m n=%n o=%o
+h=%h i=a j=b k=c l=d e f g m=%m n=%n o=%o
+h=%h i=a j=b k= l= m=%m n=%n o=%o
+h=%h i=a j=e k=y l=z A B C D E F G H I J K L M N O P Q R S T U V W X Y Z m=%m n=%n o=%o
+h=%h i=a j=b k= l= m= n=%n o=%o
+h=%h i=a j=b k= l= m= n= o=%o
+u=a v=b w=c x=d y=e z=f A=%A a=%a
+ at todo_wine@U=a V=b W=c X=d Y=e Z=f A=%A a=m
+h=%h i=a j=b k=c l=e m=%m o=%o at or_broken@h=%h i=a j=b k=c l=e m= o=%o
+h=%h i=e j=%j k=%k l=%l m=%m o=%o
+h=%h i=a j=b k=c l=d e f g m=%m n=%n o=%o at or_broken@h=%h i=a j=b k=c l=d e f g m= n=%n o=%o
+h=%h i=a j=c k= l= m=%m n=%n o=%o at or_broken@h=%h i=a j=c k= l= m= n=%n o=%o
+h=%h i=b j=c k= l= m=%m n=%n o=%o at or_broken@h=%h i=b j=c k= l= m= n=%n o=%o
+h=%h i=b j=c k= l= m=%m n=%n o=%o at or_broken@h=%h i=b j=c k= l= m= n=%n o=%o
------------ Testing del /a ------------
not-r.test not found after delete, good
r.test found before delete, good
--
1.7.9.5
More information about the wine-patches
mailing list