cmd: Add support for tokens= (for /f).

programs/cmd/builtins.c

@@ -1609,7 +1609,7 @@ static BOOL WCMD_parse_forf_options(WCHAR *options, WCHAR *eol, int *skip,
         pos++;
       }
       tokens[i++] = 0; /* Null terminate the tokens */
-      WINE_FIXME("Found tokens as '%s'\n", wine_dbgstr_w(tokens));
+      WINE_TRACE("Found tokens as '%s'\n", wine_dbgstr_w(tokens));
 
     } else {
       WINE_WARN("Unexpected data in optionsroot: '%s'\n", wine_dbgstr_w(pos));
@@ -1669,6 +1669,111 @@ static void WCMD_add_dirstowalk(DIRECTORY_STACK *dirsToWalk) {
   }
 }
 
+/**************************************************************************
+ * WCMD_for_nexttoken
+ *
+ * Parse the token= line, identifying the next highest number not processed
+ * so far. Count how many tokens are referred (including duplicates) and
+ * optionally return that, plus optionally indicate if the tokens= line
+ * ends in a star.
+ *
+ * Parameters:
+ *  lasttoken    [I]    - Identifies the token index of the last one
+ *                           returned so far (-1 used for first loop)
+ *  tokenstr     [I]    - The specified tokens= line
+ *  firstCmd     [O]    - Optionally indicate how many tokens are listed
+ *  doAll        [O]    - Optionally indicate if line ends with *
+ *  duplicates   [O]    - Optionally indicate if there is any evidence of
+ *                           overlaying tokens in the string
+ * Note the caller should keep a running track of duplicates as the tokens
+ * are recursively passed. If any have duplicates, then the * token should
+ * not be honoured.
+ */
+static int WCMD_for_nexttoken(int lasttoken, WCHAR *tokenstr,
+                              int *totalfound, BOOL *doall,
+                              BOOL *duplicates)
+{
+  WCHAR *pos = tokenstr;
+  int    nexttoken = -1;
+
+  if (totalfound) *totalfound = 0;
+  if (doall) *doall = FALSE;
+  if (duplicates) *duplicates = FALSE;
+
+  WINE_TRACE("Find next token after %d in %s was %d\n", lasttoken,
+             wine_dbgstr_w(tokenstr), nexttoken);
+
+  /* Loop through the token string, parsing it. Valid syntax is:
+     token=m or x-y with comma delimiter and optionally * to finish*/
+  while (*pos) {
+    int nextnumber1, nextnumber2 = -1;
+    WCHAR *nextchar;
+
+    /* Get the next number */
+    nextnumber1 = strtoulW(pos, &nextchar, 10);
+
+    /* If it is followed by a minus, its a range, so get the next one as well */
+    if (*nextchar == '-') {
+      nextnumber2 = strtoulW(nextchar+1, &nextchar, 10);
+
+      /* We want to return the lowest number that is higher than lasttoken
+         but only if range is positive                                     */
+      if (nextnumber2 >= nextnumber1 &&
+          lasttoken < nextnumber2) {
+
+        int nextvalue;
+        if (nexttoken == -1) {
+          nextvalue = max(nextnumber1, (lasttoken+1));
+        } else {
+          nextvalue = min(nexttoken, max(nextnumber1, (lasttoken+1)));
+        }
+
+        /* Flag if duplicates identified */
+        if (nexttoken == nextvalue && duplicates) *duplicates = TRUE;
+
+        nexttoken = nextvalue;
+      }
+
+      /* Update the running total for the whole range */
+      if (nextnumber2 >= nextnumber1 && totalfound) {
+        *totalfound = *totalfound + 1 + (nextnumber2 - nextnumber1);
+      }
+
+    } else {
+      if (totalfound) (*totalfound)++;
+
+      /* See if the number found is one we have already seen */
+      if (nextnumber1 == nexttoken && duplicates) *duplicates = TRUE;
+
+      /* We want to return the lowest number that is higher than lasttoken */
+      if (lasttoken < nextnumber1 &&
+         ((nexttoken == -1) || (nextnumber1 < nexttoken))) {
+        nexttoken = nextnumber1;
+      }
+
+    }
+
+    /* Remember if it is followed by a star, and if it is indicate a need to
+       show all tokens, unless a duplicate has been found                    */
+    if (*nextchar == '*') {
+      if (doall) *doall = TRUE;
+      if (totalfound) (*totalfound)++;
+    }
+
+    /* Step on to the next character */
+    pos = nextchar;
+    if (*pos) pos++;
+  }
+
+  /* Return result */
+  if (nexttoken == -1) nexttoken = lasttoken;
+  WINE_TRACE("Found next token after %d was %d\n", lasttoken, nexttoken);
+  if (totalfound) WINE_TRACE("Found total tokens in total %d\n", *totalfound);
+  if (doall && *doall) WINE_TRACE("Request for all tokens found\n");
+  if (duplicates && *duplicates) WINE_TRACE("Duplicate numbers found\n");
+  return nexttoken;
+}
+
 /**************************************************************************
  * WCMD_parse_line
  *
@@ -1688,6 +1793,7 @@ static void WCMD_add_dirstowalk(DIRECTORY_STACK *dirsToWalk) {
  *  forf_skip    [I/O]  - How many lines to skip first
  *  forf_eol     [I]    - The 'end of line' (comment) character
  *  forf_delims  [I]    - The delimiters to use when breaking the string apart
+ *  forf_tokens  [I]    - The tokens to use when breaking the string apart
  */
 static void WCMD_parse_line(CMD_LIST    *cmdStart,
                             const WCHAR *firstCmd,
@@ -1697,11 +1803,17 @@ static void WCMD_parse_line(CMD_LIST    *cmdStart,
                             BOOL        *doExecuted,
                             int         *forf_skip,
                             WCHAR        forf_eol,
-                            WCHAR       *forf_delims) {
+                            WCHAR       *forf_delims,
+                            WCHAR       *forf_tokens) {
 
-  WCHAR *parm, *where;
+  WCHAR *parm;
   FOR_CONTEXT oldcontext;
-  int varidx;
+  int varidx, varoffset;
+  int nexttoken, lasttoken = -1;
+  BOOL starfound = FALSE;
+  BOOL thisduplicate = FALSE;
+  BOOL anyduplicates = FALSE;
+  int  totalfound;
 
   /* Skip lines if requested */
   if (*forf_skip) {
@@ -1712,23 +1824,81 @@ static void WCMD_parse_line(CMD_LIST    *cmdStart,
   /* Save away any existing for variable context (e.g. nested for loops) */
   oldcontext = forloopcontext;
 
-  /* Extract the parameter */
+  /* Extract the parameters based on the tokens= value (There will always
-  parm = WCMD_parameter_with_delims(buffer, 0, &where, FALSE, FALSE, forf_delims);
+     be some value, as if it is not supplied, it defaults to tokens=1).
-  WINE_TRACE("Parsed parameter: %s from %s\n", wine_dbgstr_w(parm),
+     Rough logic:
-             wine_dbgstr_w(buffer));
+     Count how many tokens are named in the line, identify the lowest
-
+     Empty (set to null terminated string) that number of named variables
-  /* FIXME: Use tokens= line to populate forloopcontext */
+     While lasttoken != nextlowest
+       %letter = parameter number 'nextlowest'
+       letter++ (if >26 or >52 abort)
+       Go through token= string finding next lowest number
+     If token ends in * set %letter = raw position of token(nextnumber+1)
+   */
+  lasttoken = -1;
+  nexttoken = WCMD_for_nexttoken(lasttoken, forf_tokens, &totalfound,
+                                 NULL, &thisduplicate);
   varidx = FOR_VAR_IDX(variable);
-  if (varidx >=0) forloopcontext.variable[varidx] = heap_strdupW(parm);
 
-  if (where && where[0] != forf_eol) {
+  /* Empty out variables */
+  for (varoffset=0;
+       varidx >= 0 && varoffset<totalfound && ((varidx+varoffset)%26);
+       varoffset++) {
+    forloopcontext.variable[varidx + varoffset] = (WCHAR *)nullW;
+    /* Stop if we walk beyond z or Z */
+    if (((varidx+varoffset) % 26) == 0) break;
+  }
+
+  /* Loop extracting the tokens */
+  varoffset = 0;
+  WINE_TRACE("Parsing buffer into tokens: '%s'\n", wine_dbgstr_w(buffer));
+  while (varidx >= 0 && (nexttoken > lasttoken)) {
+    anyduplicates |= thisduplicate;
+
+    /* Extract the token number requested and set into the next variable context */
+    parm = WCMD_parameter_with_delims(buffer, (nexttoken-1), NULL, FALSE, FALSE, forf_delims);
+    WINE_TRACE("Parsed token %d(%d) as parameter %s\n", nexttoken,
+               varidx + varoffset, wine_dbgstr_w(parm));
+    if (varidx >=0) {
+      forloopcontext.variable[varidx + varoffset] = heap_strdupW(parm);
+      varoffset++;
+      if (((varidx + varoffset) %26) == 0) break;
+    }
+
+    /* Find the next token */
+    lasttoken = nexttoken;
+    nexttoken = WCMD_for_nexttoken(lasttoken, forf_tokens, NULL,
+                                   &starfound, &thisduplicate);
+  }
+
+  /* If all the rest of the tokens were requested, and there is still space in
+     the variable range, write them now                                        */
+  if (!anyduplicates && starfound && varidx >= 0 && ((varidx+varoffset) % 26)) {
+    nexttoken++;
+    WCMD_parameter_with_delims(buffer, (nexttoken-1), &parm, FALSE, FALSE, forf_delims);
+    WINE_TRACE("Parsed allremaining tokens (%d) as parameter %s\n",
+               varidx + varoffset, wine_dbgstr_w(parm));
+    forloopcontext.variable[varidx + varoffset] = heap_strdupW(parm);
+  }
+
+  /* Execute the body of the foor loop with these values */
+  if (forloopcontext.variable[varidx] && forloopcontext.variable[varidx][0] != forf_eol) {
     CMD_LIST *thisCmdStart = cmdStart;
     *doExecuted = TRUE;
     WCMD_part_execute(&thisCmdStart, firstCmd, FALSE, TRUE);
     *cmdEnd = thisCmdStart;
   }
 
-  if (varidx >=0) heap_free(forloopcontext.variable[varidx]);
+  /* Free the duplicated strings, and restore the context */
+  if (varidx >=0) {
+    int i;
+    for (i=varidx; i<MAX_FOR_VARIABLES; i++) {
+      if ((forloopcontext.variable[i] != oldcontext.variable[i]) &&
+          (forloopcontext.variable[i] != nullW)) {
+        heap_free(forloopcontext.variable[i]);
+      }
+    }
+  }
 
   /* Restore the original for variable contextx */
   forloopcontext = oldcontext;
@@ -2096,7 +2266,7 @@ void WCMD_for (WCHAR *p, CMD_LIST **cmdList) {
               /* Read line by line until end of file */
               while (WCMD_fgets(buffer, sizeof(buffer)/sizeof(WCHAR), input)) {
                 WCMD_parse_line(cmdStart, firstCmd, &cmdEnd, variable[1], buffer, &doExecuted,
-                                &forf_skip, forf_eol, forf_delims);
+                                &forf_skip, forf_eol, forf_delims, forf_tokens);
                 buffer[0] = 0;
               }
               CloseHandle (input);
@@ -2124,7 +2294,7 @@ void WCMD_for (WCHAR *p, CMD_LIST **cmdList) {
           /* Copy the item away from the global buffer used by WCMD_parameter */
           strcpyW(buffer, itemStart);
           WCMD_parse_line(cmdStart, firstCmd, &cmdEnd, variable[1], buffer, &doExecuted,
-                            &forf_skip, forf_eol, forf_delims);
+                            &forf_skip, forf_eol, forf_delims, forf_tokens);
 
           /* Only one string can be supplied in the whole set, abort future set processing */
           thisSet = NULL;

programs/cmd/tests/test_builtins.cmd

@@ -1194,6 +1194,37 @@ for /f "skip=02" %%i in (foo) do echo %%i
 for /f "skip=0x2" %%i in (foo) do echo %%i
 for /f "skip=1" %%i in ("skipme") do echo %%i > output_file
 if not exist output_file (echo no output) else (del output_file)
+echo ------ tokens= option
+rem Basic
+for /f %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m o=%%o
+for /f "tokens=2" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m o=%%o
+for /f "tokens=1,3,5-7" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m o=%%o
+rem Show * means the rest
+for /f "tokens=1,5*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m o=%%o
+for /f "tokens=6,9*" %%i in ("a b c d e f g h i j k l m n o p q r s t u v w x y z") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m o=%%o
+rem Show * means the rest (not tokenized and rebuilt)
+for /f "tokens=6,9*" %%i in ("a b c d e f g h i j k l m  n;;==  o p q r s t u v w x y z") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m o=%%o
+rem Order is irrelevant
+for /f "tokens=1,2,3*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+for /f "tokens=3,2,1*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+rem Duplicates are ignored
+for /f "tokens=1,2,1*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+rem Large tokens are allowed
+for /f "tokens=25,1,5*" %%i in ("a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D E F G H I J K L M N O P Q R S T U V W X Y Z") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+rem Show tokens blanked in advance regardless of uniqueness of requested tokens
+for /f "tokens=1,1,1,2*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+for /f "tokens=1-2,1-2,1-2" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+rem Show No wrapping from z to A BUT wrapping sort of occurs Z to a occurs
+for /f "tokens=1-20" %%u in ("a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D E F G H I J K L M N O P Q R S T U V W X Y Z") do echo u=%%u v=%%v w=%%w x=%%x y=%%y z=%%z A=%%A a=%%a
+for /f "tokens=1-20" %%U in ("a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D E F G H I J K L M N O P Q R S T U V W X Y Z") do echo U=%%U V=%%V W=%%W X=%%X Y=%%Y Z=%%Z A=%%A a=%%a
+rem Show negative ranges have no effect
+for /f "tokens=1-3,5" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m o=%%o
+for /f "tokens=3-1,5" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m o=%%o
+rem Show duplicates stop * from working
+for /f "tokens=1,2,3*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+for /f "tokens=1,1,3*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+for /f "tokens=2,2,3*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+for /f "tokens=3,2,3*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
 cd ..
 rd /s/q foobar
 

programs/cmd/tests/test_builtins.cmd.exp

@@ -842,6 +842,27 @@ no output
 c
 c
 no output
+------ tokens= option
+h=%h i=a j=%j k=%k l=%l m=%m o=%o
+h=%h i=b j=%j k=%k l=%l m=%m o=%o
+h=%h i=a j=c k=e l=f m=g o=%o
+h=%h i=a j=e k=f g l=%l m=%m o=%o
+h=%h i=f j=i k=j k l m n o p q r s t u v w x y z l=%l m=%m o=%o
+h=%h i=f j=i k=j k l m  n;;==  o p q r s t u v w x y z l=%l m=%m o=%o
+h=%h i=a j=b k=c l=d e f g m=%m n=%n o=%o
+h=%h i=a j=b k=c l=d e f g m=%m n=%n o=%o
+h=%h i=a j=b k= l= m=%m n=%n o=%o
+h=%h i=a j=e k=y l=z A B C D E F G H I J K L M N O P Q R S T U V W X Y Z m=%m n=%n o=%o
+h=%h i=a j=b k= l= m= n=%n o=%o
+h=%h i=a j=b k= l= m= n= o=%o
+u=a v=b w=c x=d y=e z=f A=%A a=%a
+@todo_wine@U=a V=b W=c X=d Y=e Z=f A=%A a=m
+h=%h i=a j=b k=c l=e m=%m o=%o@or_broken@h=%h i=a j=b k=c l=e m= o=%o
+h=%h i=e j=%j k=%k l=%l m=%m o=%o
+h=%h i=a j=b k=c l=d e f g m=%m n=%n o=%o@or_broken@h=%h i=a j=b k=c l=d e f g m= n=%n o=%o
+h=%h i=a j=c k= l= m=%m n=%n o=%o@or_broken@h=%h i=a j=c k= l= m= n=%n o=%o
+h=%h i=b j=c k= l= m=%m n=%n o=%o@or_broken@h=%h i=b j=c k= l= m= n=%n o=%o
+h=%h i=b j=c k= l= m=%m n=%n o=%o@or_broken@h=%h i=b j=c k= l= m= n=%n o=%o
 ------------ Testing del /a ------------
 not-r.test not found after delete, good
 r.test found before delete, good