Efficiently extract tokens from strings

Question

I have the current C code:

#include <string.h> // strdup
#include <stdio.h> // printf
#include <stdlib.h> // free

int main(int argc, char *argv[])
{
  const char isostr[] = "\\ISO 2022 IR 13\\ISO 2022 IR 87";

  char *query = strdup( isostr );
  char *token;
  char *str1;
  char *saveptr1;
  const char delim[] = "\\";

  for (str1 = query; ; str1 = NULL)
    {
    token = strtok_r(str1, delim, &saveptr1);
    if (token == NULL)
      break;
    printf(" --> %s\n", token);
    }
  free( query );

  return 0;
}

However it is returning:

 --> ISO 2022 IR 13
 --> ISO 2022 IR 87

while I would need to return:

 --> [null/empty]
 --> ISO 2022 IR 13
 --> ISO 2022 IR 87

strtok_r does not seems to make any difference in between the string "AA\BB\CC" and "AA\BB\CC\" or "\AA\BB\\CC".

Take a look here: http://stackoverflow.com/questions/18827546/obtaining-zero-length-string-from-strtok — Ingo Leonhardt, Sep 16 '13 at 14:22

score 2 · Accepted Answer · edited Sep 16 '13 at 14:48

2

If the delimiter is only 1 character, you can use strchr to search for it instead of tokenizing. strtok takes all the delimiter characters as a separator.

str1 = query;
while (1)
{
    char * sep = strchr(str1, '\\')
    if (sep != NULL) *sep = 0;
    printf(" --> %s\n", str1);
    if (sep == NULL) break;
    str1 = sep + 1;
}

edited Sep 16 '13 at 14:48

Sergey Kalinichenko

714,442
84
1,110
1,523

answered Sep 16 '13 at 14:27

user1969104

2,340
14
15

@dasblinkenlight I think you edited the wrong answer. – interjay Sep 16 '13 at 14:45
@interjay Ah, I did! I'm sorry! – Sergey Kalinichenko Sep 16 '13 at 14:48

Sergey Kalinichenko · Answer 2 · 2013-09-16T14:48:58.843

If the separator is always a single character, you can do it like this:

char isostr[] = "\\ISO 2022 IR 13\\ISO 2022 IR 87";
char *p = isostr;
for (;;) {
    char *next = strchr(p, '\\');
    int len = next ? next-p : strlen(p);
    // This code makes a copy and frees it right away.
    // You may prefer your approach of `strdup`-ing, and setting '\0' in place.
    char *s = malloc(len+1);
    memcpy(s, p, len);
    s[len] = '\0';
    printf("--> '%s'\n", s);
    free(s);
    if (!next) break;
    p = next+1;
}

This take care of leading and trailing empty tokens, as long as empty tokens in the middle. Here is a demo on ideone.

score 0 · Answer 3 · answered Sep 16 '13 at 15:02

You can easily implement a strtok-like function by yourselves that has that behaviour:

#include <string.h>
#include <stdio.h>
#include <stdlib.h>


/* \short tokenize a string
 * \param delim a delimiter char
 * \param str the string to tokenize or NULL for the next result
 * \return the current token or NULL if end of tokens.
 */
char* tokenize(char delim, char *str) {
    static char *stat_str = NULL;
    static char *position = NULL;
    static char *end;
    char *c;

    /* Add some assertions here... */

    if (str != NULL) {
        stat_str = str;
        end = stat_str + strlen(str) + 1;
    }
    position = stat_str;

    /* If the current position is the end of the original string, return NULL */
    if (position == end) {
        return NULL;
    }

    /* Search for the next occurence of the delimiter and change it to a null char */
    for (c = position; *c != 0; c++) {
        if (*c == delim) {
            *c = 0;
            break;
        }
    }

    /* Set the position to the next char */
    stat_str = c + 1;

    /* return the current token */
    return position;
}

int main() {
    const char str[] = "\\ISO 2022 IR 13\\ISO 2022 IR 87";

    char *query = strdup(str);

    for (char *token = tokenize('\\', query); token != NULL;token = tokenize('\\', NULL)) {
        printf("Token [%s]\n", token);
    }

    free(query);
}

Efficiently extract tokens from strings

3 Answers3