How to make the tokinezer detect empty spaces while using strtok()

Question

I am designing a c++ program, somewhere in the program i need to detect if there is a blank(empty token) next to the token used know eg.

 if(token1==start)
    {   
        token2=strtok(NULL," ");
        if(token2==NULL)
        {LCCTR=0;}
        else 
            {LCCTR=atoi(token2);}

so in the previous peice token1 is pointing to start , and i want to check if there is anumber next to the start , so I used token2=strtok(NULL," ") to point to the next token but unfortunattly the strtok function cannot detect empty spaces so it gives me an error at run time"INVALID NULL POINTER" how can i fix it or is there another function to use to detect empty spaces

#include <iostream>
#include<string>        
#include<map>
#include<iomanip>
#include<fstream>
#include<ctype.h>



using namespace std;
const int MAX=300;

int LCCTR;
int START(char* token1);
char* PASS1(char*token1);

void tokinizer()
{
   ifstream in;                 
   ofstream out;                

   char oneline[MAX];           
   in.open("infile.txt");       
   out.open("outfile.txt");


if(in.is_open())        
   {
       char *token1;
       in.getline(oneline,MAX); 
       token1 = strtok(oneline," \t");
       START (token1);
        //cout<<'\t';

 while(token1!=NULL)
  { 

            //PASS1(token1);
            //cout<<token1<<" ";


            token1=strtok(NULL," \t");

            if(NULL==token1)
            {//cout<<endl;
            //cout<<LCCTR<<'\t';
            in.getline(oneline,MAX);
            token1 = strtok(oneline," \t");
            }

    }
 }
   in.close();
   out.close();

}
int START(char* token1)
{
    string start("START");
    char*token2;
    if(token1 != start)
    {LCCTR=0;}

    else if(token1==start)
    {   
        token2=strchr(token1+2,' ');
        cout<<token2;
        if(token2==NULL)
        {LCCTR=0;}
        else 
       {LCCTR=atoi(token2);
        if(atoi(token2)>9999||atoi(token2)<0){cout<<"IVALID STARTING ADDRESS"<<endl;exit(1);}
        }   
    }
    return LCCTR;

}
char* PASS1 (char*token1)
{       
    map<string,int> operations;
    map<string,int>symtable;
    map<string,int>::iterator it;
    pair<map<string,int>::iterator,bool> ret;
    char*token3=NULL;
    char*token2=NULL;

    string test;
    string comp(" ");
    string start("START");
    string word("WORD");
    string byte("BYTE");
    string resb("RESB");
    string resw("RESW");
    string end("END");

        operations["ADD"] = 18;
        operations["AND"] = 40;
        operations["COMP"] = 28;
        operations["DIV"] = 24;
        operations["J"] = 0X3c;
        operations["JEQ"] =30;
        operations["JGT"] =34;
        operations["JLT"] =38;
        operations["JSUB"] =48;
        operations["LDA"] =00;
        operations["LDCH"] =50;
        operations["LDL"] =55;
        operations["LDX"] =04;
        operations["MUL"] =20;
        operations["OR"] =44;
        operations["RD"] =0xd8;
        operations["RSUB"] =0x4c;
        operations["STA"] =0x0c;
        operations["STCH"] =54;
        operations["STL"] =14;
        operations["STSW"] =0xe8;
        operations["STX"] =10;
        operations["SUB"] =0x1c;
        operations["TD"] =0xe0;
        operations["TIX"] =0x2c;
        operations["WD"] =0xdc;



        if(operations.find("ADD")->first==token1)
        {   token2=strtok(NULL," ");
            //test=token2;
            cout<<token2;
            //if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            //else{LCCTR=LCCTR+3;}
        }
        /*else if(operations.find("AND")->first==token1)
        {   token2=strtok(NULL," ");
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("COMP")->first==token1)
        {   token2=token1+5;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("DIV")->first==token1)
        {   token2=token1+4;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("J")->first==token1)
        {   token2=token1+2;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("JEQ")->first==token1)
        {   token2=token1+5;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("JGT")->first==token1)
        {   token2=strtok(NULL," ");
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("JLT")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("JSUB")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("LDA")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("LDCH")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("LDL")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("LDX")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("MUL")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("OR")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("RD")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("RSUB")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("STA")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("STCH")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("STL")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("STSW")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("STX")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("SUB")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("TD")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("TIX")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }
        else if(operations.find("WD")->first==token1)
        {   token2=token1+6;
            test=token2;
            if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
            else{LCCTR=LCCTR+3;}
        }*/

        //else if(


         if(word==token1)
        {LCCTR=LCCTR+3;}

        else if(byte==token1)
        {string test;
        token2=token1+7;
        test=token2;
            if(test[0]=='C')
            {token3=token1+10;
            test=token3;
            if(test.length()>15)
            {cout<<"ERROR"<<endl;
            exit(1);}
            }
            else if(test[0]=='X')
            {token3=token1+10;
            test=token3;
            if(test.length()>14)
            {cout<<"ERROR"<<endl;
            exit(1);}
            }
            LCCTR=LCCTR+test.length();
        }

        else if(resb==token1)
        {token3=token1+5;
        LCCTR=LCCTR+atoi(token3);}

        else if(resw==token1)
        {token3=token1+5;
        LCCTR=LCCTR+3*atoi(token3);}

        else if(end==token1)
        {exit(1);}



    /*else
    {   
        test=token1;
        int last=test.length();
        if(token1==start||test[0]=='C'||test[0]=='X'||ispunct(test[last])||isdigit(test[0])||isdigit(test[1])||isdigit(test[2])||isdigit(test[3])){}
        else    {
                token2=strtok(NULL," ");
                //test=token2;
                cout<<token2;
                if(token2!=NULL)
                {
                    symtable.insert( pair<string,int>(token1,LCCTR));
                    for(it=symtable.begin() ;it!=symtable.end() ;++it)
                        {/*cout<<"symbol: "<<it->first<<"      LCCTR: "<<it->second<<endl;}
                }
                else{}
        }
        }*/




return token3;

}
int main()
{
   tokinizer();
   return 0;
}

Please show us more of the code. You are passing NULL to strtok - you need to have already called it with a non-NULL pointer before that if you want it to work, but that bit of your code is not included. — The Archetypal Paul, Dec 25 '10 at 17:59
possible duplicate of [How to make the tokenizer detect empty spaces while using strtok()](http://stackoverflow.com/questions/4531201/how-to-make-the-tokenizer-detect-empty-spaces-while-using-strtok) — Yakov Galka, Dec 25 '10 at 18:47
possible duplicate of [Input line by line from an input file and tokenize using strtok() and the output into an output file](http://stackoverflow.com/questions/4328685/input-line-by-line-from-an-input-file-and-tokenize-using-strtok-and-the-output) — Paul R, Dec 25 '10 at 18:55

score 1 · Answer 1 · answered Dec 25 '10 at 18:02

Without seeing your actual code it's hard to say what the exact problem is. A generic answer follows.

strtok is stateful internally. You're can't just willy-nilly call it on on NULL.

You could have a pass that tokenizes your string and gathers the returned tokens into an array or vector. First call on start, subsequent calls (until strtok returns NULL) on NULL. You can then easily figure out what's what.

How to make the tokinezer detect empty spaces while using strtok()

1 Answers1