Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- Token* TokenizeString(char *input)
- {
- const int size_of_token = 30;
- int len = 0;
- int num_of_tokens = 0;
- int i = 0, j = 0, k = 0;
- int newline = 1;
- char *buf;
- //char **token = NULL;
- char token_word[30] = {0};
- char tmp[20] = {0};
- FILE *f = NULL;
- Token *tokens;
- len = strlen(input);
- buf = (char *)calloc(len, sizeof(char));
- if (buf == NULL)
- printf("Couldn't allocate memory for buf\n");
- num_of_tokens = len / 3;
- tokens = calloc(num_of_tokens, sizeof(Token));
- for (i = 0; i < num_of_tokens; i++)
- tokens[i].word = calloc(size_of_token, sizeof(char));
- input = RemoveRedundantNewLines(input);
- f = fopen("newline.txt", "w");
- if (f != NULL)
- fputs(input, f);
- fclose(f);
- for (i = 0; i < len; i++)
- {
- if (input[i] != ' ' && input[i] != '\n' && input[i] != '\t' && input[i] != ',')
- {
- token_word[j++] = input[i];
- }
- else if (token_word[0] != 0)
- {
- j = 0;
- tokens[k].line = newline;
- strcpy(tokens[k++].word, token_word);
- ZeroString(token_word, size_of_token);
- }
- if (input[i] == '\n')
- {
- newline++;
- }
- }
- tokens[k].line = newline;
- strcpy(tokens[k].word, token_word);
- f = fopen("tokens.txt", "w");
- for (i = 0; i <= k; i++)
- {
- sprintf(tmp, ":%d ", tokens[i].line);
- strcat(buf, tmp);
- strcat(buf, tokens[i].word);
- strcat(buf, "\n");
- }
- fputs(buf, f);
- fclose(f);
- free(buf);
- return tokens;
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement