tokenizer works

main.c

@@ -1,5 +1,3 @@
-#include <stdio.h>
-
 #include "src/parser.h"
 
 int main(int argc, char** argv) {
@@ -28,7 +26,7 @@ int main(int argc, char** argv) {
 
   inp[size] = '\0';
 
-  printf("%p\n", (void*) parse(inp));
+  sc_ast_print(sc_parse(inp));
 
   return 0;
 }

src/parser.c

@@ -1,5 +1,70 @@
 #include "parser.h"
 
-sc_ast* parse(char* input) {
+char** tokenize(char* str) {
-  return NULL;
+  char** tokens = NULL;
+  int len = 0;
+
+  while (*str) {
+    char* t;
+
+    while (*str == ' ' || *str == '\n' || *str == '\t') ++str;
+
+    tokens = realloc(tokens, (++len)*sizeof(char*));
+
+    if (*str == '(') {
+      t = malloc(2);
+      strncpy(t, "(", 2);
+      tokens[len-1] = t;
+      str++;
+    } else if (*str == ')') {
+      t = malloc(2);
+      strncpy(t, ")", 2);
+      tokens[len-1] = t;
+      str++;
+    }else {
+      t = str;
+      while (*t && *t != ' ' && *t != '(' && *t != ')') ++t;
+      tokens[len-1] = malloc(t-str+1);
+      strncpy(tokens[len-1], str, t-str);
+      tokens[len-1][t-str] = '\0';
+      str = t;
+    }
+
+    while (*str == ' ' || *str == '\n' || *str == '\t') ++str;
+  }
+  tokens = realloc(tokens, (len+1)*sizeof(char*));
+  tokens[len] = NULL;
+  return tokens;
+}
+
+sc_ast* sc_parse(char* input) {
+  char** tokens = tokenize(input);
+  sc_ast* parent = malloc(sizeof(sc_ast));
+  parent->tag = 0;
+  parent->value = NULL;
+  parent->children = NULL;
+  parent->n_children = 0;
+
+  while (*tokens) {
+    parent->n_children++;
+
+    sc_ast* child = malloc(sizeof(sc_ast));
+    parent->children = realloc(parent->children, parent->n_children*sizeof(sc_ast*));
+    child->tag = 1;
+    child->value = *tokens;
+    child->children = NULL;
+    child->n_children = 0;
+    parent->children[parent->n_children-1] = child;
+    tokens++;
+  }
+
+  return parent;
+}
+
+void sc_ast_print(sc_ast* ast) {
+  int i;
+
+  printf("(%d) %s (%lu)\n", ast->tag, ast->value, ast->value ? strlen(ast->value) : 0);
+
+  for (i = 0; i < ast->n_children; ++i) sc_ast_print(ast->children[i]);
 }

src/parser.h

@@ -1,4 +1,6 @@
+#include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 
 typedef struct sc_ast {
   short tag;
@@ -9,4 +11,5 @@ typedef struct sc_ast {
 } sc_ast;
 
 
-sc_ast* parse(char*);
+sc_ast* sc_parse(char*);
+void sc_ast_print(sc_ast*);