1 files changed, 53 insertions, 0 deletions
diff --git a/tokenizer.h b/tokenizer.h
new file mode 100644
index 0000000..1d3acdb
--- /dev/null
+++ b/tokenizer.h
@@ -0,0 +1,53 @@
+#ifndef __TOKENIZER_H
+#define __TOKENIZER_H
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+
+#include "darray.h"
+
+#define TOK_NONE	0
+#define TOK_HEX		1
+#define TOK_INT		2
+#define TOK_NL		3
+#define TOK_IF		4
+#define TOK_LBR		5
+#define TOK_RBR		6
+#define TOK_STR		7
+#define TOK_EAND	8
+#define TOK_RANGE	9
+
+#define TOK2STR(X) TOK_#X
+
+/*not best way how to define*/
+#define TOK_IN_RANGE(TOK) ((TOK_HEX>=0)&&(TOK_RANGE<=TOK_RANGE))
+
+typedef struct token
+{
+	int token;
+	char *s, *e;
+} token;
+
+typedef struct token_list
+{
+	darray *list;
+} token_list;
+
+token* token_create();
+int  token_set( token **tok, int val, char *s, char *e );
+
+token_list* tl_create();
+int tl_add_tok( token_list *tl, int t, char *s, char *e  );
+char* tl_str( token_list *tl );
+void tl_destroy( token_list *tl );
+int tl_size( token_list *tl );
+
+#define tl_tok_type(TL,IDX) (((token *)(darr_get((TL)->list,IDX)))->token)
+#define tl_get_tok(TL,IDX) ((token *)(darr_get((TL)->list,IDX)))
+
+int tok2int( token *tok );
+char *tok2str( token *tok );
+
+#endif
+\ No newline at end of file