1 files changed, 141 insertions, 0 deletions
diff --git a/tokenizer.c b/tokenizer.c
new file mode 100644
index 0000000..a7c73e2
--- /dev/null
+++ b/tokenizer.c
@@ -0,0 +1,141 @@
+#include "tokenizer.h"
+
+
+/*****************************************************************************/
+token* token_create()
+{
+	token *t = NULL;
+	t = malloc(sizeof(token));
+	memset(t,0,sizeof(token));
+	return t;
+}
+
+
+/*****************************************************************************/
+int  token_set( token **tok, int val, char *s, char *e )
+{
+	if ( tok == NULL )
+	{
+		printf("token is NULL\n");
+		return -1;
+	}
+
+	if ( !(TOK_IN_RANGE(val)) )
+	{
+		printf("Token value not in range\n");
+		return -1;
+	}
+
+	(*tok)->token = val;
+	(*tok)->s = s;
+	(*tok)->e = e;
+
+	return 0;
+}
+
+
+/*****************************************************************************/
+token_list* tl_create()
+{
+	token_list *tl = NULL;
+
+	tl = malloc( sizeof(token_list) );
+	if (tl == NULL)
+	{
+		printf("Cannot alloc mem\n");
+		goto error;
+	}
+	memset(tl, 0, sizeof(token_list));
+
+	tl->list = darr_create( sizeof(token), 100 );
+	if (tl->list == NULL)
+	{
+		printf("cannot create ->list\n");
+		goto error;
+	}
+
+	return tl;
+
+error:
+	return NULL;
+}
+
+
+/*****************************************************************************/
+int tl_add_tok( token_list *tl, int t, char *s, char *e )
+{
+	token *tok = NULL;
+	tok = token_create();
+	if (tok == NULL)
+	{
+		printf("Cannot add token\n");
+		return -1;
+	}
+	token_set( &tok, t, s, e );
+	darr_push( tl->list, (void *)tok );
+	return 0;
+}
+
+
+/*****************************************************************************/
+char* tl_str( token_list *tl )
+{
+	int len = 128;
+	int p   = 0;
+	char *s = NULL;
+	int i   = 0;
+	//s = malloc( len );
+	//s[p] = 0;
+
+
+	for (i=0; i<darr_end(tl->list); i++)
+	{
+		token *t = (token *)darr_get(tl->list, i);
+		//printf("t=0x%x\n",t);
+		if ( t != NULL )
+		{
+			printf("%02d:TOK:0x%x s:0x%08x e:0x%08x\n",i,t->token,t->s,t->e);
+		}
+	}
+
+	return s;
+error:
+	return NULL;
+}
+
+
+/*****************************************************************************/
+void tl_destroy( token_list *tl )
+{
+	if ( tl == NULL )
+		return;
+	if ( tl->list != NULL )
+	{
+		darr_clear_destroy( tl->list );
+	}
+}
+
+
+/*****************************************************************************/
+int tl_size( token_list *tl )
+{
+	return darr_end( tl->list );
+}
+
+
+int tok2int( token *tok )
+{
+	char str[128];
+	int sz = tok->e - tok->s;
+	memcpy(str,tok->s,sz);
+	str[sz] = 0;
+	//printf("!%s %d\n",str,strtol(str,NULL,16));
+	return strtol(str,NULL,16);
+}
+
+
+char *tok2str( token *tok )
+{
+
+}
+