Custom printers for user token types

I did this in a hurry, but it appears to work locally. I tested it with
a couple boring custom token type printers.

src/hammer.h

@@ -778,10 +778,25 @@ void h_benchmark_report(FILE* stream, HBenchmarkResults* results);
 //void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results);
 // }}}
 
+// {{{ result_buf printers (used by token type registry)
+
+struct result_buf;
+
+bool h_append_buf(struct result_buf *buf, const char* input, int len);
+bool h_append_buf_c(struct result_buf *buf, char v);
+bool h_append_buf_formatted(struct result_buf *buf, char* format, ...);
+
+// }}}
+
 // {{{ Token type registry
 /// Allocate a new, unused (as far as this function knows) token type.
 HTokenType h_allocate_token_type(const char* name);
 
+/// Allocate a new token type with an unambiguous print function.
+HTokenType h_allocate_token_new(
+    const char* name,
+    void (*unamb_sub)(const HParsedToken *tok, struct result_buf *buf));
+
 /// Get the token type associated with name. Returns -1 if name is unkown
 HTokenType h_get_token_type_number(const char* name);
 

src/internal.h

@@ -422,6 +422,18 @@ struct HParserVtable_ {
   bool higher; // false if primitive
 };
 
+// {{{ Token type registry internal
+
+typedef struct HTTEntry_ {
+  const char* name;
+  HTokenType value;
+  void (*unamb_sub)(const HParsedToken *tok, struct result_buf *buf);
+} HTTEntry;
+
+const HTTEntry* h_get_token_type_entry(HTokenType token_type);
+
+// }}}
+
 bool h_false(void*);
 bool h_true(void*);
 bool h_not_regular(HRVMProg*, void*);

src/pprint.c

@@ -41,10 +41,10 @@ void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta) {
     else {
       fprintf(stream, "%*s", indent, "");
       for (size_t i = 0; i < tok->bytes.len; i++) {
-	fprintf(stream,
-		"%c%02hhx",
-		(i == 0) ? '<' : '.',
-		tok->bytes.token[i]);
+        fprintf(stream,
+                "%c%02hhx",
+                (i == 0) ? '<' : '.',
+                tok->bytes.token[i]);
       }
       fprintf(stream, ">\n");
     }
@@ -54,7 +54,7 @@ void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta) {
       fprintf(stream, "%*ss -%#" PRIx64 "\n", indent, "", -tok->sint);
     else
       fprintf(stream, "%*ss %#" PRIx64 "\n", indent, "", tok->sint);
-      
+
     break;
   case TT_UINT:
     fprintf(stream, "%*su %#" PRIx64 "\n", indent, "", tok->uint);
@@ -96,7 +96,7 @@ static inline bool ensure_capacity(struct result_buf *buf, int amt) {
   return true;
 }
 
-static inline bool append_buf(struct result_buf *buf, const char* input, int len) {
+bool h_append_buf(struct result_buf *buf, const char* input, int len) {
   if (ensure_capacity(buf, len)) {
     memcpy(buf->output + buf->len, input, len);
     buf->len += len;
@@ -106,7 +106,7 @@ static inline bool append_buf(struct result_buf *buf, const char* input, int len
   }
 }
 
-static inline bool append_buf_c(struct result_buf *buf, char v) {
+bool h_append_buf_c(struct result_buf *buf, char v) {
   if (ensure_capacity(buf, 1)) {
     buf->output[buf->len++] = v;
     return true;
@@ -116,7 +116,7 @@ static inline bool append_buf_c(struct result_buf *buf, char v) {
 }
 
 /** append a formatted string to the result buffer */
-static inline bool append_buf_formatted(struct result_buf *buf, char* format, ...)
+bool h_append_buf_formatted(struct result_buf *buf, char* format, ...)
 {
   char* tmpbuf;
   int len;
@@ -125,7 +125,7 @@ static inline bool append_buf_formatted(struct result_buf *buf, char* format, ..
 
   va_start(ap, format);
   len = h_platform_vasprintf(&tmpbuf, format, ap);
-  result = append_buf(buf, tmpbuf, len);
+  result = h_append_buf(buf, tmpbuf, len);
   free(tmpbuf);
   va_end(ap);
 
@@ -134,52 +134,59 @@ static inline bool append_buf_formatted(struct result_buf *buf, char* format, ..
 
 static void unamb_sub(const HParsedToken* tok, struct result_buf *buf) {
   if (!tok) {
-    append_buf(buf, "NULL", 4);
+    h_append_buf(buf, "NULL", 4);
     return;
   }
   switch (tok->token_type) {
   case TT_NONE:
-    append_buf(buf, "null", 4);
+    h_append_buf(buf, "null", 4);
     break;
   case TT_BYTES:
     if (tok->bytes.len == 0)
-      append_buf(buf, "<>", 2);
+      h_append_buf(buf, "<>", 2);
     else {
       for (size_t i = 0; i < tok->bytes.len; i++) {
-	const char *HEX = "0123456789abcdef";
-	append_buf_c(buf, (i == 0) ? '<': '.');
-	char c = tok->bytes.token[i];
-	append_buf_c(buf, HEX[(c >> 4) & 0xf]);
-	append_buf_c(buf, HEX[(c >> 0) & 0xf]);
+        const char *HEX = "0123456789abcdef";
+        h_append_buf_c(buf, (i == 0) ? '<': '.');
+        char c = tok->bytes.token[i];
+        h_append_buf_c(buf, HEX[(c >> 4) & 0xf]);
+        h_append_buf_c(buf, HEX[(c >> 0) & 0xf]);
       }
-      append_buf_c(buf, '>');
+      h_append_buf_c(buf, '>');
     }
     break;
   case TT_SINT:
     if (tok->sint < 0)
-      append_buf_formatted(buf, "s-%#" PRIx64, -tok->sint);
+      h_append_buf_formatted(buf, "s-%#" PRIx64, -tok->sint);
     else
-      append_buf_formatted(buf, "s%#" PRIx64, tok->sint);
+      h_append_buf_formatted(buf, "s%#" PRIx64, tok->sint);
     break;
   case TT_UINT:
-    append_buf_formatted(buf, "u%#" PRIx64, tok->uint);
+    h_append_buf_formatted(buf, "u%#" PRIx64, tok->uint);
     break;
   case TT_ERR:
-    append_buf(buf, "ERR", 3);
+    h_append_buf(buf, "ERR", 3);
     break;
   case TT_SEQUENCE: {
-    append_buf_c(buf, '(');
+    h_append_buf_c(buf, '(');
     for (size_t i = 0; i < tok->seq->used; i++) {
       if (i > 0)
-	append_buf_c(buf, ' ');
+        h_append_buf_c(buf, ' ');
       unamb_sub(tok->seq->elements[i], buf);
     }
-    append_buf_c(buf, ')');
+    h_append_buf_c(buf, ')');
   }
     break;
-  default:
-    fprintf(stderr, "Unexpected token type %d\n", tok->token_type);
-    assert_message(0, "Should not reach here.");
+  default: {
+    const HTTEntry *e = h_get_token_type_entry(tok->token_type);
+    if (e) {
+      h_append_buf_c(buf, '{');
+      e->unamb_sub(tok, buf);
+      h_append_buf_c(buf, '}');
+    } else {
+      assert_message(0, "Bogus token type.");
+    }
+  }
   }
 }
   
@@ -192,7 +199,7 @@ char* h_write_result_unamb(const HParsedToken* tok) {
   };
   assert(buf.output != NULL);
   unamb_sub(tok, &buf);
-  append_buf_c(&buf, 0);
+  h_append_buf_c(&buf, 0);
   return buf.output;
 }
   

src/registry.c

@@ -26,13 +26,8 @@
 #define h_strdup strdup
 #endif
 
-typedef struct Entry_ {
-  const char* name;
-  HTokenType value;
-} Entry;
-
 static void *tt_registry = NULL;
-static Entry** tt_by_id = NULL;
+static HTTEntry** tt_by_id = NULL;
 static unsigned int tt_by_id_sz = 0;
 #define TT_START TT_USER
 static HTokenType tt_next = TT_START;
@@ -40,23 +35,31 @@ static HTokenType tt_next = TT_START;
 /*
   // TODO: These are for the extension registry, which does not yet have a good name.
 static void *ext_registry = NULL;
-static Entry** ext_by_id = NULL;
+static HTTEntry** ext_by_id = NULL;
 static int ext_by_id_sz = 0;
 static int ext_next = 0;
 */
 
 
 static int compare_entries(const void* v1, const void* v2) {
-  const Entry *e1 = (Entry*)v1, *e2 = (Entry*)v2;
+  const HTTEntry *e1 = (HTTEntry*)v1, *e2 = (HTTEntry*)v2;
   return strcmp(e1->name, e2->name);
 }
 
-HTokenType h_allocate_token_type(const char* name) {
-  Entry* new_entry = h_alloc(&system_allocator, sizeof(*new_entry));
+static void default_unamb_sub(const HParsedToken* tok,
+                              struct result_buf* buf) {
+  h_append_buf_formatted(buf, "XXX AMBIGUOUS USER TYPE %d", tok->token_type);
+}
+
+HTokenType h_allocate_token_new(
+    const char* name,
+    void (*unamb_sub)(const HParsedToken *tok, struct result_buf *buf)) {
+  HTTEntry* new_entry = h_alloc(&system_allocator, sizeof(*new_entry));
   assert(new_entry != NULL);
   new_entry->name = name;
   new_entry->value = 0;
-  Entry* probe = *(Entry**)tsearch(new_entry, &tt_registry, compare_entries);
+  new_entry->unamb_sub = unamb_sub;
+  HTTEntry* probe = *(HTTEntry**)tsearch(new_entry, &tt_registry, compare_entries);
   if (probe->value != 0) {
     // Token type already exists...
     // TODO: treat this as a bug?
@@ -81,10 +84,13 @@ HTokenType h_allocate_token_type(const char* name) {
     return probe->value;
   }
 }
+HTokenType h_allocate_token_type(const char* name) {
+  return h_allocate_token_new(name, default_unamb_sub);
+}
 HTokenType h_get_token_type_number(const char* name) {
-  Entry e;
+  HTTEntry e;
   e.name = name;
-  Entry **ret = (Entry**)tfind(&e, &tt_registry, compare_entries);
+  HTTEntry **ret = (HTTEntry**)tfind(&e, &tt_registry, compare_entries);
   if (ret == NULL)
     return 0;
   else
@@ -96,3 +102,9 @@ const char* h_get_token_type_name(HTokenType token_type) {
   else
     return tt_by_id[token_type - TT_START]->name;
 }
+const HTTEntry* h_get_token_type_entry(HTokenType token_type) {
+  if (token_type >= tt_next || token_type < TT_START)
+    return NULL;
+  else
+    return tt_by_id[token_type - TT_START];
+}