src/parsers/butnot.c

@@ -5,7 +5,6 @@ typedef struct {
   const HParser *p2;
 } HTwoParsers;
 
-
 static HParseResult* parse_butnot(void *env, HParseState *state) {
   HTwoParsers *parsers = (HTwoParsers*)env;
   // cache the initial state of the input stream
@@ -19,15 +18,18 @@ static HParseResult* parse_butnot(void *env, HParseState *state) {
   HInputStream after_p1_state = state->input_stream;
   state->input_stream = start_state;
   HParseResult *r2 = h_do_parse(parsers->p2, state);
-  // TODO(mlp): I'm pretty sure the input stream state should be the post-p1 state in all cases
+  // don't touch the input state (overrun flag) if we must suspend
+  if (want_suspend(state)) {
+    return NULL;
+  }
+  // in all other cases, the input stream should be in the post-p1 state
   state->input_stream = after_p1_state;
-  // if p2 failed, restore post-p1 state and bail out early
   if (NULL == r2) {
     return r1;
   }
   size_t r1len = token_length(r1);
   size_t r2len = token_length(r2);
-  // if both match but p1's text is shorter than than p2's (or the same length), fail
+  // if both match but p1's text no longer than p2's, fail
   if (r1len <= r2len) {
     return NULL;
   } else {

src/parsers/bytes.c

+#include "parser_internal.h"
+
+struct bytes_env {
+  size_t length;
+};
+
+static HParseResult *parse_bytes(void *env_, HParseState *state)
+{
+  struct bytes_env *env = env_;
+  uint8_t *bs;
+  size_t i;
+
+  bs = a_new(uint8_t, env->length);
+  for (i=0; i < env->length && !state->input_stream.overrun; i++)
+    bs[i] = h_read_bits(&state->input_stream, 8, false);
+
+  HParsedToken *result = a_new(HParsedToken, 1);
+  result->token_type = TT_BYTES;
+  result->bytes.token = bs;
+  result->bytes.len = env->length;
+  result->index = 0;
+  result->bit_length = 0;
+  result->bit_offset = 0;
+  return make_result(state->arena, result);
+}
+
+static const HParserVtable bytes_vt = {
+  .parse = parse_bytes,
+  .isValidRegular = h_false,	// XXX need desugar_bytes, reshape_bytes
+  .isValidCF = h_false,		// XXX need bytes_ctrvm
+};
+
+HParser *h_bytes(size_t len)
+{
+  return h_bytes__m(&system_allocator, len);
+}
+
+HParser *h_bytes__m(HAllocator *mm__, size_t len)
+{
+  struct bytes_env *env = h_new(struct bytes_env, 1);
+  env->length = len;
+  return h_new_parser(mm__, &bytes_vt, env);
+}

src/parsers/choice.c

@@ -30,6 +30,8 @@ static HParseResult* parse_choice(void *env, HParseState *state) {
     HParseResult *tmp = h_do_parse(s->p_array[i], state);
     if (NULL != tmp)
       return tmp;
+    if (want_suspend(state))
+      return NULL;		// bail out early, leaving overrun flag
   }
   // nothing succeeded, so fail
   return NULL;
@@ -163,7 +165,8 @@ HParser* h_choice__ma(HAllocator* mm__, void *args[]) {
   HParser *ret = h_new(HParser, 1);
   ret->vtable = &choice_vt; 
   ret->env = (void*)s;
-  ret->backend = PB_MIN;
+  ret->backend = h_get_default_backend();
+  ret->backend_vtable = h_get_default_backend_vtable();
   ret->desugared = NULL;
   return ret;
 }

src/parsers/difference.c

@@ -18,9 +18,12 @@ static HParseResult* parse_difference(void *env, HParseState *state) {
   HInputStream after_p1_state = state->input_stream;
   state->input_stream = start_state;
   HParseResult *r2 = h_do_parse(parsers->p2, state);
-  // TODO(mlp): I'm pretty sure the input stream state should be the post-p1 state in all cases
+  // don't touch the input state (overrun flag) if we must suspend
+  if (want_suspend(state)) {
+    return NULL;
+  }
+  // in all other cases, the input stream should be in the post-p1 state
   state->input_stream = after_p1_state;
-  // if p2 failed, restore post-p1 state and bail out early
   if (NULL == r2) {
     return r1;
   }
@@ -34,7 +37,7 @@ static HParseResult* parse_difference(void *env, HParseState *state) {
   }
 }
 
-static HParserVtable difference_vt = {
+static const HParserVtable difference_vt = {
   .parse = parse_difference,
   .isValidRegular = h_false,
   .isValidCF = h_false, // XXX should this be true if both p1 and p2 are CF?

src/parsers/end.c

 #include "parser_internal.h"
 
-static HParseResult* parse_end(void *env, HParseState *state) {
-  if (state->input_stream.index == state->input_stream.length) {
+static HParseResult* parse_end(void *env, HParseState *state)
+{
+  if (state->input_stream.index < state->input_stream.length)
+    return NULL;
+
+  assert(state->input_stream.index == state->input_stream.length);
+  if (state->input_stream.last_chunk) {
     HParseResult *ret = a_new(HParseResult, 1);
     ret->ast = NULL;
     ret->bit_length = 0;
     ret->arena = state->arena;
     return ret;
   } else {
+    state->input_stream.overrun = true;	// need more input
     return NULL;
   }
 }

src/parsers/epsilon.c

@@ -29,7 +29,8 @@ HParser* h_epsilon_p__m(HAllocator* mm__) {
   HParser *epsilon_p = h_new(HParser, 1);
   epsilon_p->desugared = NULL;
   epsilon_p->backend_data = NULL;
-  epsilon_p->backend = 0;
+  epsilon_p->backend = h_get_default_backend();
+  epsilon_p->backend_vtable = h_get_default_backend_vtable();
   epsilon_p->vtable = &epsilon_vt;
   return epsilon_p;
 }

src/parsers/many.c

@@ -21,17 +21,16 @@ static HParseResult *parse_many(void* env, HParseState *state) {
     if (count > 0 && env_->sep != NULL) {
       HParseResult *sep = h_do_parse(env_->sep, state);
       if (!sep)
-	goto err0;
+	goto stop;
     }
     HParseResult *elem = h_do_parse(env_->p, state);
     if (!elem)
-      goto err0;
+      goto stop;
     if (elem->ast)
       h_carray_append(seq, (void*)elem->ast);
     count++;
   }
-  if (count < env_->count)
-    goto err;
+  assert(count == env_->count);
  succ:
   ; // necessary for the label to be here...
   HParsedToken *res = a_new(HParsedToken, 1);
@@ -41,13 +40,13 @@ static HParseResult *parse_many(void* env, HParseState *state) {
   res->bit_length = 0;
   res->bit_offset = 0;
   return make_result(state->arena, res);
- err0:
+ stop:
+  if (want_suspend(state))
+    return NULL;		// bail out early, leaving overrun flag
   if (count >= env_->count) {
     state->input_stream = bak;
     goto succ;
   }
- err:
-  state->input_stream = bak;
   return NULL;
 }
 

src/parsers/not.c

@@ -4,10 +4,11 @@ static HParseResult* parse_not(void* env, HParseState* state) {
   HInputStream bak = state->input_stream;
   if (h_do_parse((HParser*)env, state))
     return NULL;
-  else {
-    state->input_stream = bak;
-    return make_result(state->arena, NULL);
-  }
+  if (want_suspend(state))
+    return NULL;		// bail out early, leaving overrun flag
+  // regular parse failure -> success
+  state->input_stream = bak;
+  return make_result(state->arena, NULL);
 }
 
 static const HParserVtable not_vt = {

src/parsers/optional.c

@@ -6,6 +6,8 @@ static HParseResult* parse_optional(void* env, HParseState* state) {
   HParseResult *res0 = h_do_parse((HParser*)env, state);
   if (res0)
     return res0;
+  if (want_suspend(state))
+    return NULL;		// bail out early, leaving overrun flag
   state->input_stream = bak;
   HParsedToken *ast = a_new(HParsedToken, 1);
   ast->token_type = TT_NONE;

src/parsers/parser_internal.h

 /*
  * NOTE: This is an internal header and installed for use by extensions. The
  * API is not guaranteed stable.
-*/
+ */
 
-#ifndef HAMMER_PARSE_INTERNAL__H
-#define HAMMER_PARSE_INTERNAL__H
+#ifndef HAMMER_PARSER_INTERNAL__H
+#define HAMMER_PARSER_INTERNAL__H
 #include "../hammer.h"
 #include "../internal.h"
 #include "../backends/regex.h"
@@ -32,6 +32,11 @@ static inline size_t token_length(HParseResult *pr) {
   }
 }
 
+// tell if we should suspend a parser, i.e. overrun occured before final input
+static inline bool want_suspend(HParseState *state) {
+  return state->input_stream.overrun && !state->input_stream.last_chunk;
+}
+
 /* Epsilon rules happen during desugaring. This handles them. */
 static inline void desugar_epsilon(HAllocator *mm__, HCFStack *stk__, void *env) {
   HCFS_BEGIN_CHOICE() {
@@ -41,4 +46,4 @@ static inline void desugar_epsilon(HAllocator *mm__, HCFStack *stk__, void *env)
   } HCFS_END_CHOICE();
 }
 
-#endif // HAMMER_PARSE_INTERNAL__H
+#endif // HAMMER_PARSER_INTERNAL__H

src/parsers/permutation.c

@@ -29,6 +29,9 @@ static int parse_permutation_tail(const HSequence *s,
     if(set[i]) {
       match = h_do_parse(ps[i], state);
 
+      if (want_suspend(state))
+        return 0;		// bail out early, leaving overrun flag
+
       // save result
       if(match)
 	seq->elements[i] = (void *)match->ast;
@@ -178,7 +181,8 @@ HParser* h_permutation__ma(HAllocator* mm__, void *args[]) {
   HParser *ret = h_new(HParser, 1);
   ret->vtable = &permutation_vt; 
   ret->env = (void*)s;
-  ret->backend = PB_MIN;
+  ret->backend = h_get_default_backend();
+  ret->backend_vtable = h_get_default_backend_vtable();
   ret->desugared = NULL;
   return ret;
 }

src/parsers/seek.c

@@ -25,6 +25,10 @@ static HParseResult *parse_seek(void *env, HParseState *state)
     pos = 0;
     break;
   case SEEK_END:
+    if (!stream->last_chunk) {	/* the end is not yet known! */
+      stream->overrun = true;	/* we need more input */
+      return NULL;
+    }
     pos = h_input_stream_length(stream);
     break;
   case SEEK_CUR:

src/parsers/sequence.c

@@ -174,7 +174,8 @@ HParser* h_sequence__ma(HAllocator* mm__, void *args[]) {
   HParser *ret = h_new(HParser, 1);
   ret->vtable = &sequence_vt; 
   ret->env = (void*)s; 
-  ret->backend = PB_MIN;
+  ret->backend = h_get_default_backend();
+  ret->backend_vtable = h_get_default_backend_vtable();
   ret->desugared = NULL;
   return ret;
 }

src/parsers/whitespace.c

@@ -8,6 +8,8 @@ static HParseResult* parse_whitespace(void* env, HParseState *state) {
   do {
     bak = state->input_stream;
     c = h_read_bits(&state->input_stream, 8, false);
+    if (want_suspend(state))
+      return NULL;		// bail out early, leaving overrun flag
     if (state->input_stream.overrun)
       break;
   } while (isspace((int)c));

src/parsers/xor.c

@@ -11,10 +11,14 @@ static HParseResult* parse_xor(void *env, HParseState *state) {
   // cache the initial state of the input stream
   HInputStream start_state = state->input_stream;
   HParseResult *r1 = h_do_parse(parsers->p1, state);
+  if (want_suspend(state))
+    return NULL;		// bail out early, leaving overrun flag
   HInputStream after_p1_state = state->input_stream;
   // reset input stream, parse again
   state->input_stream = start_state;
   HParseResult *r2 = h_do_parse(parsers->p2, state);
+  if (want_suspend(state))
+    return NULL;		// bail out early, leaving overrun flag
   if (NULL == r1) {
     if (NULL != r2) {
       return r2;

src/t_names.c

+#include <glib.h>
+#include <string.h>
+#include "test_suite.h"
+#include "hammer.h"
+
+static void test_tt_backend_description(void) {
+  const char *desc = NULL;
+
+  /* Check that we get some */
+  desc = h_get_descriptive_text_for_backend(PB_PACKRAT);
+  g_check_cmp_ptr(desc, !=, NULL);
+  desc = h_get_descriptive_text_for_backend(PB_REGULAR);
+  g_check_cmp_ptr(desc, !=, NULL);
+  desc = h_get_descriptive_text_for_backend(PB_LLk);
+  g_check_cmp_ptr(desc, !=, NULL);
+  desc = h_get_descriptive_text_for_backend(PB_LALR);
+  g_check_cmp_ptr(desc, !=, NULL);
+  desc = h_get_descriptive_text_for_backend(PB_GLR);
+  g_check_cmp_ptr(desc, !=, NULL);
+}
+
+/* Reference backend names */
+static const char * packrat_name = "packrat";
+static const char * regular_name = "regex";
+static const char * llk_name = "llk";
+static const char * lalr_name = "lalr";
+static const char * glr_name = "glr";
+
+static void test_tt_backend_short_name(void) {
+  const char *name = NULL;
+
+  name = h_get_name_for_backend(PB_PACKRAT);
+  g_check_maybe_string_eq(name, packrat_name);
+  name = h_get_name_for_backend(PB_REGULAR);
+  g_check_maybe_string_eq(name, regular_name);
+  name = h_get_name_for_backend(PB_LLk);
+  g_check_maybe_string_eq(name, llk_name);
+  name = h_get_name_for_backend(PB_LALR);
+  g_check_maybe_string_eq(name, lalr_name);
+  name = h_get_name_for_backend(PB_GLR);
+  g_check_maybe_string_eq(name, glr_name);
+}
+
+static void test_tt_query_backend_by_name(void) {
+  HParserBackend be;
+
+  be = h_query_backend_by_name(packrat_name);
+  g_check_inttype("%d", HParserBackend, be, ==, PB_PACKRAT);
+  be = h_query_backend_by_name(regular_name);
+  g_check_inttype("%d", HParserBackend, be, ==, PB_REGULAR);
+
+}
+
+static void test_tt_get_backend_with_params_by_name(void) {
+  HParserBackendWithParams * be_w_p = NULL;
+
+  /*requests to use default params, or for backends without params*/
+  be_w_p = h_get_backend_with_params_by_name(packrat_name);
+  g_check_inttype("%d", HParserBackend, be_w_p->backend, ==, PB_PACKRAT);
+  g_check_maybe_string_eq(be_w_p->requested_name, packrat_name);
+  h_free_backend_with_params(be_w_p);
+
+  be_w_p = h_get_backend_with_params_by_name(glr_name);
+  g_check_inttype("%d", HParserBackend, be_w_p->backend, ==, PB_GLR);
+  g_check_maybe_string_eq(be_w_p->requested_name, glr_name);
+  h_free_backend_with_params(be_w_p);
+
+  /* request with params */
+  be_w_p = h_get_backend_with_params_by_name("lalr(1)");
+  g_check_inttype("%d", HParserBackend, be_w_p->backend, ==, PB_LALR);
+  g_check_maybe_string_eq(be_w_p->requested_name, lalr_name);
+  g_check_cmp_size((uintptr_t)be_w_p->params, ==, 1);
+  h_free_backend_with_params(be_w_p);
+
+  /*request for default params - alternative possible style */
+  be_w_p = h_get_backend_with_params_by_name("glr()");
+  g_check_inttype("%d", HParserBackend, be_w_p->backend, ==, PB_GLR);
+  g_check_maybe_string_eq(be_w_p->requested_name, glr_name);;
+  h_free_backend_with_params(be_w_p);
+
+  /*request for a backend not in the enum of backends included in hammer */
+  be_w_p = h_get_backend_with_params_by_name("llvm()");
+  g_check_inttype("%d", HParserBackend, be_w_p->backend, ==, PB_INVALID);
+  g_check_maybe_string_eq(be_w_p->requested_name, "llvm");
+  h_free_backend_with_params(be_w_p);
+
+  be_w_p = h_get_backend_with_params_by_name("packrat(0)");
+  g_check_inttype("%d", HParserBackend, be_w_p->backend, ==, PB_PACKRAT);
+  g_check_maybe_string_eq(be_w_p->requested_name, packrat_name);
+  h_free_backend_with_params(be_w_p);
+
+  be_w_p = h_get_backend_with_params_by_name("glr(1,2)");
+  g_check_inttype("%d", HParserBackend, be_w_p->backend, ==, PB_GLR);
+  g_check_maybe_string_eq(be_w_p->requested_name, glr_name);
+  g_check_cmp_size((uintptr_t)be_w_p->params, ==, 1);
+  h_free_backend_with_params(be_w_p);
+
+  be_w_p = h_get_backend_with_params_by_name("glr(1,vnvnvn)");
+  g_check_inttype("%d", HParserBackend, be_w_p->backend, ==, PB_GLR);
+  g_check_maybe_string_eq(be_w_p->requested_name, glr_name);
+  g_check_cmp_size((uintptr_t)be_w_p->params, ==, 1);
+  h_free_backend_with_params(be_w_p);
+
+  be_w_p = h_get_backend_with_params_by_name("lalr(1, 2)");
+  g_check_inttype("%d", HParserBackend, be_w_p->backend, ==, PB_LALR);
+  g_check_maybe_string_eq(be_w_p->requested_name, lalr_name);
+  g_check_cmp_size((uintptr_t)be_w_p->params, ==, 1);
+  h_free_backend_with_params(be_w_p);
+
+  be_w_p = h_get_backend_with_params_by_name("llk(k=2)");
+  g_check_inttype("%d", HParserBackend, be_w_p->backend, ==, PB_LLk);
+  g_check_maybe_string_eq(be_w_p->requested_name, llk_name);
+  g_check_cmp_size((uintptr_t)be_w_p->params, ==, 2);
+  h_free_backend_with_params(be_w_p);
+
+}
+
+static void test_tt_h_get_descriptive_text_for_backend_with_params(void){
+  HAllocator *mm__ = &system_allocator;
+  HParserBackendWithParams *be_with_params = h_get_backend_with_params_by_name("llk(1)");
+  char * descr = h_get_descriptive_text_for_backend_with_params(be_with_params);
+  g_check_maybe_string_eq(descr, "LL(1) parser backend");
+  h_free(descr);
+  h_free_backend_with_params(be_with_params);
+  be_with_params = h_get_backend_with_params_by_name("lalr(1)");
+  descr = h_get_descriptive_text_for_backend_with_params(be_with_params);
+  g_check_maybe_string_eq(descr, "LALR(1) parser backend");
+  h_free(descr);
+  h_free_backend_with_params(be_with_params);
+  be_with_params = h_get_backend_with_params_by_name("glr(2)");
+  descr = h_get_descriptive_text_for_backend_with_params(be_with_params);
+  g_check_maybe_string_eq(descr, "GLR(2) parser backend");
+  h_free(descr);
+  h_free_backend_with_params(be_with_params);
+  be_with_params = h_get_backend_with_params_by_name("glr()");
+  descr = h_get_descriptive_text_for_backend_with_params(be_with_params);
+  g_check_maybe_string_eq(descr, "GLR(k) parser backend (default k is 1)");
+  h_free(descr);
+  h_free_backend_with_params(be_with_params);
+}
+
+static void test_tt_h_get_name_for_backend_with_params(void){
+  HAllocator *mm__ = &system_allocator;
+  HParserBackendWithParams *be_with_params = h_get_backend_with_params_by_name("llk(1)");
+  char * descr = h_get_name_for_backend_with_params(be_with_params);
+  g_check_maybe_string_eq(descr, "LL(1)");
+  h_free(descr);
+  h_free_backend_with_params(be_with_params);
+  be_with_params = h_get_backend_with_params_by_name("lalr(2)");
+  descr = h_get_name_for_backend_with_params(be_with_params);
+  g_check_maybe_string_eq(descr, "LALR(2)");
+  h_free(descr);
+  h_free_backend_with_params(be_with_params);
+  be_with_params = h_get_backend_with_params_by_name("glr(1)");
+  descr = h_get_name_for_backend_with_params(be_with_params);
+  g_check_maybe_string_eq(descr, "GLR(1)");
+  h_free(descr);
+  h_free_backend_with_params(be_with_params);
+  be_with_params = h_get_backend_with_params_by_name("glr");
+  descr = h_get_name_for_backend_with_params(be_with_params);
+  g_check_maybe_string_eq(descr, "GLR(k)");
+  h_free(descr);
+  h_free_backend_with_params(be_with_params);
+}
+
+/* test that we can request a backend with params from character
+ * and compile a parser using it */
+static void test_tt_h_compile_for_backend_with_params(void) {
+  HParserBackendWithParams * be_w_p = NULL;
+
+  be_w_p = h_get_backend_with_params_by_name("llk(1)");
+  g_check_inttype("%d", HParserBackend, be_w_p->backend, ==, PB_LLk);
+
+  HParser *p = h_many(h_sequence(h_ch('A'), h_ch('B'), NULL));
+
+  int r = h_compile_for_backend_with_params(p, be_w_p);
+
+  h_free_backend_with_params(be_w_p);
+  be_w_p = NULL;
+
+  if (r != 0) {
+    g_test_message("Compile failed");
+    g_test_fail();
+  }
+}
+
+void register_names_tests(void) {
+  g_test_add_func("/core/names/tt_backend_short_name", test_tt_backend_short_name);
+  g_test_add_func("/core/names/tt_backend_description", test_tt_backend_description);
+  g_test_add_func("/core/names/tt_query_backend_by_name", test_tt_query_backend_by_name);
+  g_test_add_func("/core/names/tt_get_backend_with_params_by_name", test_tt_get_backend_with_params_by_name);
+  g_test_add_func("/core/names/tt_test_tt_h_get_descriptive_text_for_backend_with_params",
+                  test_tt_h_get_descriptive_text_for_backend_with_params);
+  g_test_add_func("/core/names/test_tt_h_get_name_for_backend_with_params",
+                  test_tt_h_get_name_for_backend_with_params);
+  g_test_add_func("/core/names/tt_h_compile_for_backend_with_params", test_tt_h_compile_for_backend_with_params);
+ }

src/t_parser.c

@@ -65,6 +65,29 @@ static void test_bits(gconstpointer backend) {
   g_check_parse_failed(bits_, (HParserBackend)GPOINTER_TO_INT(backend), "a", 1);
 }
 
+static void test_bytes(gconstpointer backend) {
+  HParserBackend be = (HParserBackend)GPOINTER_TO_INT(backend);
+  const HParser *p;
+
+  p = h_bytes(0);
+  g_check_parse_match(p, be, "", 0, "<>");
+  g_check_parse_match(p, be, "abc", 3, "<>");
+
+  p = h_bytes(1);
+  g_check_parse_failed(p, be, "", 0);
+  g_check_parse_match(p, be, " ", 1, "<20>");
+  g_check_parse_match(p, be, "abc", 3, "<61>");
+
+  p = h_bytes(5);
+  g_check_parse_failed(p, be, "", 0);
+  g_check_parse_failed(p, be, "1", 1);
+  g_check_parse_failed(p, be, "12", 2);
+  g_check_parse_failed(p, be, "123", 3);
+  g_check_parse_failed(p, be, "1234", 4);
+  g_check_parse_match(p, be, "12345", 5, "<31.32.33.34.35>");
+  g_check_parse_match(p, be, "12345abc", 8, "<31.32.33.34.35>");
+}
+
 //@MARK_START
 static void test_int64(gconstpointer backend) {
   const HParser *int64_ = h_int64();
@@ -507,7 +530,72 @@ static void test_rightrec(gconstpointer backend) {
   g_check_parse_match(rr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", 3, "(u0x61 (u0x61 (u0x61)))");
 }
 
-static void test_iterative(gconstpointer backend) {
+static void test_iterative_single(gconstpointer backend) {
+  HParserBackend be = (HParserBackend)GPOINTER_TO_INT(backend);
+  HParser *p;
+
+  p = h_token((uint8_t*)"foobar", 6);
+  g_check_parse_chunk_match(p, be, "foobar",6, "<66.6f.6f.62.61.72>");
+  g_check_parse_chunk_match(p, be, "foobarbaz",9, "<66.6f.6f.62.61.72>");
+  g_check_parse_chunk_failed(p, be, "foubar",6);
+  g_check_parse_chunk_failed(p, be, "foopar",6);
+  g_check_parse_chunk_failed(p, be, "foobaz",6);
+
+  p = h_sequence(h_ch('f'), h_token((uint8_t*)"ooba", 4), h_ch('r'), NULL);
+  g_check_parse_chunk_match(p, be, "foobar",6, "(u0x66 <6f.6f.62.61> u0x72)");
+  g_check_parse_chunk_match(p, be, "foobarbaz",9, "(u0x66 <6f.6f.62.61> u0x72)");
+  g_check_parse_chunk_failed(p, be, "foubar",6);
+  g_check_parse_chunk_failed(p, be, "foopar",6);
+  g_check_parse_chunk_failed(p, be, "foobaz",6);
+
+  p = h_choice(h_token((uint8_t*)"foobar", 6),
+               h_token((uint8_t*)"phupar", 6), NULL);
+  g_check_parse_chunk_match(p, be, "foobar",6, "<66.6f.6f.62.61.72>");
+  g_check_parse_chunk_match(p, be, "foobarbaz",9, "<66.6f.6f.62.61.72>");
+  g_check_parse_chunk_match(p, be, "phupar",6, "<70.68.75.70.61.72>");
+  g_check_parse_chunk_failed(p, be, "foubar",6);
+  g_check_parse_chunk_failed(p, be, "foobaz",6);
+
+  p = h_sequence(h_ch('f'), h_choice(h_token((uint8_t*)"oo", 2),
+                                     h_token((uint8_t*)"uu", 2), NULL), NULL);
+  g_check_parse_chunk_match(p, be, "foo",3, "(u0x66 <6f.6f>)");
+  g_check_parse_chunk_match(p, be, "fuu",3, "(u0x66 <75.75>)");
+  g_check_parse_chunk_failed(p, be, "goo",3);
+  g_check_parse_chunk_failed(p, be, "fou",3);
+  g_check_parse_chunk_failed(p, be, "fuo",3);
+}
+
+// this test applies to backends that support the iterative API, but not actual
+// chunked operation. in such cases, passing multiple chunks should fail the
+// parse rather than treating the end of the first chunk as the end of input.
+#if 0
+static void test_iterative_dummy(gconstpointer backend) {
+  HParserBackend be = (HParserBackend)GPOINTER_TO_INT(backend);
+  HParser *p;
+
+  HParser *x = h_ch('x');
+  HParser *y = h_ch('y');
+  HParser *e = h_epsilon_p();
+
+  p = h_many(x);
+  g_check_parse_chunks_failed(p, be, "xxx",3, "xxx",3);
+  g_check_parse_chunks_failed(p, be, "xxx",3, "",0);
+
+  p = h_optional(x);
+  g_check_parse_chunks_failed(p, be, "",0, "xxx",3);
+
+  p = h_choice(x, e, NULL);
+  g_check_parse_chunks_failed(p, be, "",0, "xxx",3);
+
+  // these are ok because the parse succeeds without overrun.
+  p = h_choice(e, x, NULL);
+  g_check_parse_chunks_match(p, be, "",0, "xxx",3, "NULL");
+  p = h_choice(y, x, NULL);
+  g_check_parse_chunks_match(p, be, "y",1, "xxx",3, "u0x79");
+}
+#endif
+
+static void test_iterative_multi(gconstpointer backend) {
   HParserBackend be = (HParserBackend)GPOINTER_TO_INT(backend);
   HParser *p;
 
@@ -567,6 +655,39 @@ static void test_iterative_lookahead(gconstpointer backend) {
   g_check_parse_chunks_failed_(p, "fo",2, "b",1);
 }
 
+static void test_iterative_seek(gconstpointer backend) {
+  HParserBackend be = (HParserBackend)GPOINTER_TO_INT(backend);
+  const HParser *p;
+
+  // seeking should work across chunk boundaries...
+
+  p = h_sequence(h_ch('a'), h_seek(40, SEEK_SET), h_ch('f'), NULL);
+  g_check_parse_chunks_match(p, be, "a",1, "bcdef",5, "(u0x61 u0x28 u0x66)");
+  g_check_parse_chunks_failed(p, be, "a",1, "bcdex",5);
+  g_check_parse_chunks_failed(p, be, "a",1, "bc",2);
+
+  p = h_sequence(h_ch('a'), h_seek(40, SEEK_SET), h_end_p(), NULL);
+  g_check_parse_chunks_match(p, be, "ab",2, "cde",3, "(u0x61 u0x28)");
+  g_check_parse_chunks_failed(p, be, "ab",2, "cdex",4);
+  g_check_parse_chunks_failed(p, be, "ab",2, "c",1);
+
+  p = h_sequence(h_ch('a'), h_seek(0, SEEK_END), h_end_p(), NULL);
+  g_check_parse_chunks_match(p, be, "abc",3, "de",2, "(u0x61 u0x28)");
+  g_check_parse_chunks_match(p, be, "abc",3, "",0, "(u0x61 u0x18)");
+
+  p = h_sequence(h_ch('a'), h_seek(-16, SEEK_END), h_ch('x'), NULL);
+  g_check_parse_chunks_match(p, be, "abcd",4, "xy",2, "(u0x61 u0x20 u0x78)");
+  g_check_parse_chunks_match(p, be, "abxy",4, "",0, "(u0x61 u0x10 u0x78)");
+  g_check_parse_chunks_failed(p, be, "a",1, "bc",2);
+  g_check_parse_chunks_failed(p, be, "",0, "x",1);
+
+  p = h_sequence(h_ch('a'), h_seek(32, SEEK_CUR), h_ch('f'), NULL);
+  g_check_parse_chunks_match(p, be, "abcde",5, "f",1, "(u0x61 u0x28 u0x66)");
+  g_check_parse_chunks_failed(p, be, "xbcde",5, "f",1);
+  g_check_parse_chunks_failed(p, be, "abcde",5, "x",1);
+  g_check_parse_chunks_failed(p, be, "abc",3, "",0);
+}
+
 static void test_iterative_result_length(gconstpointer backend) {
   HParserBackend be = (HParserBackend)GPOINTER_TO_INT(backend);
   HParser *p = h_token((uint8_t*)"foobar", 6);
@@ -887,6 +1008,7 @@ void register_parser_tests(void) {
   g_test_add_data_func("/core/parser/packrat/ch_range", GINT_TO_POINTER(PB_PACKRAT), test_ch_range);
   g_test_add_data_func("/core/parser/packrat/bits0", GINT_TO_POINTER(PB_PACKRAT), test_bits0);
   g_test_add_data_func("/core/parser/packrat/bits", GINT_TO_POINTER(PB_PACKRAT), test_bits);
+  g_test_add_data_func("/core/parser/packrat/bytes", GINT_TO_POINTER(PB_PACKRAT), test_bytes);
   g_test_add_data_func("/core/parser/packrat/int64", GINT_TO_POINTER(PB_PACKRAT), test_int64);
   g_test_add_data_func("/core/parser/packrat/int32", GINT_TO_POINTER(PB_PACKRAT), test_int32);
   g_test_add_data_func("/core/parser/packrat/int16", GINT_TO_POINTER(PB_PACKRAT), test_int16);
@@ -933,6 +1055,11 @@ void register_parser_tests(void) {
   g_test_add_data_func("/core/parser/packrat/bind", GINT_TO_POINTER(PB_PACKRAT), test_bind);
   g_test_add_data_func("/core/parser/packrat/result_length", GINT_TO_POINTER(PB_PACKRAT), test_result_length);
   //g_test_add_data_func("/core/parser/packrat/token_position", GINT_TO_POINTER(PB_PACKRAT), test_token_position);
+  g_test_add_data_func("/core/parser/packrat/iterative/single", GINT_TO_POINTER(PB_PACKRAT), test_iterative_single);
+  g_test_add_data_func("/core/parser/packrat/iterative/multi", GINT_TO_POINTER(PB_PACKRAT), test_iterative_multi);
+  g_test_add_data_func("/core/parser/packrat/iterative/lookahead", GINT_TO_POINTER(PB_PACKRAT), test_iterative_lookahead);
+  g_test_add_data_func("/core/parser/packrat/iterative/seek", GINT_TO_POINTER(PB_PACKRAT), test_iterative_seek);
+  g_test_add_data_func("/core/parser/packrat/iterative/result_length", GINT_TO_POINTER(PB_PACKRAT), test_iterative_result_length);
   g_test_add_data_func("/core/parser/packrat/skip", GINT_TO_POINTER(PB_PACKRAT), test_skip);
   g_test_add_data_func("/core/parser/packrat/seek", GINT_TO_POINTER(PB_PACKRAT), test_seek);
   g_test_add_data_func("/core/parser/packrat/tell", GINT_TO_POINTER(PB_PACKRAT), test_tell);
@@ -978,7 +1105,8 @@ void register_parser_tests(void) {
   g_test_add_data_func("/core/parser/llk/rightrec", GINT_TO_POINTER(PB_LLk), test_rightrec);
  g_test_add_data_func("/core/parser/llk/result_length", GINT_TO_POINTER(PB_LLk), test_result_length);
   //g_test_add_data_func("/core/parser/llk/token_position", GINT_TO_POINTER(PB_LLk), test_token_position);
-  g_test_add_data_func("/core/parser/llk/iterative", GINT_TO_POINTER(PB_LLk), test_iterative);
+  g_test_add_data_func("/core/parser/llk/iterative/single", GINT_TO_POINTER(PB_LLk), test_iterative_single);
+  g_test_add_data_func("/core/parser/llk/iterative/multi", GINT_TO_POINTER(PB_LLk), test_iterative_multi);
   g_test_add_data_func("/core/parser/llk/iterative/lookahead", GINT_TO_POINTER(PB_LLk), test_iterative_lookahead);
   g_test_add_data_func("/core/parser/llk/iterative/result_length", GINT_TO_POINTER(PB_LLk), test_iterative_result_length);
   g_test_add_data_func("/core/parser/llk/drop_from", GINT_TO_POINTER(PB_LLk), test_drop_from);
@@ -1064,7 +1192,8 @@ void register_parser_tests(void) {
   g_test_add_data_func("/core/parser/lalr/rightrec", GINT_TO_POINTER(PB_LALR), test_rightrec);
   g_test_add_data_func("/core/parser/lalr/result_length", GINT_TO_POINTER(PB_LALR), test_result_length);
   g_test_add_data_func("/core/parser/lalr/token_position", GINT_TO_POINTER(PB_LALR), test_token_position);
-  g_test_add_data_func("/core/parser/lalr/iterative", GINT_TO_POINTER(PB_LALR), test_iterative);
+  g_test_add_data_func("/core/parser/lalr/iterative/single", GINT_TO_POINTER(PB_LALR), test_iterative_single);
+  g_test_add_data_func("/core/parser/lalr/iterative/multi", GINT_TO_POINTER(PB_LALR), test_iterative_multi);
   g_test_add_data_func("/core/parser/lalr/iterative/lookahead", GINT_TO_POINTER(PB_LALR), test_iterative_lookahead);
   g_test_add_data_func("/core/parser/lalr/iterative/result_length", GINT_TO_POINTER(PB_LALR), test_iterative_result_length);
   g_test_add_data_func("/core/parser/lalr/drop_from", GINT_TO_POINTER(PB_LALR), test_drop_from);

src/t_regression.c

 #include <glib.h>
 #include <stdint.h>
+#include <stdlib.h>
 #include "glue.h"
 #include "hammer.h"
 #include "test_suite.h"
@@ -384,6 +385,16 @@ static void test_issue91() {
   g_check_cmp_int(r, ==, -2);
 }
 
+// a different instance of issue 91
+static void test_issue87() {
+  HParser *a = h_ch('a');
+  HParser *a2 = h_ch_range('a', 'a');
+  HParser *p = h_many(h_many(h_choice(a, a2, NULL)));
+
+  int r = h_compile(p, PB_LALR, NULL);
+  g_check_cmp_int(r, ==, -2);
+}
+
 static void test_issue92() {
   HParser *a = h_ch('a');
   HParser *b = h_ch('b');
@@ -462,6 +473,176 @@ static void test_issue83() {
   g_check_cmp_int(r, ==, 0);
 }
 
+/*
+ * This is Meg's cut-down bug 60 test case
+ */
+
+static void test_bug60() {
+    /* There is probably an even smaller example that shows the issue */
+
+	  HParser *zed = NULL;
+	  HParser *alpha = NULL;
+	  HParser *vchar = NULL;
+	  HParser *why = NULL;
+	  HParser *plural_zed = NULL;
+	  HParser *plural_zed_zed = NULL;
+	  HParser *a_to_zed = NULL;
+	  HParser *alphas = NULL;
+	  HParser *rule = NULL;
+	  HParser *rulelist = NULL;
+	  HParser *p = NULL;
+	  HParseResult *r = NULL;
+	  int n;
+
+	  zed =  h_ch('z');
+
+	  vchar = h_ch_range(0x79, 0x7a); /* allows y and z */
+
+	  alpha = h_ch('a');
+
+	  why = h_ch('y');
+
+	  plural_zed = h_sequence(
+	      why,
+	      h_many(h_choice(alpha, vchar, NULL)),
+	      NULL);
+	  plural_zed_zed = h_choice(plural_zed, zed, NULL);
+	  alphas = h_choice(alpha, h_sequence(plural_zed_zed, alpha, NULL), NULL);
+
+	  a_to_zed = h_sequence(
+	      zed,
+	      h_many(h_sequence(h_many1(alphas), zed, NULL)),
+	      NULL);
+	  rule = h_sequence(a_to_zed, plural_zed_zed, NULL);
+	  rulelist = h_many1(h_choice(
+	      rule,
+	      h_sequence(h_many(alphas), plural_zed_zed, NULL),
+	      NULL));
+
+	  p = rulelist;
+
+	  g_check_parse_ok(p, PB_GLR, "ayzza", 5);
+	  g_check_parse_match(p, PB_GLR, "ayzza", 5, "(((u0x61) (u0x79 (u0x7a u0x7a u0x61))))");
+
+}
+
+/*
+ * This is the original bug60 test case; cut down from an ABNF parser
+ */
+
+#define BUG60_ABNF_SCAN_UP_TO 64
+
+static void test_bug60_abnf() {
+  HParser *newline = NULL;
+  HParser *alpha = NULL;
+  HParser *sp = NULL;
+  HParser *vchar = NULL;
+  HParser *equal = NULL;
+  HParser *semicolon = NULL;
+  HParser *comment = NULL;
+  HParser *c_nl = NULL;
+  HParser *c_sp = NULL;
+  HParser *defined_as = NULL;
+  HParser *alphas = NULL;
+  HParser *rule = NULL;
+  HParser *rulelist = NULL;
+  HParser *p = NULL;
+  int i, j, r, s_size;
+  char *s = NULL;
+  const char *test_string_template = "x = y z%s;%s\n\n";
+  char buf_1[BUG60_ABNF_SCAN_UP_TO+1];
+  char buf_2[2*BUG60_ABNF_SCAN_UP_TO+1];
+
+  newline = h_ch('\n');
+  alpha = h_choice(h_ch_range('A', 'Z'), h_ch_range('a', 'z'), NULL);
+  sp = h_ch(' ');
+  vchar = h_ch_range(0x21, 0x7e);
+  equal = h_ch('=');
+  semicolon = h_ch(';');
+  comment = h_sequence(
+    semicolon,
+    h_many(h_choice(sp, vchar, NULL)),
+    newline,
+    NULL);
+  c_nl = h_choice(comment, newline, NULL);
+  c_sp = h_choice(sp, h_sequence(c_nl, sp, NULL), NULL);
+  defined_as = h_sequence(h_many(c_sp), equal, h_many(c_sp), NULL);
+  alphas = h_sequence(
+    alpha,
+    h_many(h_sequence(h_many1(c_sp), alpha, NULL)),
+    h_many(c_sp),
+    NULL);
+  rule = h_sequence(alpha, defined_as, alphas, c_nl, NULL);
+  rulelist = h_many1(h_choice(
+    rule,
+    h_sequence(h_many(c_sp), c_nl, NULL),
+    NULL));
+
+  p = rulelist;
+  g_check_compilable(p, PB_GLR, 1);
+
+  /* Have a buffer for the string */
+  s_size = strlen(test_string_template) + 3*BUG60_ABNF_SCAN_UP_TO + 1;
+  s = malloc(s_size);
+  g_check_cmp_ptr(s, !=, NULL);
+
+  /*
+   * Try to parse all the different strings according to the template up to
+   * the scan limit.
+   *
+   * Correct behavior: it parses for all values of i, j.
+   * Bugged behavior: when i % 3 != 0, parse failures begin to occur at
+   * j == (i / 3) + (i % 3).
+   */
+  for (i = 0; i < BUG60_ABNF_SCAN_UP_TO; ++i) {
+    memset(buf_1, ' ', i);
+    buf_1[i] = 0;
+    for (j = 0; j < 2*BUG60_ABNF_SCAN_UP_TO; ++j) {
+      memset(buf_2, 'x', j);
+      buf_2[j] = 0;
+      snprintf(s, s_size, test_string_template, buf_1, buf_2);
+      g_check_parse_ok_no_compile(p, s, strlen(s));
+    }
+  }
+
+  free(s);
+}
+
+static void test_bug95_lalr_k_param() {
+
+/*
+ * The LR backends had been left hardcoded to k = 1 and were ignoring the param value
+ * This example compiles for LALR(k) where k = 2
+ * but does not compile for k = 1
+ * This shows that the parameter is being used rather than ignored.
+ * The default value of k when the param == NULL is set to k = 1
+ */
+  HParser *a = h_ch('a');
+  HParser *b = h_ch('b');
+  HParser *c = h_ch('c');
+  HParser *B = h_indirect();
+  HParser *A = h_sequence (a, B, c, NULL);
+  HParser *B_ = h_choice(B, b, NULL);
+  h_bind_indirect(B, B_);
+
+  HParser *p = A;
+
+  if (h_compile(p, PB_LALR, (void *)1) == 0) {
+    g_test_message("should not compile for lalr(1)");
+    g_test_fail();
+  }
+
+  g_check_compilable(p, PB_LALR, 2);
+
+  g_check_parse_match_no_compile(p, "abc",3, "(u0x61 u0x62 u0x63)");
+
+  /* The next test shows that the default works for compile with param == NULL */
+  /* (default is k = 1) */
+
+  g_check_parse_match(h_sequence(a, b, c, NULL), PB_LALR, "abc",3, "(u0x61 u0x62 u0x63)");
+
+}
+
 void register_regression_tests(void) {
   g_test_add_func("/core/regression/bug118", test_bug118);
   g_test_add_func("/core/regression/seq_index_path", test_seq_index_path);
@@ -475,6 +656,10 @@ void register_regression_tests(void) {
   g_test_add_func("/core/regression/flatten_null", test_flatten_null);
   //XXX g_test_add_func("/core/regression/ast_length_index", test_ast_length_index);
   g_test_add_func("/core/regression/issue91", test_issue91);
+  g_test_add_func("/core/regression/issue87", test_issue87);
   g_test_add_func("/core/regression/issue92", test_issue92);
   g_test_add_func("/core/regression/issue83", test_issue83);
+  g_test_add_func("/core/regression/bug60", test_bug60);
+  g_test_add_func("/core/regression/bug60_abnf", test_bug60_abnf);
+  g_test_add_func("/core/regression/bug95_lalrk_param", test_bug95_lalr_k_param);
 }

src/test_suite.c

@@ -25,6 +25,7 @@ extern void register_parser_tests();
 extern void register_grammar_tests();
 extern void register_misc_tests();
 extern void register_mm_tests();
+extern void register_names_tests();
 extern void register_benchmark_tests();
 extern void register_regression_tests();
 
@@ -38,6 +39,7 @@ int main(int argc, char** argv) {
   register_grammar_tests();
   register_misc_tests();
   register_mm_tests();
+  register_names_tests();
   register_regression_tests();
   if (g_test_slow() || g_test_perf())
     register_benchmark_tests();

src/test_suite.h

@@ -35,6 +35,18 @@
     }									\
   } while(0)
 
+/* Comparison for ptr types; only == and != will work */
+#define g_check_cmp_ptr(p1, op, p2) do { \
+    const void *_p1 = (p1); \
+    const void *_p2 = (p2); \
+    if (!(_p1 op _p2)) { \
+      g_test_message("Check failed: (%s): (%p %s %p)", \
+        #p1 " " #op " " #p2, \
+        _p1, #op, _p2); \
+      g_test_fail(); \
+    } \
+  } while (0);
+
 #define g_check_bytes(len, n1, op, n2) do {	\
     const uint8_t *_n1 = (n1);			\
     const uint8_t *_n2 = (n2);			\
@@ -56,6 +68,32 @@
     }							\
   } while(0)
 
+#define g_check_maybe_string_eq(n1, n2) do { \
+    const char *_n1 = (n1); \
+    const char *_n2 = (n2); \
+    if (_n1 != _n2 && _n1 != NULL && _n2 != NULL) { \
+      if (!(strcmp(_n1, _n2) == 0)) { \
+        g_test_message("Check failed: (%s) (\"%s\" == \"%s\")",	\
+		                   #n1 " == " #n2, _n1, _n2); \
+        g_test_fail(); \
+      } \
+    } else { \
+      if (_n1 != NULL || _n2 != NULL) { \
+        if (_n1 != NULL && _n2 == NULL) { \
+          g_test_message("Check failed: (%s) (\"%s\" == NULL)", \
+                         #n1 " == " #n2, _n1); \
+          g_test_fail(); \
+        } else if (_n1 == NULL && _n2 != NULL) { \
+          g_test_message("Check failed: (%s) (NULL == \"%s\")", \
+                         #n1 " == " #n2, _n2); \
+          g_test_fail(); \
+        } \
+        /* else both are not-NULL, but point to the same string - succeed */ \
+      } \
+      /* else both are NULL, succeed */ \
+    } \
+  } while(0)
+
 #define g_check_regular(lang) do {			\
     if (!lang->isValidRegular(lang->env)) {		\
       g_test_message("Language is not regular");	\
@@ -71,24 +109,36 @@
   } while(0)
 
 #define g_check_compilable(lang, backend, params) do {	\
-    if (!h_compile(lang, backend, params)) {		\
-      g_test_message("Language is not %s(%s)", #backend, params);	\
+    int r = h_compile((HParser *)(lang), (HParserBackend)(backend), (void *)(params)); \
+    if (r != 0) {		\
+      g_test_message("Language is not %s(%s)", #backend, #params);	\
       g_test_fail();					\
     }							\
   } while(0)
 
-  
+#define print_arena_stats(arena) do {					\
+    if (g_test_verbose()) {						\
+      HArenaStats stats;						\
+      h_allocator_stats(arena, &stats);					\
+      g_test_message("Parse used %zd bytes, wasted %zd bytes. "		\
+                     "Inefficiency: %5f%%",				\
+                     stats.used, stats.wasted,				\
+                     stats.wasted * 100. / (stats.used+stats.wasted));	\
+    }									\
+  } while(0)
+
 #define g_check_parse_failed__m(mm__, parser, backend, input, inp_len) do { \
     int skip = h_compile__m(mm__, (HParser *)(parser), (HParserBackend)backend, NULL); \
     if(skip != 0) {	\
-      g_test_message("Compile failed");					\
+      g_test_message("Compile failed on line %d", __LINE__);					\
       g_test_fail();							\
       break;	\
     }	\
-    HParseResult *result = h_parse__m(mm__, parser, (const uint8_t*)input, inp_len); \
-    if (NULL != result) {						\
-      h_parse_result_free(result);					\
-      g_test_message("Check failed: shouldn't have succeeded, but did"); \
+    HParseResult *res = h_parse__m(mm__, parser, (const uint8_t*)input, inp_len); \
+    if (NULL != res) {						\
+      print_arena_stats(res->arena); \
+      h_parse_result_free(res);					\
+      g_test_message("Check failed: parse shouldn't have succeeded, but did on line %d", __LINE__); \
       g_test_fail();							\
     }									\
   } while(0)
@@ -96,25 +146,27 @@
 #define g_check_parse_failed(p, be, input, len)				\
     g_check_parse_failed__m(&system_allocator, p, be, input, len)
 
-#define print_arena_stats(arena) do {					\
-    if (g_test_verbose()) {						\
-      HArenaStats stats;						\
-      h_allocator_stats(arena, &stats);					\
-      g_test_message("Parse used %zd bytes, wasted %zd bytes. "		\
-                     "Inefficiency: %5f%%",				\
-                     stats.used, stats.wasted,				\
-                     stats.wasted * 100. / (stats.used+stats.wasted));	\
+#define g_check_parse_failed_no_compile__m(mm__, parser, input, inp_len) do { \
+    HParseResult *res = h_parse__m(mm__, parser, (const uint8_t*)input, inp_len); \
+    if (NULL != res) {						\
+      print_arena_stats(res->arena); \
+      h_parse_result_free(res);					\
+      g_test_message("Check failed: shouldn't have succeeded, but did on ;ine %d", __LINE__); \
+      g_test_fail();							\
     }									\
   } while(0)
 
-#define g_check_parse_ok(parser, backend, input, inp_len) do {		\
-    int skip = h_compile((HParser *)(parser), (HParserBackend) backend, NULL); \
+#define g_check_parse_failed_no_compile(p, input, len)				\
+    g_check_parse_failed__m(&system_allocator, p, input, len)
+
+#define g_check_parse_ok__m(mm__, parser, backend, input, inp_len) do {		\
+    int skip = h_compile__m(mm__, (HParser *)(parser), (HParserBackend) backend, NULL); \
     if(skip) {								\
       g_test_message("Compile failed");					\
       g_test_fail();							\
       break;								\
     }									\
-    HParseResult *res = h_parse(parser, (const uint8_t*)input, inp_len); \
+    HParseResult *res = h_parse__m(mm__, parser, (const uint8_t*)input, inp_len); \
     if (!res) {								\
       g_test_message("Parse failed on line %d", __LINE__);		\
       g_test_fail();							\
@@ -124,14 +176,48 @@
     }									\
   } while(0)
 
-#define g_check_parse_match(parser, backend, input, inp_len, result) do { \
-    int skip = h_compile((HParser *)(parser), (HParserBackend) backend, NULL); \
+#define g_check_parse_ok(p, be, input, len)				\
+    g_check_parse_ok__m(&system_allocator, p, be, input, len)
+
+#define g_check_parse_ok_no_compile__m(mm__, parser, input, inp_len) do {		\
+    HParseResult *res = h_parse__m(mm__, parser, (const uint8_t*)input, inp_len); \
+    if (!res) {								\
+      g_test_message("Parse failed on line %d", __LINE__);		\
+      g_test_fail();							\
+    } else {								\
+      print_arena_stats(res->arena);					\
+      h_parse_result_free(res);						\
+    }									\
+  } while(0)
+
+#define g_check_parse_ok_no_compile(p, input, len)				\
+    g_check_parse_ok_no_compile__m(&system_allocator, p, input, len)
+
+#define g_check_parse_match__m(mm__, parser, backend, input, inp_len, result) do { \
+    int skip = h_compile__m(mm__, (HParser *)(parser), (HParserBackend) backend, NULL); \
     if(skip) {								\
       g_test_message("Compile failed");					\
       g_test_fail();							\
       break;								\
     }									\
-    HParseResult *res = h_parse(parser, (const uint8_t*)input, inp_len); \
+    HParseResult *res = h_parse__m(mm__, (HParser *)(parser), (const uint8_t*)input, inp_len); \
+    if (!res) {								\
+      g_test_message("Parse failed on line %d", __LINE__);		\
+      g_test_fail();							\
+    } else {								\
+      char* cres = h_write_result_unamb(res->ast);			\
+      g_check_string(cres, ==, result);					\
+      (&system_allocator)->free(&system_allocator, cres);		\
+      print_arena_stats(res->arena);					\
+      h_parse_result_free(res);						\
+    }									\
+  } while(0)
+
+#define g_check_parse_match(parser, backend, input, inp_len, result) \
+    g_check_parse_match__m(&system_allocator, parser, backend, input, inp_len, result)
+
+#define g_check_parse_match_no_compile__m(mm__, parser, input, inp_len, result) do { \
+    HParseResult *res = h_parse__m(mm__, (HParser *)(parser), (const uint8_t*)input, inp_len); \
     if (!res) {								\
       g_test_message("Parse failed on line %d", __LINE__);		\
       g_test_fail();							\
@@ -144,6 +230,9 @@
     }									\
   } while(0)
 
+#define g_check_parse_match_no_compile(parser, input, inp_len, result) \
+    g_check_parse_match_no_compile__m(&system_allocator, parser, input, inp_len, result)
+
 #define g_check_parse_chunks_failed__m(mm__, parser, backend, chunk1, c1_len, chunk2, c2_len) do { \
     int skip = h_compile__m(mm__, (HParser *)(parser), (HParserBackend)backend, NULL); \
     if(skip) {								\
@@ -155,7 +244,7 @@
   } while(0)
 
 #define g_check_parse_chunks_failed___m(mm__, parser, chunk1, c1_len, chunk2, c2_len) do { \
-    HSuspendedParser *s = h_parse_start__m(mm__, parser);		\
+    HSuspendedParser *s = h_parse_start__m(mm__, (HParser *)(parser));		\
     if(!s) {								\
       g_test_message("Chunk-wise parsing not available");		\
       g_test_fail();							\
@@ -188,7 +277,7 @@
   } while(0)
 
 #define g_check_parse_chunks_ok_(parser, chunk1, c1_len, chunk2, c2_len) do {	\
-    HSuspendedParser *s = h_parse_start(parser);			\
+    HSuspendedParser *s = h_parse_start((HParser *)(parser));			\
     if(!s) {								\
       g_test_message("Chunk-wise parsing not available");		\
       g_test_fail();							\
@@ -217,7 +306,7 @@
   } while(0)
 
 #define g_check_parse_chunks_match_(parser, chunk1, c1_len, chunk2, c2_len, result) do { \
-    HSuspendedParser *s = h_parse_start(parser);			\
+    HSuspendedParser *s = h_parse_start((HParser *)(parser));			\
     if(!s) {								\
       g_test_message("Chunk-wise parsing not available");		\
       g_test_fail();							\
@@ -238,6 +327,97 @@
     }									\
   } while(0)
 
+#define g_check_parse_chunk_failed__m(mm__, parser, backend, chunk1, c1_len) do { \
+    int skip = h_compile__m(mm__, (HParser *)(parser), (HParserBackend)backend, NULL); \
+    if(skip) {								\
+      g_test_message("Compile failed");					\
+      g_test_fail();							\
+      break;								\
+    }									\
+    g_check_parse_chunk_failed___m(mm__, parser, chunk1, c1_len); \
+  } while(0)
+
+#define g_check_parse_chunk_failed___m(mm__, parser, chunk1, c1_len) do { \
+    HSuspendedParser *s = h_parse_start__m(mm__, (HParser *)(parser));		\
+    if(!s) {								\
+      g_test_message("Chunk-wise parsing not available");		\
+      g_test_fail();							\
+      break;								\
+    }									\
+    h_parse_chunk(s, (const uint8_t*)chunk1, c1_len);			\
+    HParseResult *res = h_parse_finish(s);				\
+    if (NULL != res) {							\
+      h_parse_result_free(res);						\
+      g_test_message("Check failed: shouldn't have succeeded, but did"); \
+      g_test_fail();							\
+    }									\
+  } while(0)
+
+#define g_check_parse_chunk_failed(p, be, c1, c1_len) \
+  g_check_parse_chunk_failed__m(&system_allocator, p, be, c1, c1_len)
+
+#define g_check_parse_chunk_failed_(p, c1, c1_len) \
+  g_check_parse_chunk_failed___m(&system_allocator, p, c1, c1_len)
+
+#define g_check_parse_chunk_ok(parser, backend, chunk1, c1_len) do {	\
+    int skip = h_compile((HParser *)(parser), (HParserBackend)backend, NULL); \
+    if(skip) {								\
+      g_test_message("Compile failed");					\
+      g_test_fail();							\
+      break;								\
+    }									\
+    g_check_parse_chunk_ok_(parser, chunk1, c1_len); \
+  } while(0)
+
+#define g_check_parse_chunk_ok_(parser, chunk1, c1_len) do {	\
+    HSuspendedParser *s = h_parse_start((HParser *)(parser));			\
+    if(!s) {								\
+      g_test_message("Chunk-wise parsing not available");		\
+      g_test_fail();							\
+      break;								\
+    }									\
+    h_parse_chunk(s, (const uint8_t*)chunk1, c1_len);			\
+    HParseResult *res = h_parse_finish(s);				\
+    if (!res) {								\
+      g_test_message("Parse failed on line %d", __LINE__);		\
+      g_test_fail();							\
+    } else {								\
+      print_arena_stats(res->arena);					\
+      h_parse_result_free(res);						\
+    }									\
+  } while(0)
+
+#define g_check_parse_chunk_match(parser, backend, chunk1, c1_len, result) do { \
+    int skip = h_compile((HParser *)(parser), (HParserBackend) backend, NULL); \
+    if(skip) {								\
+      g_test_message("Compile failed");					\
+      g_test_fail();							\
+      break;								\
+    }									\
+    g_check_parse_chunk_match_(parser, chunk1, c1_len, result); \
+  } while(0)
+
+#define g_check_parse_chunk_match_(parser, chunk1, c1_len, result) do { \
+    HSuspendedParser *s = h_parse_start((HParser *)(parser));			\
+    if(!s) {								\
+      g_test_message("Chunk-wise parsing not available");		\
+      g_test_fail();							\
+      break;								\
+    }									\
+    h_parse_chunk(s, (const uint8_t*)chunk1, c1_len);			\
+    HParseResult *res = h_parse_finish(s);				\
+    if (!res) {								\
+      g_test_message("Parse failed on line %d", __LINE__);		\
+      g_test_fail();							\
+    } else {								\
+      char* cres = h_write_result_unamb(res->ast);			\
+      g_check_string(cres, ==, result);					\
+      (&system_allocator)->free(&system_allocator, cres);		\
+      print_arena_stats(res->arena);					\
+      h_parse_result_free(res);						\
+    }									\
+  } while(0)
+
 #define g_check_hashtable_present(table, key) do {			\
     if(!h_hashtable_present(table, key)) {				\
       g_test_message("Check failed: key should have been in table, but wasn't"); \
@@ -313,7 +493,6 @@
 #define g_check_cmp_uint32(n1, op, n2) g_check_inttype("%u", uint32_t, n1, op, n2)
 #define g_check_cmp_uint64(n1, op, n2) g_check_inttype("%" PRIu64, uint64_t, n1, op, n2)
 #define g_check_cmp_size(n1, op, n2) g_check_inttype("%zu", size_t, n1, op, n2)
-#define g_check_cmp_ptr(n1, op, n2) g_check_inttype("%p", void *, n1, op, n2)
 #define g_check_cmpfloat(n1, op, n2) g_check_inttype("%g", float, n1, op, n2)
 #define g_check_cmpdouble(n1, op, n2) g_check_inttype("%g", double, n1, op, n2)