.gitignore

+# generated files
 *.o
-*~
+*.os
+*.so
 *.a
+*.gem
+*.pyc
 *.class
-*.so
-jni/com*.h
-src/test_suite
-lib/hush
+libhammer.pc
+build/
 examples/dns
 examples/base64
 examples/base64_sem1
 examples/base64_sem2
-TAGS
-*.swp
-*.swo
+jni/com*.h
+src/test_suite
+
+# coverage and profiling stuff
+*.gcov
+*.gcda
+*.gcno
+gmon.out
+
+# editor leftovers
+*~
+*.sw?
 \#*
+
+# misc
+lib/hush
+TAGS
 docs/milestone2.dot.pdf
 *.dot.pdf
 Session.vim
-*.gcov
 cscope.out
-build/
-libhammer.pc
 .sconsign.dblite
-*.os
-*.pyc
-*.gem

Makefile

@@ -3,7 +3,7 @@
 # and kick off a recursive make
 # Also, "make src/all" turns into "make -C src all"
 
-SUBDIRS = src examples jni
+SUBDIRS = src examples src/bindings/jni
 
 include config.mk
 TOPLEVEL=.

README.md

@@ -44,7 +44,10 @@ Installing
 * mono-devel and mono-mcs (>= 3.0.6) (for .NET bindings)
 * [nunit](http://www.nunit.org/) (for testing .NET bindings)
 
-To build, type `scons`. To run the built-in test suite, type `scons test`. For a debug build, add `--variant=debug`.
+To build, type `scons`.
+To run the built-in test suite, type `scons test`.
+To avoid the test dependencies, add `--no-tests`.
+For a debug build, add `--variant=debug`.
 
 To build bindings, pass a "bindings" argument to scons, e.g. `scons bindings=python`. `scons bindings=python test` will build Python bindings and run tests for both C and Python. `--variant=debug` is valid here too. You can build more than one set of bindings at a time; just separate them with commas, e.g. `scons bindings=python,perl`.
 

SConstruct

@@ -5,6 +5,7 @@ from __future__ import absolute_import, division, print_function
 import os
 import os.path
 import platform
+import subprocess
 import sys
 
 default_install_dir='/usr/local'
@@ -14,13 +15,24 @@ if platform.system() == 'Windows':
 vars = Variables(None, ARGUMENTS)
 vars.Add(PathVariable('DESTDIR', "Root directory to install in (useful for packaging scripts)", None, PathVariable.PathIsDirCreate))
 vars.Add(PathVariable('prefix', "Where to install in the FHS", "/usr/local", PathVariable.PathAccept))
-vars.Add(ListVariable('bindings', 'Language bindings to build', 'none', ['cpp', 'dotnet', 'perl', 'php', 'python', 'ruby']))
+vars.Add(ListVariable('bindings', 'Language bindings to build', 'none', ['cpp', 'dotnet', 'jni', 'perl', 'php', 'python', 'ruby']))
 vars.Add('python', 'Python interpreter', 'python')
 
 tools = ['default', 'scanreplace']
 if 'dotnet' in ARGUMENTS.get('bindings', []):
 	tools.append('csharp/mono')
 
+# add the clang tool if necessary
+if os.getenv('CC') == 'clang' or platform.system() == 'Darwin':
+	tools.append('clang')
+else:
+	# try to detect if cc happens to be clang by inspecting --version
+	cc = os.getenv('CC') or 'cc'
+	ver = subprocess.run([cc, '--version'], capture_output=True).stdout
+	if b'clang' in ver.split():
+		tools.append('clang')
+		os.environ['CC'] = cc	# make sure we call it as we saw it
+
 envvars = {'PATH' : os.environ['PATH']}
 if 'PKG_CONFIG_PATH' in os.environ:
     envvars['PKG_CONFIG_PATH'] = os.environ['PKG_CONFIG_PATH']
@@ -73,24 +85,33 @@ AddOption('--coverage',
           action='store_true',
           help='Build with coverage instrumentation')
 
+AddOption('--force-debug',
+          dest='force_debug',
+          default=False,
+          action='store_true',
+          help='Build with debug symbols, even in the opt variant')
+
+AddOption('--gprof',
+          dest='gprof',
+          default=False,
+          action="store_true",
+          help='Build with profiling instrumentation for gprof')
+
 AddOption('--in-place',
           dest='in_place',
           default=False,
           action='store_true',
           help='Build in-place, rather than in the build/<variant> tree')
 
-AddOption('--tests',
+AddOption('--no-tests',
           dest='with_tests',
           default=env['PLATFORM'] != 'win32',
-          action='store_true',
-          help='Build tests')
+          action='store_false',
+          help='Do not build tests')
 
 env['CC'] = os.getenv('CC') or env['CC']
 env['CXX'] = os.getenv('CXX') or env['CXX']
-
-if os.getenv('CC') == 'clang' or env['PLATFORM'] == 'darwin':
-    env.Replace(CC='clang',
-                CXX='clang++')
+env['CFLAGS'] = os.getenv('CFLAGS') or env['CFLAGS']
 
 # Language standard and warnings
 if env['CC'] == 'cl':
@@ -106,7 +127,15 @@ if env['CC'] == 'cl':
         ]
     )
 else:
-    env.MergeFlags('-std=c99 -D_POSIX_C_SOURCE=200809L -Wall -Wextra -Werror -Wno-unused-parameter -Wno-attributes -Wno-unused-variable')
+    if env['PLATFORM'] == 'darwin':
+        # It's reported -D_POSIX_C_SOURCE breaks the Mac OS build; I think we
+        # may need _DARWIN_C_SOURCE instead/in addition to, but let's wait to
+        # have access to a Mac to test/repo
+        env.MergeFlags('-std=c99 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-attributes -Wno-unused-variable')
+    else:
+        # Using -D_POSIX_C_SOURCE=200809L here, not on an ad-hoc basis when,
+        # #including, is important
+        env.MergeFlags('-std=c99 -D_POSIX_C_SOURCE=200809L -Wall -Wextra -Werror -Wno-unused-parameter -Wno-attributes -Wno-unused-variable')
 
 # Linker options
 if env['PLATFORM'] == 'darwin':
@@ -120,14 +149,31 @@ else:
     env.MergeFlags('-lrt')
 
 if GetOption('coverage'):
-    env.Append(CFLAGS=['--coverage'],
-               CXXFLAGS=['--coverage'],
-               LDFLAGS=['--coverage'])
+    env.Append(CCFLAGS=['--coverage'],
+               LDFLAGS=['--coverage'],
+               LINKFLAGS=['--coverage'])
     if env['CC'] == 'gcc':
         env.Append(LIBS=['gcov'])
     else:
         env.ParseConfig('llvm-config --ldflags')
 
+if GetOption('force_debug'):
+    if env['CC'] == 'cl':
+        env.Append(CCFLAGS=['/Z7'])
+    else:
+        env.Append(CCFLAGS=['-g'])
+
+if GetOption('gprof'):
+    if env['CC'] == 'gcc' and env['CXX'] == 'g++':
+        env.Append(CCFLAGS=['-pg'],
+		   LDFLAGS=['-pg'],
+                   LINKFLAGS=['-pg'])
+        env['GPROF'] = 1
+    else:
+        print("Can only use gprof with gcc")
+        Exit(1)
+        
+
 dbg = env.Clone(VARIANT='debug')
 if env['CC'] == 'cl':
     dbg.Append(CCFLAGS=['/Z7'])

examples/SConscript

@@ -3,11 +3,18 @@ from __future__ import absolute_import, division, print_function
 Import('env')
 
 example = env.Clone()
-example.Append(LIBS="hammer", LIBPATH="../src")
+
+if 'GPROF' in env and env['GPROF'] == 1:
+    hammer_lib_name="hammer_pg"
+else:
+    hammer_lib_name="hammer"
+
+example.Append(LIBS=hammer_lib_name, LIBPATH="../src")
 
 dns = example.Program('dns', ['dns.c', 'rr.c', 'dns_common.c'])
+ttuser = example.Program('ttuser', 'ttuser.c')
 base64 = example.Program('base64', 'base64.c')
 base64_sem1 = example.Program('base64_sem1', 'base64_sem1.c')
 base64_sem2 = example.Program('base64_sem2', 'base64_sem2.c')
 ties = example.Program('ties', ['ties.c', 'grammar.c'])
-env.Alias("examples", [dns, base64, base64_sem1, base64_sem2, ties])
\ No newline at end of file
+env.Alias("examples", [dns, ttuser, base64, base64_sem1, base64_sem2, ties])

examples/base64.c

@@ -11,8 +11,6 @@
 #include <inttypes.h>
 #include "../src/hammer.h"
 
-#define DEBUG
-
 const HParser* document = NULL;
 
 void init_parser(void)
@@ -27,65 +25,17 @@ void init_parser(void)
     HParser *equals = h_ch('=');
 
     HParser *bsfdig = h_choice(alpha, digit, plus, slash, NULL);
-    HParser *bsfdig_4bit = h_choice(
-        h_ch('A'), h_ch('E'), h_ch('I'), h_ch('M'), h_ch('Q'), h_ch('U'),
-        h_ch('Y'), h_ch('c'), h_ch('g'), h_ch('k'), h_ch('o'), h_ch('s'),
-        h_ch('w'), h_ch('0'), h_ch('4'), h_ch('8'), NULL);
-    HParser *bsfdig_2bit = h_choice(h_ch('A'), h_ch('Q'), h_ch('g'), h_ch('w'), NULL);
-
-    HParser *base64_quad = h_sequence(bsfdig, bsfdig, bsfdig, bsfdig, NULL);
-    HParser *base64_quads = h_many(base64_quad);
-
-    HParser *base64_2 = h_sequence(bsfdig, bsfdig, bsfdig_4bit, equals, h_end_p(), NULL);
-    HParser *base64_1 = h_sequence(bsfdig, bsfdig_2bit, equals, equals, h_end_p(), NULL);
-    HParser *base64_ending = h_choice(h_end_p(), base64_2, base64_1, NULL);
-    HParser *base64 = h_sequence(base64_quads, base64_ending, NULL);
-    // why does this parse "A=="?!
-    // why does this parse "aaA=" but not "aA=="?!
-
-    document = base64;
-}
-
-
-#include <string.h>
-#include <assert.h>
-#define TRUE (1)
-#define FALSE (0)
-
-void assert_parse(int expected, char *data) {
-    HParseResult *result;
-
-    size_t datasize = strlen(data);
-    result = h_parse(document, (void*)data, datasize);
-    if((result != NULL) != expected) {
-        fprintf(stderr, "Test failed: %s\n", data);
-    }
-#ifdef DEBUG
-    else {
-        fprintf(stderr, "Test succeeded: %s\n", data);
-    }
-
-    if (result != NULL) {
-        fprintf(stderr, "parsed=%" PRId64 " bytes\n", result->bit_length / 8);
-        h_pprint(stdout, result->ast, 0, 0);
-    }
-#endif
-
-    h_parse_result_free(result);
-}
-
-void test() {
-    assert_parse(TRUE, "");
-    assert_parse(TRUE, "YQ==");
-    assert_parse(TRUE, "YXU=");
-    assert_parse(TRUE, "YXVy");
-    assert_parse(TRUE, "QVVSIFNBUkFG");
-    assert_parse(TRUE, "QVVSIEhFUlUgU0FSQUY=");
-    assert_parse(FALSE, "A");
-    assert_parse(FALSE, "A=");
-    assert_parse(FALSE, "A==");
-    assert_parse(FALSE, "AAA==");
-    assert_parse(FALSE, "aa==");
+    HParser *bsfdig_4bit = h_in((uint8_t *)"AEIMQUYcgkosw048", 16);
+    HParser *bsfdig_2bit = h_in((uint8_t *)"AQgw", 4);
+    HParser *base64_3 = h_repeat_n(bsfdig, 4);
+    HParser *base64_2 = h_sequence(bsfdig, bsfdig, bsfdig_4bit, equals, NULL);
+    HParser *base64_1 = h_sequence(bsfdig, bsfdig_2bit, equals, equals, NULL);
+    HParser *base64 = h_sequence(h_many(base64_3),
+                                       h_optional(h_choice(base64_2,
+                                                           base64_1, NULL)),
+                                       NULL);
+
+    document = h_sequence(h_whitespace(base64), h_whitespace(h_end_p()), NULL);
 }
 
 
@@ -99,8 +49,6 @@ int main(int argc, char **argv)
 
     init_parser();
 
-    test();
-
     inputsize = fread(input, 1, sizeof(input), stdin);
     fprintf(stderr, "inputsize=%zu\ninput=", inputsize);
     fwrite(input, 1, inputsize, stderr);

examples/ttuser.c

+/*
+ * Example parser that demonstrates the use of user-defined token types.
+ *
+ * Note the custom printer function that hooks into h_pprint().
+ */
+
+#include "../src/hammer.h"
+#include "../src/glue.h"
+
+
+/*
+ * custom tokens
+ */
+
+HTokenType TT_SUBJ, TT_PRED, TT_OBJ, TT_ADJ, TT_ADVC;
+
+void
+pprint(FILE *stream, const HParsedToken *tok, int indent, int delta)
+{
+	/* 
+	 * Pretty-printer rules:
+	 *
+	 *  - Output 'indent' spaces after every newline you produce.
+	 *  - Do not add indent on the first line of output.
+	 *  - Do not add a trailing newline.
+	 *  - Indent sub-objects by adding 'delta' to 'indent'.
+	 */
+
+	if (((HParsedToken *)tok->user)->token_type == TT_SEQUENCE)
+		fprintf(stream, "\n%*s", indent, "");
+	h_pprint(stream, tok->user, indent, delta);
+}
+
+/* XXX define umamb_sub as well */
+
+void
+init(void)
+{
+	TT_SUBJ = h_allocate_token_new("subject", NULL, pprint);
+	TT_PRED = h_allocate_token_new("predicate", NULL, pprint);
+	TT_OBJ  = h_allocate_token_new("object", NULL, pprint);
+	TT_ADJ  = h_allocate_token_new("adjective", NULL, pprint);
+	TT_ADVC = h_allocate_token_new("adverbial clause", NULL, pprint);
+}
+
+
+/*
+ * semantic actions
+ *
+ * Normally these would be more interesting, but for this example, we just wrap
+ * our tokens in their intended types.
+ */
+HParsedToken *act_subj(const HParseResult *p, void *u) {
+	return H_MAKE(SUBJ, (void *)p->ast);
+}
+HParsedToken *act_pred(const HParseResult *p, void *u) {
+	return H_MAKE(PRED, (void *)p->ast);
+}
+HParsedToken *act_obj(const HParseResult *p, void *u) {
+	return H_MAKE(OBJ, (void *)p->ast);
+}
+HParsedToken *act_adj(const HParseResult *p, void *u) {
+	return H_MAKE(ADJ, (void *)p->ast);
+}
+HParsedToken *act_advc(const HParseResult *p, void *u) {
+	return H_MAKE(ADVC, (void *)p->ast);
+}
+
+
+/*
+ * grammar
+ */
+
+HParser *
+build_parser(void)
+{
+	/* words */
+	#define W(X)	h_whitespace(h_literal((const uint8_t *)(#X)))
+	H_RULE(art,	h_choice(W(a), W(the), NULL));
+	H_RULE(noun,	h_choice(W(cat), W(dog), W(fox), W(tiger), W(lion),
+			    W(bear), W(fence), W(tree), W(car), W(cow), NULL));
+	H_RULE(verb,	h_choice(W(eats), W(jumps), W(falls), NULL));
+	H_ARULE(adj,	h_choice(W(quick), W(slow), W(happy), W(lazy), W(cyan),
+			    W(magenta), W(yellow), W(black), W(brown), NULL));
+	H_RULE(adverb,	h_choice(W(with), W(over), W(after), NULL));
+	#undef W
+
+	/* phrases */
+	H_RULE(nphrase,	h_sequence(art, h_many(adj), noun, NULL));
+
+	/* sentence structure */
+	H_ARULE(subj,	nphrase);
+	H_ARULE(pred,	verb);
+	H_ARULE(obj,	nphrase);
+	H_ARULE(advc,	h_sequence(adverb, nphrase, NULL));
+	H_RULE(sentnc,	h_sequence(subj, pred,
+			    h_optional(obj), h_optional(advc), NULL));
+
+	return sentnc;
+}
+
+
+/*
+ * main routine: read, parse, print
+ *
+ * input e.g.:
+ * "the quick brown fox jumps the fence with a cyan lion"
+ */
+
+#include <stdio.h>
+#include <inttypes.h>
+
+int
+main(int argc, char **argv)
+{
+	uint8_t input[1024];
+	size_t sz;
+	const HParser *parser;
+	const HParseResult *result;
+
+	init();
+	parser = build_parser();
+
+	sz = fread(input, 1, sizeof(input), stdin);
+	if (!feof(stdin)) {
+		fprintf(stderr, "too much input\n");
+		return 1;
+	}
+
+	result = h_parse(parser, input, sz);
+	if (!result) {
+		fprintf(stderr, "no parse\n");
+		return 1;
+	}
+
+        h_pprintln(stdout, result->ast);
+        fprintf(stderr, "consumed %" PRId64 "/%zu bytes.\n",
+	    result->bit_length / 8, sz);
+        return 0;
+}

src/SConscript

@@ -6,6 +6,9 @@ import os.path
 
 Import('env testruns')
 
+# Bump this if you break binary compatibility (e.g. renumber backends)
+hammer_shlib_version = "1.0.0"
+
 dist_headers = [
     'hammer.h',
     'allocator.h',
@@ -21,7 +24,9 @@ parsers_headers = [
 
 backends_headers = [
     'backends/regex.h',
-    'backends/contextfree.h'
+    'backends/contextfree.h',
+    'backends/missing.h',
+    'backends/params.h'
 ]
 
 parsers = ['parsers/%s.c'%s for s in
@@ -30,6 +35,7 @@ parsers = ['parsers/%s.c'%s for s in
             'attr_bool',
             'bind',
             'bits',
+            'bytes',
             'butnot',
             'ch',
             'charset',
@@ -52,10 +58,11 @@ parsers = ['parsers/%s.c'%s for s in
             'unimplemented',
             'whitespace',
             'xor',
-            'value']]
+            'value',
+            'seek']]
 
 backends = ['backends/%s.c' % s for s in
-            ['packrat', 'llk', 'regex', 'glr', 'lalr', 'lr', 'lr0']]
+            ['missing', 'packrat', 'llk', 'regex', 'glr', 'lalr', 'lr', 'lr0', 'params']]
 
 misc_hammer_parts = [
     'allocator.c',
@@ -87,6 +94,7 @@ ctests = ['t_benchmark.c',
           't_grammar.c',
           't_misc.c',
           't_mm.c',
+          't_names.c',
           't_regression.c']
 
 
@@ -98,9 +106,21 @@ if env['PLATFORM'] == 'win32':
     # prevent collision between .lib from dll and .lib for static lib
     static_library_name = 'hammer_s'
 
-libhammer_shared = env.SharedLibrary('hammer', parsers + backends + misc_hammer_parts)
-libhammer_static = env.StaticLibrary(static_library_name, parsers + backends + misc_hammer_parts)
+if 'GPROF' in env and env['GPROF'] == 1:
+    # Disable the shared library (it won't work with gprof) and rename the static one
+    build_shared_library=False
+    static_library_name = 'hammer_pg'
+
+# Markers for later
+libhammer_static = None
+libhammer_shared = None
+
 if build_shared_library:
+    libhammer_shared = env.SharedLibrary('hammer', parsers + backends + misc_hammer_parts, \
+                                     SHLIBVERSION=hammer_shlib_version)
+libhammer_static = env.StaticLibrary(static_library_name, parsers + backends + misc_hammer_parts)
+
+if libhammer_shared is not None:
     Default(libhammer_shared, libhammer_static)
     env.Install('$libpath', [libhammer_static, libhammer_shared])
 else:
@@ -115,14 +135,20 @@ env.Install('$pkgconfigpath', '../../../libhammer.pc')
 if GetOption('with_tests'):
     testenv = env.Clone()
     testenv.ParseConfig('pkg-config --cflags --libs glib-2.0')
-    testenv.Append(LIBS=['hammer'])
+    if libhammer_shared is not None:
+        testenv.Append(LIBS=['hammer'])
+    else:
+        testenv.Append(LIBS=[static_library_name])
     testenv.Prepend(LIBPATH=['.'])
     ctestexec = testenv.Program('test_suite', ctests + ['test_suite.c'], LINKFLAGS='--coverage' if testenv.GetOption('coverage') else None)
     ctest = Alias('testc', [ctestexec], ''.join(['env LD_LIBRARY_PATH=', os.path.dirname(ctestexec[0].path), ' ', ctestexec[0].path]))
     AlwaysBuild(ctest)
     testruns.append(ctest)
 
-Export('libhammer_static libhammer_shared')
+if libhammer_shared is not None:
+    Export('libhammer_static libhammer_shared')
+else:
+    Export('libhammer_static')
 
 for b in env['bindings']:
     env.SConscript(['bindings/%s/SConscript' % b])

src/allocator.c

@@ -29,28 +29,46 @@ struct arena_link {
   // For efficiency, we should probably allocate the arena links in 
   // their own slice, and link to a block directly. That can be
   // implemented later, though, with no change in interface.
-  struct arena_link *next; // It is crucial that this be the first item; so that 
-                           // any arena link can be casted to struct arena_link**.
-
+  struct arena_link *next;
   size_t free;
   size_t used;
   uint8_t rest[];
-} ;
+};
 
 struct HArena_ {
   struct arena_link *head;
   struct HAllocator_ *mm__;
+  /* does mm__ zero blocks for us? */
+  bool malloc_zeros;
   size_t block_size;
   size_t used;
   size_t wasted;
+#ifdef DETAILED_ARENA_STATS
+  size_t mm_malloc_count, mm_malloc_bytes;
+  size_t memset_count, memset_bytes;
+  size_t arena_malloc_count, arena_malloc_bytes;
+  size_t arena_su_malloc_count, arena_su_malloc_bytes;
+  size_t arena_si_malloc_count, arena_si_malloc_bytes;
+  size_t arena_lu_malloc_count, arena_lu_malloc_bytes;
+  size_t arena_li_malloc_count, arena_li_malloc_bytes;
+#endif
 
   jmp_buf *except;
 };
 
+static void * h_arena_malloc_raw(HArena *arena, size_t size, bool need_zero);
+
 void* h_alloc(HAllocator* mm__, size_t size) {
   void *p = mm__->alloc(mm__, size);
   if(!p)
-    h_platform_errx(1, "memory allocation failed (%uB requested)\n", (unsigned int)size);
+    h_platform_errx(1, "memory allocation failed (%zuB requested)\n", size);
+  return p;
+}
+
+void* h_realloc(HAllocator* mm__, void* ptr, size_t size) {
+  void *p = mm__->realloc(mm__, ptr, size);
+  if(!p)
+    h_platform_errx(1, "memory reallocation failed (%zuB requested)\n", size);
   return p;
 }
 
@@ -61,7 +79,6 @@ HArena *h_new_arena(HAllocator* mm__, size_t block_size) {
   struct arena_link *link = (struct arena_link*)h_alloc(mm__, sizeof(struct arena_link) + block_size);
   assert(ret != NULL);
   assert(link != NULL);
-  memset(link, 0, sizeof(struct arena_link) + block_size);
   link->free = block_size;
   link->used = 0;
   link->next = NULL;
@@ -69,6 +86,19 @@ HArena *h_new_arena(HAllocator* mm__, size_t block_size) {
   ret->block_size = block_size;
   ret->used = 0;
   ret->mm__ = mm__;
+#ifdef DETAILED_ARENA_STATS
+  ret->mm_malloc_count = 2;
+  ret->mm_malloc_bytes = sizeof(*ret) + sizeof(struct arena_link) + block_size;
+  ret->memset_count = 0;
+  ret->memset_bytes = 0;
+  ret->arena_malloc_count = ret->arena_malloc_bytes = 0;
+  ret->arena_su_malloc_count = ret->arena_su_malloc_bytes = 0;
+  ret->arena_si_malloc_count = ret->arena_si_malloc_bytes = 0;
+  ret->arena_lu_malloc_count = ret->arena_lu_malloc_bytes = 0;
+  ret->arena_li_malloc_count = ret->arena_li_malloc_bytes = 0;
+#endif
+  /* XXX provide a mechanism to indicate mm__ returns zeroed blocks */
+  ret->malloc_zeros = false;
   ret->wasted = sizeof(struct arena_link) + sizeof(struct HArena_) + block_size;
   ret->except = NULL;
   return ret;
@@ -90,39 +120,120 @@ static void *alloc_block(HArena *arena, size_t size)
   return block;
 }
 
-void* h_arena_malloc(HArena *arena, size_t size) {
+void * h_arena_malloc_noinit(HArena *arena, size_t size) {
+  return h_arena_malloc_raw(arena, size, false);
+}
+
+void * h_arena_malloc(HArena *arena, size_t size) {
+  return h_arena_malloc_raw(arena, size, true);
+}
+
+static void * h_arena_malloc_raw(HArena *arena, size_t size,
+                                 bool need_zero) {
+  struct arena_link *link = NULL;
+  void *ret = NULL;
+
   if (size <= arena->head->free) {
-    // fast path..
-    void* ret = arena->head->rest + arena->head->used;
+    /* fast path.. */
+    ret = arena->head->rest + arena->head->used;
     arena->used += size;
     arena->wasted -= size;
     arena->head->used += size;
     arena->head->free -= size;
-    return ret;
+
+#ifdef DETAILED_ARENA_STATS
+    ++(arena->arena_malloc_count);
+    arena->arena_malloc_bytes += size;
+    if (need_zero) {
+      ++(arena->arena_si_malloc_count);
+      arena->arena_si_malloc_bytes += size;
+    } else {
+      ++(arena->arena_su_malloc_count);
+      arena->arena_su_malloc_bytes += size;
+    }
+#endif
   } else if (size > arena->block_size) {
-    // We need a new, dedicated block for it, because it won't fit in a standard sized one.
-    // This involves some annoying casting...
-    arena->used += size;
-    arena->wasted += sizeof(struct arena_link*);
-    void* link = alloc_block(arena, size + sizeof(struct arena_link*));
+    /*
+     * We need a new, dedicated block for it, because it won't fit in a
+     * standard sized one.
+     *
+     * NOTE:
+     *
+     * We used to do a silly casting dance to treat blocks like this
+     * as special cases and make the used/free fields part of the allocated
+     * block, but the old code was not really proper portable C and depended
+     * on a bunch of implementation-specific behavior.  We could have done it
+     * better with a union in struct arena_link, but the memory savings is
+     * only 0.39% for a 64-bit machine, a 4096-byte block size and all
+     * large allocations *only just one byte* over the block size, so I
+     * question the utility of it.  We do still slip the large block in
+     * one position behind the list head so it doesn't cut off a partially
+     * filled list head.
+     *
+     * -- andrea
+     */
+    link = alloc_block(arena, size + sizeof(struct arena_link));
     assert(link != NULL);
-    memset(link, 0, size + sizeof(struct arena_link*));
-    *(struct arena_link**)link = arena->head->next;
-    arena->head->next = (struct arena_link*)link;
-    return (void*)(((uint8_t*)link) + sizeof(struct arena_link*));
+    arena->used += size;
+    arena->wasted += sizeof(struct arena_link);
+    link->used = size;
+    link->free = 0;
+    link->next = arena->head->next;
+    arena->head->next = link;
+    ret = link->rest;
+
+#ifdef DETAILED_ARENA_STATS
+    ++(arena->arena_malloc_count);
+    arena->arena_malloc_bytes += size;
+    if (need_zero) {
+      ++(arena->arena_li_malloc_count);
+      arena->arena_li_malloc_bytes += size;
+    } else {
+      ++(arena->arena_lu_malloc_count);
+      arena->arena_lu_malloc_bytes += size;
+    }
+#endif
   } else {
-    // we just need to allocate an ordinary new block.
-    struct arena_link *link = alloc_block(arena, sizeof(struct arena_link) + arena->block_size);
+    /* we just need to allocate an ordinary new block. */
+    link = alloc_block(arena, sizeof(struct arena_link) + arena->block_size);
     assert(link != NULL);
-    memset(link, 0, sizeof(struct arena_link) + arena->block_size);
+#ifdef DETAILED_ARENA_STATS
+    ++(arena->mm_malloc_count);
+    arena->mm_malloc_bytes += sizeof(struct arena_link) + arena->block_size;
+#endif
     link->free = arena->block_size - size;
     link->used = size;
     link->next = arena->head;
     arena->head = link;
     arena->used += size;
     arena->wasted += sizeof(struct arena_link) + arena->block_size - size;
-    return link->rest;
+    ret = link->rest;
+
+#ifdef DETAILED_ARENA_STATS
+    ++(arena->arena_malloc_count);
+    arena->arena_malloc_bytes += size;
+    if (need_zero) {
+      ++(arena->arena_si_malloc_count);
+      arena->arena_si_malloc_bytes += size;
+    } else {
+      ++(arena->arena_su_malloc_count);
+      arena->arena_su_malloc_bytes += size;
+    }
+#endif
   }
+
+  /*
+   * Zeroize if necessary
+   */
+  if (need_zero && !(arena->malloc_zeros)) {
+    memset(ret, 0, size);
+#ifdef DETAILED_ARENA_STATS
+    ++(arena->memset_count);
+    arena->memset_bytes += size;
+#endif
+  }
+
+  return ret;
 }
 
 void h_arena_free(HArena *arena, void* ptr) {
@@ -146,4 +257,49 @@ void h_delete_arena(HArena *arena) {
 void h_allocator_stats(HArena *arena, HArenaStats *stats) {
   stats->used = arena->used;
   stats->wasted = arena->wasted;
+#ifdef DETAILED_ARENA_STATS
+  stats->mm_malloc_count = arena->mm_malloc_count;
+  stats->mm_malloc_bytes = arena->mm_malloc_bytes;
+  stats->memset_count = arena->memset_count;
+  stats->memset_bytes = arena->memset_bytes;
+  stats->arena_malloc_count = arena->arena_malloc_count;
+  stats->arena_malloc_bytes = arena->arena_malloc_bytes;
+  stats->arena_su_malloc_count = arena->arena_su_malloc_count;
+  stats->arena_su_malloc_bytes = arena->arena_su_malloc_bytes;
+  stats->arena_si_malloc_count = arena->arena_si_malloc_count;
+  stats->arena_si_malloc_bytes = arena->arena_si_malloc_bytes;
+  stats->arena_lu_malloc_count = arena->arena_lu_malloc_count;
+  stats->arena_lu_malloc_bytes = arena->arena_lu_malloc_bytes;
+  stats->arena_li_malloc_count = arena->arena_li_malloc_count;
+  stats->arena_li_malloc_bytes = arena->arena_li_malloc_bytes;
+#endif
+}
+
+void* h_arena_realloc(HArena *arena, void* ptr, size_t n) {
+  struct arena_link *link;
+  void* ret;
+  size_t ncopy;
+
+  // XXX this is really wasteful, but maybe better than nothing?
+  //
+  // first, we walk the blocks to find our ptr. since we don't know how large
+  // the original allocation was, we must always make a new one and copy as
+  // much data from the old block as there could have been.
+
+  for (link = arena->head; link; link = link->next) {
+    if (ptr >= (void *)link->rest && ptr <= (void *)link->rest + link->used)
+      break;	/* found it */
+  }
+  assert(link != NULL);
+
+  ncopy = (void *)link->rest + link->used - ptr;
+  if (n < ncopy)
+    ncopy = n;
+
+  ret = h_arena_malloc_noinit(arena, n);
+  assert(ret != NULL);
+  memcpy(ret, ptr, ncopy);
+  h_arena_free(arena, ptr);
+
+  return ret;
 }

src/allocator.h

@@ -38,6 +38,8 @@ extern "C" {
 # define ATTR_MALLOC(n)
 #endif
 
+/* #define DETAILED_ARENA_STATS */
+
 // TODO(thequux): Turn this into an "HAllocatorVtable", and add a wrapper that also takes an environment pointer.
 typedef struct HAllocator_ {
   void* (*alloc)(struct HAllocator_* allocator, size_t size);
@@ -46,12 +48,15 @@ typedef struct HAllocator_ {
 } HAllocator;
 
 void* h_alloc(HAllocator* allocator, size_t size) ATTR_MALLOC(2);
+void* h_realloc(HAllocator* allocator, void* ptr, size_t size);
 
 typedef struct HArena_ HArena ; // hidden implementation
 
 HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for default...
 
+void* h_arena_malloc_noinit(HArena *arena, size_t count) ATTR_MALLOC(2);
 void* h_arena_malloc(HArena *arena, size_t count) ATTR_MALLOC(2);
+void* h_arena_realloc(HArena *arena, void* ptr, size_t count);
 void h_arena_free(HArena *arena, void* ptr); // For future expansion, with alternate memory managers.
 void h_delete_arena(HArena *arena);
 void h_arena_set_except(HArena *arena, jmp_buf *except);
@@ -59,6 +64,26 @@ void h_arena_set_except(HArena *arena, jmp_buf *except);
 typedef struct {
   size_t used;
   size_t wasted;
+#ifdef DETAILED_ARENA_STATS
+  size_t mm_malloc_count;
+  size_t mm_malloc_bytes;
+  size_t memset_count;
+  size_t memset_bytes;
+  size_t arena_malloc_count;
+  size_t arena_malloc_bytes;
+  /* small, uninited */
+  size_t arena_su_malloc_count;
+  size_t arena_su_malloc_bytes;
+  /* small, inited */
+  size_t arena_si_malloc_count;
+  size_t arena_si_malloc_bytes;
+  /* large, uninited */
+  size_t arena_lu_malloc_count;
+  size_t arena_lu_malloc_bytes;
+  /* large, inited */
+  size_t arena_li_malloc_count;
+  size_t arena_li_malloc_bytes;
+#endif
 } HArenaStats;
 
 void h_allocator_stats(HArena *arena, HArenaStats *stats);

src/backends/glr.c

 #include <assert.h>
 #include "lr.h"
+#include "params.h"
 
 static bool glr_step(HParseResult **result, HSlist *engines,
                      HLREngine *engine, const HLRAction *action);
@@ -14,7 +15,7 @@ int h_glr_compile(HAllocator* mm__, HParser* parser, const void* params)
   }
   int result = h_lalr_compile(mm__, parser, params);
 
-  if(result == -1 && parser->backend_data) {
+  if(result == -2 && parser->backend_data) {
     // table is there, just has conflicts? nevermind, that's okay.
     result = 0;
   }
@@ -174,9 +175,9 @@ static bool glr_step(HParseResult **result, HSlist *engines,
     HSlistNode *x;
     for(x=engines->head; x; x=x->next) {
       HLREngine *eng = x->elem;
-      if(eng->state == engine->state) {
-        x->elem = lrengine_merge(eng, engine);
-        break;
+      if(eng->state == engine->state && eng->input.index == engine->input.index) {
+    	x->elem = lrengine_merge(eng, engine);
+    	break;
       }
     }
     if(!x)  // no merge happened
@@ -225,6 +226,8 @@ HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream*
       HLREngine *engine = h_slist_pop(engines);
       const HLRAction *action = h_lrengine_action(engine);
       glr_step(&result, engback, engine, action);
+      // XXX detect ambiguous results - two engines terminating at the same pos
+      // -> kill both engines, i.e. ignore if there is a later unamb. success
     }
 
     // swap the lists
@@ -239,12 +242,54 @@ HParseResult *h_glr_parse(HAllocator* mm__, const HParser* parser, HInputStream*
   return result;
 }
 
+char * h_glr_get_description(HAllocator *mm__,
+                             HParserBackend be, void *param) {
+  const char *backend_name = "GLR";
+  size_t k;
+  char *descr = NULL;
 
+  k = h_get_param_k(param);
+
+  descr = h_format_description_with_param_k(mm__, backend_name, k);
+
+  return descr;
+}
+
+char * h_glr_get_short_name(HAllocator *mm__,
+                            HParserBackend be, void *param) {
+  const char *backend_name = "GLR";
+
+  size_t k;
+  char *name = NULL;
+
+  k = h_get_param_k(param);
+
+  name = h_format_name_with_param_k(mm__, backend_name, k);
+
+  return name;
+}
+
+
+int h_glr_extract_params(HParserBackendWithParams * be_with_params, backend_with_params_t * be_with_params_t) {
+
+  return h_extract_param_k(be_with_params, be_with_params_t);
+}
 
 HParserBackendVTable h__glr_backend_vtable = {
   .compile = h_glr_compile,
   .parse = h_glr_parse,
-  .free = h_glr_free
+  .free = h_glr_free,
+
+  .copy_params = h_copy_numeric_param,
+  /* No free_param needed, since it's not actually allocated */
+
+  /* Name/param resolution functions */
+  .backend_short_name = "glr",
+  .backend_description = "GLR(k) parser backend",
+  .get_description_with_params = h_glr_get_description,
+  .get_short_name_with_params = h_glr_get_short_name,
+
+  .extract_params = h_glr_extract_params
 };
 
 

src/backends/lalr.c

 #include <assert.h>
 #include "contextfree.h"
 #include "lr.h"
-
+#include "params.h"
 
 
 /* LALR-via-SLR grammar transformation */
@@ -31,18 +31,24 @@ static size_t follow_transition(const HLRTable *table, size_t x, HCFChoice *A)
 {
   HLRAction *action = lrtable_lookup(table, x, A);
   assert(action != NULL);
+
+  // we are interested in a transition out of state x, i.e. a shift action.
+  // while there could also be reduce actions associated with A in state x,
+  // those are not what we are here for. so if action is a conflict, search it
+  // for the shift. there will only be one and it will be the bottom element.
+  if(action->type == HLR_CONFLICT) {
+    HSlistNode *x;
+    for(x=action->branches->head; x; x=x->next) {
+      action = x->elem;
+      assert(action->type != HLR_CONFLICT); // no nesting of conflicts
+      if(action->type == HLR_SHIFT)
+        break;
+    }
+    assert(x != NULL && x->next == NULL);   // shift found at the bottom
+  }
   assert(action->type == HLR_SHIFT);
-  return action->nextstate;
-}
 
-static inline HLRTransition *transition(HArena *arena,
-                                        size_t x, const HCFChoice *A, size_t y)
-{
-  HLRTransition *t = h_arena_malloc(arena, sizeof(HLRTransition));
-  t->from = x;
-  t->symbol = A;
-  t->to = y;
-  return t;
+  return action->nextstate;
 }
 
 // no-op on terminal symbols
@@ -69,8 +75,8 @@ static void transform_productions(const HLRTable *table, HLREnhGrammar *eg,
     HCFChoice **iBj = items;
     for(; *B; B++, iBj++) {
       size_t j = follow_transition(table, i, *B);
-      HLRTransition *i_B_j = transition(arena, i, *B, j);
-      *iBj = h_hashtable_get(eg->tmap, i_B_j);
+      HLRTransition i_B_j = {i, *B, j};
+      *iBj = h_hashtable_get(eg->tmap, &i_B_j);
       assert(*iBj != NULL);
       i = j;
     }
@@ -269,6 +275,7 @@ HCFChoice *h_desugar_augmented(HAllocator *mm__, HParser *parser)
 
 int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params)
 {
+  size_t k = params? (uintptr_t)params : DEFAULT_KMAX;
   // generate (augmented) CFG from parser
   // construct LR(0) DFA
   // build LR(0) table
@@ -279,18 +286,18 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params)
   }
   HCFGrammar *g = h_cfgrammar_(mm__, h_desugar_augmented(mm__, parser));
   if(g == NULL)     // backend not suitable (language not context-free)
-    return -1;
+    return 2;
 
   HLRDFA *dfa = h_lr0_dfa(g);
   if (dfa == NULL) {     // this should normally not happen
     h_cfgrammar_free(g);
-    return -1;
+    return 3;
   }
 
   HLRTable *table = h_lr0_table(g, dfa);
   if (table == NULL) {   // this should normally not happen
     h_cfgrammar_free(g);
-    return -1;
+    return 4;
   }
 
   if(has_conflicts(table)) {
@@ -300,7 +307,7 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params)
     if(eg == NULL) {    // this should normally not happen
       h_cfgrammar_free(g);
       h_lrtable_free(table);
-      return -1;
+      return 5;
     }
 
     // go through the inadequate states; replace inadeq with a new list
@@ -329,10 +336,14 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params)
           if(match_any_production(table, eg, lhs, item->rhs, state)) {
             // the left-hand symbol's follow set is this production's
             // contribution to the lookahead
-            const HStringMap *fs = h_follow(1, eg->grammar, lhs);
+            const HStringMap *fs = h_follow(k, eg->grammar, lhs);
             assert(fs != NULL);
             assert(fs->epsilon_branch == NULL);
-            assert(!h_stringmap_empty(fs));
+            // NB: there is a case where fs can be empty: when reducing by lhs
+            // would lead to certain parse failure, by means of h_nothing_p()
+            // for instance. in that case, the below code correctly adds no
+            // reduce action.
+            assert(!h_stringmap_empty(fs)); // XXX
 
             // for each lookahead symbol, put action into table cell
             if(terminals_put(table->tmap[state], fs, action) < 0)
@@ -345,11 +356,13 @@ int h_lalr_compile(HAllocator* mm__, HParser* parser, const void* params)
         h_slist_push(table->inadeq, (void *)(uintptr_t)state);
       }
     }
+
+    h_cfgrammar_free(eg->grammar);
   }
 
   h_cfgrammar_free(g);
   parser->backend_data = table;
-  return has_conflicts(table)? -1 : 0;
+  return has_conflicts(table)? -2 : 0;
 }
 
 void h_lalr_free(HParser *parser)
@@ -357,22 +370,62 @@ void h_lalr_free(HParser *parser)
   HLRTable *table = parser->backend_data;
   h_lrtable_free(table);
   parser->backend_data = NULL;
-  parser->backend = PB_PACKRAT;
+  parser->backend_vtable = h_get_default_backend_vtable();
+  parser->backend = h_get_default_backend();
+}
+
+char * h_lalr_get_description(HAllocator *mm__,
+                              HParserBackend be, void *param) {
+  const char *backend_name = "LALR";
+  size_t k;
+  char *descr = NULL;
+
+  k = h_get_param_k(param);
+
+  descr = h_format_description_with_param_k(mm__, backend_name, k);
+
+  return descr;
 }
 
+char * h_lalr_get_short_name(HAllocator *mm__,
+                            HParserBackend be, void *param) {
+  const char *backend_name = "LALR";
+
+  size_t k;
+  char *name = NULL;
+
+  k = h_get_param_k(param);
+
+  name = h_format_name_with_param_k(mm__, backend_name, k);
+
+  return name;
+}
 
 
+int h_lalr_extract_params(HParserBackendWithParams * be_with_params, backend_with_params_t * be_with_params_t) {
+
+  return h_extract_param_k(be_with_params, be_with_params_t);
+
+}
+
 HParserBackendVTable h__lalr_backend_vtable = {
   .compile = h_lalr_compile,
   .parse = h_lr_parse,
   .free = h_lalr_free,
   .parse_start = h_lr_parse_start,
   .parse_chunk = h_lr_parse_chunk,
-  .parse_finish = h_lr_parse_finish
-};
-
+  .parse_finish = h_lr_parse_finish,
+  .copy_params = h_copy_numeric_param,
+  /* No free_param needed, since it's not actually allocated */
 
+  /* Name/param resolution functions */
+  .backend_short_name = "lalr",
+  .backend_description = "LALR(k) parser backend",
+  .get_description_with_params = h_lalr_get_description,
+  .get_short_name_with_params = h_lalr_get_short_name,
 
+  .extract_params = h_lalr_extract_params
+};
 
 // dummy!
 int test_lalr(void)

src/backends/llk.c

@@ -2,8 +2,7 @@
 #include "../internal.h"
 #include "../cfgrammar.h"
 #include "../parsers/parser_internal.h"
-
-static const size_t DEFAULT_KMAX = 1;
+#include "params.h"
 
 
 /* Generating the LL(k) parse table */
@@ -238,7 +237,7 @@ int h_llk_compile(HAllocator* mm__, HParser* parser, const void* params)
     // the table was ambiguous
     h_cfgrammar_free(grammar);
     h_llktable_free(table);
-    return -1;
+    return -2;
   }
   parser->backend_data = table;
 
@@ -254,7 +253,8 @@ void h_llk_free(HParser *parser)
   HLLkTable *table = parser->backend_data;
   h_llktable_free(table);
   parser->backend_data = NULL;
-  parser->backend = PB_PACKRAT;
+  parser->backend_vtable = h_get_default_backend_vtable();
+  parser->backend = h_get_default_backend();
 }
 
 
@@ -606,6 +606,38 @@ HParseResult *h_llk_parse_finish(HSuspendedParser *s)
   return llk_parse_finish_(s->mm__, s->backend_state);
 }
 
+char * h_llk_get_description(HAllocator *mm__,
+                             HParserBackend be, void *param) {
+  const char *backend_name = "LL";
+  size_t k, len;
+  char *descr = NULL;
+
+  k = h_get_param_k(param);
+
+  descr = h_format_description_with_param_k(mm__, backend_name, k);
+
+  return descr;
+}
+
+char * h_llk_get_short_name(HAllocator *mm__,
+                            HParserBackend be, void *param) {
+  const char *backend_name = "LL";
+
+  size_t k;
+  char *name = NULL;
+
+  k = h_get_param_k(param);
+
+  name = h_format_name_with_param_k(mm__, backend_name, k);
+
+  return name;
+}
+
+int h_llk_extract_params(HParserBackendWithParams * be_with_params, backend_with_params_t *be_with_params_t) {
+
+  return h_extract_param_k(be_with_params, be_with_params_t);
+}
+
 
 HParserBackendVTable h__llk_backend_vtable = {
   .compile = h_llk_compile,
@@ -614,7 +646,19 @@ HParserBackendVTable h__llk_backend_vtable = {
 
   .parse_start = h_llk_parse_start,
   .parse_chunk = h_llk_parse_chunk,
-  .parse_finish = h_llk_parse_finish
+  .parse_finish = h_llk_parse_finish,
+
+  .copy_params = h_copy_numeric_param,
+  /* No free_param needed, since it's not actually allocated */
+
+  /* Name/param resolution functions */
+  .backend_short_name = "llk",
+  .backend_description = "LL(k) parser backend",
+  .get_description_with_params = h_llk_get_description,
+  .get_short_name_with_params = h_llk_get_short_name,
+
+  /*extraction of params from string*/
+  .extract_params = h_llk_extract_params
 };
 
 

src/backends/lr.c

@@ -420,6 +420,12 @@ void h_lr_parse_start(HSuspendedParser *s)
   s->backend_state = engine;
 }
 
+// cf. comment before run_trace in regex.c
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunknown-pragmas"
+#pragma GCC diagnostic ignored "-Wclobbered"
+#endif
 bool h_lr_parse_chunk(HSuspendedParser* s, HInputStream *stream)
 {
   HLREngine *engine = s->backend_state;
@@ -457,6 +463,10 @@ bool h_lr_parse_chunk(HSuspendedParser* s, HInputStream *stream)
   *stream = engine->input;
   return !run;  // done if engine no longer running
 }
+// Reenable -Wclobber
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
 
 HParseResult *h_lr_parse_finish(HSuspendedParser *s)
 {

src/backends/missing.c

+#include "missing.h"
+
+/* Placeholder backend that always fails */
+
+int h_missing_compile(HAllocator* mm__, HParser* parser, const void* params) {
+  /* Always fail */
+
+  return -1;
+}
+
+HParseResult *h_missing_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream) {
+  /* Always fail */
+
+  return NULL;
+}
+
+void h_missing_free(HParser *parser) {
+  /* No-op */
+}
+
+HParserBackendVTable h__missing_backend_vtable = {
+  .compile = h_missing_compile,
+  .parse = h_missing_parse,
+  .free = h_missing_free,
+};

src/backends/missing.h

+#ifndef HAMMER_BACKENDS_MISSING__H
+#define HAMMER_BACKENDS_MISSING__H
+
+#include "../hammer.h"
+#include "../internal.h"
+
+#endif /* !defined(HAMMER_BACKENDS_MISSING__H) */

src/backends/packrat.c

@@ -3,6 +3,17 @@
 #include "../internal.h"
 #include "../parsers/parser_internal.h"
 
+/* #define DETAILED_PACKRAT_STATISTICS */
+
+#ifdef DETAILED_PACKRAT_STATISTICS
+static size_t packrat_hash_count = 0;
+static size_t packrat_hash_bytes = 0;
+static size_t packrat_cmp_count = 0;
+static size_t packrat_cmp_bytes = 0;
+#endif
+
+static uint32_t cache_key_hash(const void* key);
+
 // short-hand for creating lowlevel parse cache values (parse result case)
 static
 HParserCacheValue * cached_result(HParseState *state, HParseResult *result) {
@@ -23,64 +34,73 @@ HParserCacheValue *cached_lr(HParseState *state, HLeftRec *lr) {
   return ret;
 }
 
-// Really library-internal tool to perform an uncached parse, and handle any common error-handling.
-static inline HParseResult* perform_lowlevel_parse(HParseState *state, const HParser *parser) {
-  // TODO(thequux): these nested conditions are ugly. Factor this appropriately, so that it is clear which codes is executed when.
-  HParseResult *tmp_res;
-  if (parser) {
-    HInputStream bak = state->input_stream;
-    tmp_res = parser->vtable->parse(parser->env, state);
-    if (tmp_res) {
-      tmp_res->arena = state->arena;
-      if (!state->input_stream.overrun) {
-	size_t bit_length = h_input_stream_pos(&state->input_stream) - h_input_stream_pos(&bak);
-	if (tmp_res->bit_length == 0) { // Don't modify if forwarding.
-	  tmp_res->bit_length = bit_length;
-	}
-	if (tmp_res->ast && tmp_res->ast->bit_length != 0) {
-	  ((HParsedToken*)(tmp_res->ast))->bit_length = bit_length;
-	}
-      } else
-	tmp_res->bit_length = 0;
-    }
-  } else
-    tmp_res = NULL;
-  if (state->input_stream.overrun)
-    return NULL; // overrun is always failure.
-#ifdef CONSISTENCY_CHECK
-  if (!tmp_res) {
-    state->input_stream = INVALID;
-    state->input_stream.input = key->input_pos.input;
+// internal helper to perform an uncached parse and common error-handling
+static inline
+HParseResult *perform_lowlevel_parse(HParseState *state, const HParser *parser)
+{
+  HParseResult *res;
+  HInputStream bak;
+  size_t len;
+
+  if (!parser)
+    return NULL;
+
+  bak = state->input_stream;
+  res = parser->vtable->parse(parser->env, state);
+
+  if (!res)
+    return NULL;	// NB: input position is considered invalid on failure
+
+  // combinators' parse functions by design do not have to check for overrun.
+  // turn such bogus successes into parse failure.
+  if (state->input_stream.overrun) {
+    res->bit_length = 0;
+    return NULL;
   }
-#endif
-  return tmp_res;
+
+  // update result length
+  res->arena = state->arena;
+  len = h_input_stream_pos(&state->input_stream) - h_input_stream_pos(&bak);
+  if (res->bit_length == 0)	// Don't modify if forwarding.
+    res->bit_length = len;
+  if (res->ast && res->ast->bit_length != 0)
+    ((HParsedToken *)(res->ast))->bit_length = len;
+
+  return res;
 }
 
-HParserCacheValue* recall(HParserCacheKey *k, HParseState *state) {
-  HParserCacheValue *cached = h_hashtable_get(state->cache, k);
+HParserCacheValue* recall(HParserCacheKey *k, HParseState *state, HHashValue keyhash) {
+  HParserCacheValue *cached = h_hashtable_get_precomp(state->cache, k, keyhash);
   HRecursionHead *head = h_hashtable_get(state->recursion_heads, &k->input_pos);
-  if (!head) { // No heads found
+
+  if (!head) {
+    /* No heads found */
     return cached;
-  } else { // Some heads found
+  } else {
+    /* Some heads found */
     if (!cached && head->head_parser != k->parser && !h_slist_find(head->involved_set, k->parser)) {
-      // Nothing in the cache, and the key parser is not involved
+      /* Nothing in the cache, and the key parser is not involved */
       cached = cached_result(state, NULL);
       cached->input_stream = k->input_pos;
     }
     if (h_slist_find(head->eval_set, k->parser)) {
-      // Something is in the cache, and the key parser is in the eval set. Remove the key parser from the eval set of the head. 
+      /*
+       * Something is in the cache, and the key parser is in the eval set.
+       * Remove the key parser from the eval set of the head.
+       */
       head->eval_set = h_slist_remove_all(head->eval_set, k->parser);
       HParseResult *tmp_res = perform_lowlevel_parse(state, k->parser);
-      // update the cache
+      /* update the cache */
       if (!cached) {
-	cached = cached_result(state, tmp_res);
-	h_hashtable_put(state->cache, k, cached);
+        cached = cached_result(state, tmp_res);
+        h_hashtable_put_precomp(state->cache, k, cached, keyhash);
       } else {
-	cached->value_type = PC_RIGHT;
-	cached->right = tmp_res;
-	cached->input_stream = state->input_stream;
+        cached->value_type = PC_RIGHT;
+        cached->right = tmp_res;
+        cached->input_stream = state->input_stream;
       }
     }
+
     return cached;
   }
 }
@@ -180,36 +200,52 @@ HParseResult* lr_answer(HParserCacheKey *k, HParseState *state, HLeftRec *growab
 /* Warth's recursion. Hi Alessandro! */
 HParseResult* h_do_parse(const HParser* parser, HParseState *state) {
   HParserCacheKey *key = a_new(HParserCacheKey, 1);
+  HHashValue keyhash;
+  HLeftRec *base = NULL;
+  HParserCacheValue *m = NULL, *cached = NULL;
+
   key->input_pos = state->input_stream; key->parser = parser;
-  HParserCacheValue *m = NULL;
+  keyhash = cache_key_hash(key);
+
   if (parser->vtable->higher) {
-    m = recall(key, state);
+    m = recall(key, state, keyhash);
   }
-  // check to see if there is already a result for this object...
+
+  /* check to see if there is already a result for this object... */
   if (!m) {
-    // It doesn't exist, so create a dummy result to cache
-    HLeftRec *base = NULL;
-    // But only cache it now if there's some chance it could grow; primitive parsers can't
+    /*
+     * But only cache it now if there's some chance it could grow; primitive
+     * parsers can't
+     */
     if (parser->vtable->higher) {
       base = a_new(HLeftRec, 1);
       base->seed = NULL; base->rule = parser; base->head = NULL;
       h_slist_push(state->lr_stack, base);
-      // cache it
-      h_hashtable_put(state->cache, key, cached_lr(state, base));
-      // parse the input
+      /* cache it */
+      h_hashtable_put_precomp(state->cache, key,
+                              cached_lr(state, base), keyhash);
     }
+
+    /* parse the input */
     HParseResult *tmp_res = perform_lowlevel_parse(state, parser);
     if (parser->vtable->higher) {
-      // the base variable has passed equality tests with the cache
+      /* the base variable has passed equality tests with the cache */
       h_slist_pop(state->lr_stack);
-      // update the cached value to our new position
-      HParserCacheValue *cached = h_hashtable_get(state->cache, key);
+      /* update the cached value to our new position */
+      cached = h_hashtable_get_precomp(state->cache, key, keyhash);
       assert(cached != NULL);
       cached->input_stream = state->input_stream;
     }
-    // setupLR, used below, mutates the LR to have a head if appropriate, so we check to see if we have one
+
+    /*
+     * setupLR, used below, mutates the LR to have a head if appropriate,
+     * so we check to see if we have one
+     */
     if (!base || NULL == base->head) {
-      h_hashtable_put(state->cache, key, cached_result(state, tmp_res));
+      if (parser->vtable->higher) {
+        h_hashtable_put_precomp(state->cache, key,
+                                cached_result(state, tmp_res), keyhash);
+      }
       return tmp_res;
     } else {
       base->seed = tmp_res;
@@ -217,7 +253,7 @@ HParseResult* h_do_parse(const HParser* parser, HParseState *state) {
       return res;
     }
   } else {
-    // it exists!
+    /* it exists! */
     state->input_stream = m->input_stream;
     if (PC_LEFT == m->value_type) {
       setupLR(parser, state, m->left);
@@ -229,27 +265,46 @@ HParseResult* h_do_parse(const HParser* parser, HParseState *state) {
 }
 
 int h_packrat_compile(HAllocator* mm__, HParser* parser, const void* params) {
+  parser->backend_vtable = &h__packrat_backend_vtable;
   parser->backend = PB_PACKRAT;
   return 0; // No compilation necessary, and everything should work
 	    // out of the box.
 }
 
 void h_packrat_free(HParser *parser) {
-  parser->backend = PB_PACKRAT; // revert to default, oh that's us
+  parser->backend_vtable = h_get_default_backend_vtable();
+  parser->backend = h_get_default_backend();
 }
 
 static uint32_t cache_key_hash(const void* key) {
+#ifdef DETAILED_PACKRAT_STATISTICS
+  ++(packrat_hash_count);
+  packrat_hash_bytes += sizeof(HParserCacheKey);
+#endif
   return h_djbhash(key, sizeof(HParserCacheKey));
 }
+
 static bool cache_key_equal(const void* key1, const void* key2) {
+#ifdef DETAILED_PACKRAT_STATISTICS
+  ++(packrat_cmp_count);
+  packrat_cmp_bytes += sizeof(HParserCacheKey);
+#endif
   return memcmp(key1, key2, sizeof(HParserCacheKey)) == 0;
 }
 
 static uint32_t pos_hash(const void* key) {
+#ifdef DETAILED_PACKRAT_STATISTICS
+  ++(packrat_hash_count);
+  packrat_hash_bytes += sizeof(HInputStream);
+#endif
   return h_djbhash(key, sizeof(HInputStream));
 }
 
 static bool pos_equal(const void* key1, const void* key2) {
+#ifdef DETAILED_PACKRAT_STATISTICS
+  ++(packrat_cmp_count);
+  packrat_cmp_bytes += sizeof(HInputStream);
+#endif
   return memcmp(key1, key2, sizeof(HInputStream)) == 0;
 }
 
@@ -271,7 +326,9 @@ HParseResult *h_packrat_parse(HAllocator* mm__, const HParser* parser, HInputStr
   parse_state->lr_stack = h_slist_new(arena);
   parse_state->recursion_heads = h_hashtable_new(arena, pos_equal, pos_hash);
   parse_state->arena = arena;
+  parse_state->symbol_table = NULL;
   HParseResult *res = h_do_parse(parser, parse_state);
+  *input_stream = parse_state->input_stream;
   h_slist_free(parse_state->lr_stack);
   h_hashtable_free(parse_state->recursion_heads);
   // tear down the parse state
@@ -282,8 +339,121 @@ HParseResult *h_packrat_parse(HAllocator* mm__, const HParser* parser, HInputStr
   return res;
 }
 
+// The following naive implementation of the iterative (chunked) parsing API
+// concatenates chunks and blindly re-runs the full parse on every call to
+// h_packrat_parse_chunk.
+//
+// NB: A full implementation will still have to concatenate the chunks to
+// support arbitrary backtracking, but should be able save much, if not all, of
+// the HParseState between calls.
+// Cutting unneeded past input should also be possible but is complicated by
+// the fact that only higher-order combinators are saved to the packrat cache,
+// so former input to bare primitive combinators must remain available.
+//
+// Note: The iterative API expects us to always consume an entire input chunk
+// when we suspend, even if packrat later backtracks into it. We will produce
+// the correct parse result and accurately consume from a final chunk, but all
+// earlier chunks will be reported as fully consumed and as being part of the
+// HParseResult in terms of its bit_length field.
+
+void h_packrat_parse_start(HSuspendedParser *s)
+{
+  // nothing to do here, we allocate lazily below
+}
+
+bool h_packrat_parse_chunk(HSuspendedParser *s, HInputStream *input)
+{
+  HAllocator *mm__ = s->mm__;
+  HParseResult *res;
+  HInputStream *cat;
+  size_t newlen;
+
+  if (s->backend_state == NULL) {	// this is the first chunk
+    // attempt to finish the parse on just the given input.
+    res = h_packrat_parse(mm__, s->parser, input);
+    if (input->last_chunk || !input->overrun) {
+      s->backend_state = res;		// pass on the result
+      return true;			// and signal we're done
+    }
+
+    // we ran out of input and are expecting more
+    // allocate and initialize an input stream to concatenate the chunks
+    cat = h_new(HInputStream, 1);
+    *cat = *input;
+    cat->input = h_alloc(mm__, input->length);
+    memcpy((void *)cat->input, input->input, input->length);
+    s->backend_state = cat;
+
+    goto suspend;
+  }
+
+  // we have received additional input - append it to the saved stream
+  cat = s->backend_state;
+  assert(input->pos == cat->length);
+  if (input->length > SIZE_MAX - cat->length)
+    h_platform_errx(1, "input length would overflow");
+  newlen = cat->length + input->length;
+  cat->input = h_realloc(mm__, (void *)cat->input, newlen);
+  memcpy((void *)cat->input + cat->length, input->input, input->length);
+  cat->length = newlen;
+  cat->last_chunk = input->last_chunk;
+
+  // reset our input stream and call the parser on it (again)
+  cat->index      = 0;
+  cat->bit_offset = 0;
+  cat->margin     = 0;
+  cat->endianness = DEFAULT_ENDIANNESS;
+  cat->overrun    = false;
+  res = h_packrat_parse(mm__, s->parser, cat);
+  assert(cat->index <= cat->length);
+  input->overrun    = cat->overrun;
+
+  // suspend if the parser still needs more input
+  if (input->overrun && !input->last_chunk)
+    goto suspend;
+  // otherwise the parse is finished...
+
+  // report final input position
+  if (cat->index < input->pos) {	// parser just needed some lookahead
+    input->index      = 0;		// don't consume this last chunk
+    input->bit_offset = 0;
+    input->margin     = 0;
+  } else {
+    input->index      = cat->index - input->pos;
+    input->bit_offset = cat->bit_offset;
+    input->margin     = cat->margin;
+    input->endianness = cat->endianness;
+  }
+
+  // clean up and return the result
+  h_free((void *)cat->input);
+  h_free(cat);
+  s->backend_state = res;
+
+  return true;				// don't call me again.
+
+suspend:
+  input->index = input->length;		// consume the entire chunk on suspend
+  input->margin = 0;
+  input->bit_offset = 0;
+  return false;				// come back with more input.
+}
+
+HParseResult *h_packrat_parse_finish(HSuspendedParser *s)
+{
+  return s->backend_state;
+}
+
 HParserBackendVTable h__packrat_backend_vtable = {
   .compile = h_packrat_compile,
   .parse = h_packrat_parse,
-  .free = h_packrat_free
+  .free = h_packrat_free,
+  .parse_start = h_packrat_parse_start,
+  .parse_chunk = h_packrat_parse_chunk,
+  .parse_finish = h_packrat_parse_finish,
+  /* Name/param resolution functions */
+  .backend_short_name = "packrat",
+  .backend_description = "Packrat parser with Warth's recursion",
+  .get_description_with_params = h_get_description_with_no_params,
+  .get_short_name_with_params = h_get_short_name_with_no_params
 };

src/backends/params.c

+#include "params.h"
+
+
+
+size_t h_get_param_k(void *param) {
+  uintptr_t params_int;
+
+  params_int = (uintptr_t)param;
+  return (size_t)params_int;
+
+}
+
+
+
+char * h_format_description_with_param_k(HAllocator *mm__, const char *backend_name, size_t k){
+  const char *format_str = "%s(%zu) parser backend";
+
+  const char *generic_descr_format_str =
+    "%s(k) parser backend (default k is %zu)";
+  size_t len;
+  char *descr = NULL;
+
+
+  if (k > 0) {
+    /* A specific k was given */
+    /* Measure how big a buffer we need */
+    len = snprintf(NULL, 0, format_str, backend_name, k);
+    /* Allocate it and do the real snprintf */
+    descr = h_new(char, len + 1);
+    if (descr) {
+      snprintf(descr, len + 1, format_str, backend_name, k);
+    }
+  } else {
+    /*
+     * No specific k, would use DEFAULT_KMAX.  We say what DEFAULT_KMAX
+     * was compiled in in the description.
+     */
+    len = snprintf(NULL, 0, generic_descr_format_str, backend_name, DEFAULT_KMAX);
+    /* Allocate and do the real snprintf */
+    descr = h_new(char, len + 1);
+    if (descr) {
+      snprintf(descr, len + 1, generic_descr_format_str, backend_name, DEFAULT_KMAX);
+    }
+  }
+
+  return descr;
+}
+
+char * h_format_name_with_param_k(HAllocator *mm__, const char *backend_name, size_t k){
+  const char  *format_str = "%s(%zu)", *generic_name = "%s(k)";
+  size_t len;
+  char *name = NULL;
+
+  if (k > 0) {
+    /* A specific k was given */
+    /* Measure how big a buffer we need */
+    len = snprintf(NULL, 0, format_str, backend_name, k);
+    /* Allocate it and do the real snprintf */
+    name = h_new(char, len + 1);
+    if (name) {
+      snprintf(name, len + 1, format_str, backend_name, k);
+    }
+  } else {
+    /* No specific k */
+    len = snprintf(NULL, 0, generic_name, backend_name, k);
+    name = h_new(char, len + 1);
+    if (name) {
+      snprintf(name, len + 1, generic_name, backend_name);
+    }
+  }
+
+  return name;
+}
+
+/*TODO better error handling*/
+int h_extract_param_k(HParserBackendWithParams * be_with_params, backend_with_params_t * be_with_params_t) {
+
+  be_with_params->params = NULL;
+
+  int param_0 = -1;
+  int success = 0;
+  uintptr_t param;
+
+  size_t expected_params_len = 1;
+  backend_params_t params_t = be_with_params_t->params;
+  size_t actual_params_len = params_t.len;
+
+  if(actual_params_len >= expected_params_len) {
+    backend_param_with_name_t param_t = params_t.params[0];
+    success = sscanf((char*)param_t.param.param, "%d", &param_0);
+  }
+
+  if(success) {
+    param = (uintptr_t) param_0;
+    be_with_params->params = (void *)param;
+  }
+
+  return success;
+
+}

src/backends/params.h

+#ifndef HAMMER_BACKENDS_PARAMS__H
+#define HAMMER_BACKENDS_PARAMS__H
+
+#include "../hammer.h"
+#include "../internal.h"
+
+static const size_t DEFAULT_KMAX = 1;
+
+
+
+size_t h_get_param_k(void *param);
+
+char * h_format_description_with_param_k(HAllocator *mm__, const char *backend_name, size_t k);
+
+char * h_format_name_with_param_k(HAllocator *mm__, const char *backend_name, size_t k);
+
+int h_extract_param_k(HParserBackendWithParams * be_with_params, backend_with_params_t * be_with_params_t);
+
+#endif /* !defined(HAMMER_BACKENDS_PARAMS__H) */

src/backends/regex.c

@@ -7,6 +7,8 @@
 
 #undef a_new
 #define a_new(typ, count) a_new_(arena, typ, count)
+#undef a_new0
+#define a_new0(typ, count) a_new0_(arena, typ, count)
 // Stack VM
 typedef enum HSVMOp_ {
   SVM_PUSH, // Push a mark. There is no VM insn to push an object.
@@ -67,13 +69,13 @@ void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_
     goto end;
 
   HSArray *heads_n = heads_a, *heads_p = heads_b;
-  uint8_t *insn_seen = a_new(uint8_t, prog->length); // 0 -> not seen, 1->processed, 2->queued
-  HRVMThread *ip_queue = a_new(HRVMThread, prog->length);
+  uint8_t *insn_seen = a_new0(uint8_t, prog->length); // 0 -> not seen, 1->processed, 2->queued
+  HRVMThread *ip_queue = a_new0(HRVMThread, prog->length);
   size_t ipq_top;
 
 #define THREAD ip_queue[ipq_top-1]
 #define PUSH_SVM(op_, arg_) do { \
-	  HRVMTrace *nt = a_new(HRVMTrace, 1); \
+	  HRVMTrace *nt = a_new0(HRVMTrace, 1); \
 	  nt->arg = (arg_);		       \
 	  nt->opcode = (op_);		       \
 	  nt->next = THREAD.trace;	       \
@@ -81,7 +83,7 @@ void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_
 	  THREAD.trace = nt;		       \
   } while(0)
 
-  ((HRVMTrace*)h_sarray_set(heads_n, 0, a_new(HRVMTrace, 1)))->opcode = SVM_NOP; // Initial thread
+  ((HRVMTrace*)h_sarray_set(heads_n, 0, a_new0(HRVMTrace, 1)))->opcode = SVM_NOP; // Initial thread
   
   size_t off = 0;
   int live_threads = 1; // May be redundant
@@ -223,7 +225,7 @@ bool svm_stack_ensure_cap(HAllocator *mm__, HSVMContext *ctx, size_t addl) {
  * the second return; here, the only variables that could matter for
  * are arena and ctx (because they're referenced in "goto fail").
  */
-#ifdef __GNUC__
+#if defined(__GNUC__) && !defined(__clang__)
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunknown-pragmas"
 #pragma GCC diagnostic ignored "-Wclobbered"
@@ -257,7 +259,7 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace,
       if (!svm_stack_ensure_cap(mm__, ctx, 1)) {
 	goto fail;
       }
-      tmp_res = a_new(HParsedToken, 1);
+      tmp_res = a_new0(HParsedToken, 1);
       tmp_res->token_type = TT_MARK;
       tmp_res->index = cur->input_pos;
       tmp_res->bit_offset = 0;
@@ -288,7 +290,7 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace,
       break;
     case SVM_ACCEPT:
       assert(ctx->stack_count <= 1);
-      HParseResult *res = a_new(HParseResult, 1);
+      HParseResult *res = a_new0(HParseResult, 1);
       if (ctx->stack_count == 1) {
 	res->ast = ctx->stack[0];
       } else {
@@ -311,7 +313,7 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace,
   return NULL;
 }
 // Reenable -Wclobber
-#ifdef __GNUC__
+#if defined(__GNUC__) && !defined(__clang__)
 #pragma GCC diagnostic pop
 #endif
 
@@ -415,7 +417,8 @@ static void h_regex_free(HParser *parser) {
   h_free(prog->actions);
   h_free(prog);
   parser->backend_data = NULL;
-  parser->backend = PB_PACKRAT;
+  parser->backend_vtable = h_get_default_backend_vtable();
+  parser->backend = h_get_default_backend();
 }
 
 static int h_regex_compile(HAllocator *mm__, HParser* parser, const void* params) {
@@ -428,9 +431,10 @@ static int h_regex_compile(HAllocator *mm__, HParser* parser, const void* params
   prog->actions = NULL;
   prog->allocator = mm__;
   if (setjmp(prog->except)) {
-    return false;
+    return 3;
   }
   if (!h_compile_regex(prog, parser)) {
+    // this shouldn't normally fail when isValidRegular() returned true
     h_free(prog->insns);
     h_free(prog->actions);
     h_free(prog);
@@ -449,7 +453,12 @@ static HParseResult *h_regex_parse(HAllocator* mm__, const HParser* parser, HInp
 HParserBackendVTable h__regex_backend_vtable = {
   .compile = h_regex_compile,
   .parse = h_regex_parse,
-  .free = h_regex_free
+  .free = h_regex_free,
+  /* Name/param resolution functions */
+  .backend_short_name = "regex",
+  .backend_description = "Regular expression matcher (broken)",
+  .get_description_with_params = h_get_description_with_no_params,
+  .get_short_name_with_params = h_get_short_name_with_no_params
 };
 
 #ifndef NDEBUG