jni/com_upstandinghackers_hammer_Hammer.c → src/bindings/jni/com_upstandinghackers_hammer_Hammer.c

@@ -157,6 +157,13 @@ JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_in
 }
 
 
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_notIn
+  (JNIEnv *env, jclass class, jbyteArray charset, jint length)
+{
+    RETURNWRAP(env, h_not_in((uint8_t*) ((*env)->GetByteArrayElements(env, charset, NULL)), (size_t)length));
+}
+
+
 JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_endP
   (JNIEnv *env, jclass class)
 {
@@ -227,6 +234,34 @@ JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_choice
 }
 
 
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_permutation
+  (JNIEnv *env, jclass class, jobjectArray permutation)
+{
+    jsize length;
+    void **parsers;
+    int i;
+    jobject current;
+    const HParser *result;
+
+    length = (*env)->GetArrayLength(env, permutation);
+    parsers = malloc(sizeof(HParser *)*(length+1));
+    if(NULL==parsers)
+    {
+        return NULL;
+    }
+
+    for(i=0; i<length; i++)
+    {
+        current = (*env)->GetObjectArrayElement(env, permutation, (jsize)i);
+        parsers[i] = UNWRAP(env, current);
+    }
+    parsers[length] = NULL;
+
+    result = h_permutation__a(parsers);
+    RETURNWRAP(env, result);
+}
+
+
 JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_butNot
   (JNIEnv *env, jclass class, jobject p, jobject q)
 {
@@ -332,4 +367,29 @@ JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_indirect
 }
 
 
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_with_endianness
+  (JNIEnv *env, jclass class, jbyte endianess, jobject p)
+{
+    RETURNWRAP(env, h_with_endianness((char) endianess, UNWRAP(env, p)));
+}
 
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_skip
+  (JNIEnv *env, jclass class, jint n)
+{
+    RETURNWRAP(env, h_skip((size_t) n));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_seek
+  (JNIEnv *env, jclass class, jint offset, jint whence)
+{
+    RETURNWRAP(env, h_seek((ssize_t) offset, (int) whence));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_tell
+  (JNIEnv *env, jclass class)
+{
+    RETURNWRAP(env, h_tell());
+}

src/bindings/lua/hammer.lua

+local ffi = require("ffi")
+ffi.cdef[[
+typedef enum HParserBackend_ {
+  PB_MIN = 0,
+  PB_PACKRAT = PB_MIN, // PB_MIN is always the default.
+  PB_REGULAR,
+  PB_LLk,
+  PB_LALR,
+  PB_GLR,
+  PB_MAX = PB_GLR
+} HParserBackend;
+
+typedef enum HTokenType_ {
+  TT_NONE = 1,
+  TT_BYTES = 2,
+  TT_SINT = 4,
+  TT_UINT = 8,
+  TT_SEQUENCE = 16,
+  TT_RESERVED_1, // reserved for backend-specific internal use
+  TT_ERR = 32,
+  TT_USER = 64,
+  TT_MAX
+} HTokenType;
+
+typedef struct HBytes_ {
+  const uint8_t *token;
+  size_t len;
+} HBytes;
+
+typedef struct HArena_ HArena ; // hidden implementation
+
+typedef struct HCountedArray_ {
+  size_t capacity;
+  size_t used;
+  HArena * arena;
+  struct HParsedToken_ **elements;
+} HCountedArray;
+
+typedef struct HParsedToken_ {
+  HTokenType token_type;
+  union {
+    HBytes bytes;
+    int64_t sint;
+    uint64_t uint;
+    double dbl;
+    float flt;
+    HCountedArray *seq; // a sequence of HParsedToken's
+    void *user;
+  };
+  size_t index;
+  size_t bit_length;
+  char bit_offset;
+} HParsedToken;
+
+typedef struct HParseResult_ {
+  const HParsedToken *ast;
+  int64_t bit_length;
+  HArena * arena;
+} HParseResult;
+
+typedef struct HParserVtable_ HParserVtable;
+typedef struct HCFChoice_ HCFChoice;
+
+typedef struct HParser_ {
+  const HParserVtable *vtable;
+  HParserBackend backend;
+  void* backend_data;
+  void *env;
+  HCFChoice *desugared;
+} HParser;
+
+typedef struct HAllocator_ HAllocator;
+
+typedef HParsedToken* (*HAction)(const HParseResult *p, void* user_data);
+typedef bool (*HPredicate)(HParseResult *p, void* user_data);
+typedef HParser* (*HContinuation)(HAllocator *mm__, const HParsedToken *x, void *env);
+
+HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);
+HParser* h_token(const uint8_t *str, const size_t len);
+HParser* h_ch(const uint8_t c);
+HParser* h_ch_range(const uint8_t lower, const uint8_t upper);
+HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);
+HParser* h_bits(size_t len, bool sign);
+HParser* h_int64();
+HParser* h_int32();
+HParser* h_int16();
+HParser* h_int8();
+HParser* h_uint64();
+HParser* h_uint32();
+HParser* h_uint16();
+HParser* h_uint8();
+HParser* h_whitespace(const HParser* p);
+HParser* h_left(const HParser* p, const HParser* q);
+HParser* h_right(const HParser* p, const HParser* q);
+HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);
+HParser* h_action(const HParser* p, const HAction a, void* user_data);
+HParser* h_in(const uint8_t *charset, size_t length);
+HParser* h_not_in(const uint8_t *charset, size_t length);
+HParser* h_end_p();
+HParser* h_nothing_p();
+HParser* h_sequence(HParser* p, ...);
+HParser* h_choice(HParser* p, ...);
+HParser* h_permutation(HParser* p, ...);
+HParser* h_butnot(const HParser* p1, const HParser* p2);
+HParser* h_difference(const HParser* p1, const HParser* p2);
+HParser* h_xor(const HParser* p1, const HParser* p2);
+HParser* h_many(const HParser* p);
+HParser* h_many1(const HParser* p);
+HParser* h_repeat_n(const HParser* p, const size_t n);
+HParser* h_optional(const HParser* p);
+HParser* h_ignore(const HParser* p);
+HParser* h_sepBy(const HParser* p);
+HParser* h_sepBy1(const HParser* p);
+HParser* h_epsilon_p();
+HParser* h_length_value(const HParser* length, const HParser* value);
+HParser* h_attr_bool(const HParser* p, HPredicate pred, void* user_data);
+HParser* h_and(const HParser* p);
+HParser* h_not(const HParser* p);
+HParser* h_indirect(const HParser* p);
+void h_bind_indirect(HParser* indirect, const HParser* inner);
+HParser* h_with_endianness(char endianness, const HParser* p);
+HParser* h_put_value(const HParser* p, const char* name);
+HParser* h_get_value(const char* name);
+HParser* h_bind(const HParser *p, HContinuation k, void *env);
+
+int h_compile(HParser* parser, HParserBackend backend, const void* params);
+
+static const uint8_t BYTE_BIG_ENDIAN = 0x1;
+static const uint8_t BIT_BIG_ENDIAN = 0x2;
+static const uint8_t BYTE_LITTLE_ENDIAN = 0x0;
+static const uint8_t BIT_LITTLE_ENDIAN = 0x0;
+]]
+local h = ffi.load("hammer")
+
+local function helper(a, n, b, ...)
+  if   n == 0 then return a
+  else             return b, helper(a, n-1, ...) end
+end
+local function append(a, ...)
+  return helper(a, select('#', ...), ...)
+end
+
+local mt = {
+  __index = {
+    parse = function(p, str) return h.h_parse(p, str, #str) end,
+  },
+}
+local hammer = {}
+hammer.parser = ffi.metatype("HParser", mt)
+
+local counted_array
+local arr_mt = {
+  __index = function(table, key)
+    return table.elements[key]
+  end,
+  __len = function(table) return table.used end,
+  __ipairs = function(table)
+    local i, n = 0, #table
+    return function()
+      i = i + 1
+      if i <= n then
+        return i, table.elements[i]
+      end
+    end
+  end,
+  __call = function(self)
+    ret = {}
+    for i, v in ipairs(self)
+      do ret[#ret+1] = v
+    end
+    return ret
+  end
+}
+counted_array = ffi.metatype("HCountedArray", arr_mt)
+
+local bytes_mt = {
+  __call = function(self)
+    local ret = ""
+    for i = 0, tonumber(ffi.cast("uintptr_t", ffi.cast("void *", self.len)))-1
+      do ret = ret .. string.char(self.token[i])
+    end
+    return ret
+  end
+}
+local byte_string = ffi.metatype("HBytes", bytes_mt)
+
+local token_types = ffi.new("HTokenType")
+
+local parsed_token
+local tok_mt = {
+  __call = function(self)
+     if self.token_type == ffi.C.TT_BYTES then
+       return self.bytes()
+     elseif self.token_type == ffi.C.TT_SINT then
+       return tonumber(ffi.cast("intptr_t", ffi.cast("void *", self.sint)))
+     elseif self.token_type == ffi.C.TT_UINT then
+       return tonumber(ffi.cast("uintptr_t", ffi.cast("void *", self.uint)))
+     elseif self.token_type == ffi.C.TT_SEQUENCE then
+       return self.seq()
+     end
+  end
+}
+parsed_token = ffi.metatype("HParsedToken", tok_mt)
+
+function hammer.token(str)
+  return h.h_token(str, #str)
+end
+function hammer.ch(c)
+  if type(c) == "number" then
+    return h.h_ch(c)
+  else
+    return h.h_ch(c:byte())
+  end
+end
+function hammer.ch_range(lower, upper)
+  if type(lower) == "number" and type(upper) == "number" then
+    return h.h_ch_range(lower, upper)
+  -- FIXME this is really not thorough type checking
+  else
+    return h.h_ch_range(lower:byte(), upper:byte())
+  end
+end
+function hammer.int_range(parser, lower, upper)
+  return h.h_int_range(parser, lower, upper)
+end
+function hammer.bits(len, sign)
+  return h.h_bits(len, sign)
+end
+function hammer.int64()
+  return h.h_int64()
+end
+function hammer.int32()
+  return h.h_int32()
+end
+function hammer.int16()
+  return h.h_int16()
+end
+function hammer.int8()
+  return h.h_int8()
+end
+function hammer.uint64()
+  return h.h_uint64()
+end
+function hammer.uint32()
+  return h.h_uint32()
+end
+function hammer.uint16()
+  return h.h_uint16()
+end
+function hammer.uint8()
+  return h.h_uint8()
+end
+function hammer.whitespace(parser)
+  return h.h_whitespace(parser)
+end
+function hammer.left(parser1, parser2)
+  return h.h_left(parser1, parser2)
+end
+function hammer.right(parser1, parser2)
+  return h.h_right(parser1, parser2)
+end
+function hammer.middle(parser1, parser2, parser3)
+  return h.h_middle(parser1, parser2, parser3)
+end
+-- There could also be an overload of this that doesn't
+-- bother with the env pointer, and passes it as NIL by
+-- default, but I'm not going to deal with overloads now.
+function hammer.action(parser, action, user_data)
+  local cb = ffi.cast("HAction", action)
+  return h.h_action(parser, cb, user_data)
+end
+function hammer.in_(charset)
+  local cs = ffi.new("const unsigned char[" .. #charset .. "]", charset)
+  return h.h_in(cs, #charset)
+end
+function hammer.not_in(charset)
+  return h.h_not_in(charset, #charset)
+end
+function hammer.end_p()
+  return h.h_end_p()
+end
+function hammer.nothing_p()
+  return h.h_nothing_p()
+end
+function hammer.sequence(parser, ...)
+  local parsers = append(nil, ...)
+  return h.h_sequence(parser, parsers)
+end
+function hammer.choice(parser, ...)
+  local parsers = append(nil, ...)
+  return h.h_choice(parser, parsers)
+end
+function hammer.permutation(parser, ...)
+  local parsers = append(nil, ...)
+  return h.h_permutation(parser, parsers)
+end
+function hammer.butnot(parser1, parser2)
+  return h.h_butnot(parser1, parser2)
+end
+function hammer.difference(parser1, parser2)
+  return h.h_difference(parser1, parser2)
+end
+function hammer.xor(parser1, parser2)
+  return h.h_xor(parser1, parser2)
+end
+function hammer.many(parser)
+  return h.h_many(parser)
+end
+function hammer.many1(parser)
+  return h.h_many1(parser)
+end
+function hammer.repeat_n(parser, n)
+  return h.h_repeat_n(parser, n)
+end
+function hammer.optional(parser)
+  return h.h_optional(parser)
+end
+function hammer.ignore(parser)
+  return h.h_ignore(parser)
+end
+function hammer.sepBy(parser)
+  return h.h_sepBy(parser)
+end
+function hammer.sepBy1(parser)
+  return h.h_sepBy1(parser)
+end
+function hammer.epsilon_p()
+  return h.h_epsilon_p()
+end
+function hammer.length_value(length, value)
+  return h.h_length_value(length, value)
+end
+function hammer.attr_bool(parser, predicate, user_data)
+  local cb = ffi.cast("HPredicate", predicate)
+  return h.h_attr_bool(parser, cb, user_data)
+end
+function hammer.and_(parser)
+  return h.h_and(parser)
+end
+function hammer.not_(parser)
+  return h.h_not(parser)
+end
+function hammer.indirect(parser)
+  return h.h_indirect(parser)
+end
+function hammer.bind_indirect(indirect, inner)
+  return h.h_bind_indirect(indirect, inner)
+end
+function hammer.with_endianness(endianness, parser)
+  return h.h_with_endianness(endianness, parser)
+end
+function hammer.put_value(parser, name)
+  return h.h_put_value(parser, name)
+end
+function hammer.get_value(name)
+  return h.h_get_value(name)
+end
+function hammer.bind(parser, continuation, env)
+  local cb = ffi.cast("HContinuation", continuation)
+  return h.h_bind(parser, cb, env)
+end
+
+function hammer.compile(parser, backend, params)
+  return h.h_compile(parser, backend, params)
+end
+
+hammer.BYTE_BIG_ENDIAN = 0x1;
+hammer.BIT_BIG_ENDIAN = 0x2;
+hammer.BYTE_LITTLE_ENDIAN = 0x0;
+hammer.BIT_LITTLE_ENDIAN = 0x0;
+return hammer
\ No newline at end of file

src/bindings/perl/SConscript

 # -*- python -*-
+
+from __future__ import absolute_import, division, print_function
+
 import os.path
 Import("env libhammer_shared testruns targets")
 
@@ -20,7 +23,7 @@ if 'PERL5LIB' in os.environ:
 
 swig = ['hammer.i']
 
-hammer_wrap = perlenv.Command(['hammer_wrap.c', 'hammer.pm'], swig, "swig $SWIGFLAGS $SOURCE")
+hammer_wrap = perlenv.Command(['hammer_wrap.c', 'hammer.pm'], swig, "swig3.0 $SWIGFLAGS $SOURCE")
 makefile = perlenv.Command(['Makefile'], ['Makefile.PL'], "perl $SOURCE CC=" + perlenv['ENV']['CC'])
 
 targetdir = os.path.dirname(str(hammer_wrap[0].path))

src/bindings/php/SConscript

 # -*- python -*-
+
+from __future__ import absolute_import, division, print_function
+
 import os, os.path
 Import('env libhammer_shared testruns')
 
@@ -11,7 +14,7 @@ phpenv.Append(LIBS = ['hammer'])
 phpenv.Append(LIBPATH = ['../../']) 
 
 swig = ['hammer.i']
-bindings_src = phpenv.Command(['hammer.php', 'hammer_wrap.c', 'php_hammer.h'], swig, 'swig -php -DHAMMER_INTERNAL__NO_STDARG_H -Isrc/ $SOURCE')
+bindings_src = phpenv.Command(['hammer.php', 'hammer_wrap.c', 'php_hammer.h'], swig, 'swig3.0 -php -DHAMMER_INTERNAL__NO_STDARG_H -Isrc/ $SOURCE')
 libhammer_php = phpenv.SharedLibrary('hammer', ['hammer_wrap.c'])
 Default(swig, bindings_src, libhammer_php)
 

src/bindings/python/SConscript

 # -*- python -*-
+
+from __future__ import absolute_import, division, print_function
+
 import os, os.path
 Import('env libhammer_shared testruns targets')
 
@@ -7,17 +10,18 @@ pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0)
 swig = pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE"))
 setup = ['setup.py']
 pydir = os.path.join(env['BUILD_BASE'], 'src/bindings/python')
-libhammer_python = pythonenv.Command(['hammer.py', 'hammer_wrap.c'], [swig, setup], 'python ' + os.path.join(pydir, 'setup.py') + ' build_ext --inplace')
+pysetup = os.path.join(pydir, 'setup.py')
+libhammer_python = pythonenv.Command(['hammer.py', 'hammer_wrap.c'], [swig, setup],  '%s %s build_ext --inplace' % (env['python'], pysetup))
 Default(libhammer_python)
 
 pytestenv = pythonenv.Clone()
 pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0]))
 pytests = ['hammer_tests.py']
-pytestexec = pytestenv.Command(['hammer.pyc', 'hammer_tests.pyc'], pytests + libhammer_python, "LD_LIBRARY_PATH=" + os.path.dirname(str(libhammer_shared[0])) + " nosetests -vv $SOURCE")
+pytestexec = pytestenv.Command(['hammer.pyc', 'hammer_tests.pyc'], pytests + libhammer_python, "LD_LIBRARY_PATH=%s %s -mnose -vv $SOURCE" % (os.path.dirname(str(libhammer_shared[0])), env['python']))
 pytest = Alias("testpython", [pytestexec], pytestexec)
 AlwaysBuild(pytestexec)
 testruns.append(pytest)
 
-pyinstallexec = pythonenv.Command(None, libhammer_python, 'python ' + os.path.join(pydir, 'setup.py ') + ' install')
+pyinstallexec = pythonenv.Command(None, libhammer_python, '%s %s install' % (env['python'], pysetup))
 pyinstall = Alias("installpython", [pyinstallexec], pyinstallexec)
 targets.append(pyinstall)

src/bindings/ruby/SConscript

 # -*- python -*-
+
+from __future__ import absolute_import, division, print_function
+
 import os.path
 Import("env libhammer_shared testruns targets")
 

src/bindings/ruby/hammer-parser.gemspec

 #encoding: UTF-8
 Gem::Specification.new do |s|
   s.name          = 'hammer-parser'
-  s.version       = '0.1.0'
+  s.version       = '0.2.0'
   s.summary       = 'Ruby bindings to the hammer parsing library.'
   s.description   = s.summary # TODO: longer description?
   s.authors       = ['Meredith L. Patterson', 'TQ Hirsch', 'Jakob Rath']
@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
     "lib/hammer/internal.rb",
     "lib/hammer/parser.rb",
     "lib/hammer/parser_builder.rb",
-    "lib/hammer.rb",
+    "lib/hammer-parser.rb",
     "lib/minitest/hamer-parser_plugin.rb",
     "test/autogen_test.rb",
     "test/parser_test.rb"

src/bindings/ruby/lib/hammer/parser.rb

@@ -19,8 +19,10 @@ module Hammer
       @dont_gc = dont_gc.dup
     end
 
+    # dont_gc is required to build a fuzzer from the declaration of Hammer::Parser object.
     attr_reader :name
     attr_reader :h_parser
+    attr_reader :dont_gc
 
     # Parse the given data. Returns the parse result if successful, nil otherwise.
     #

src/bindings/swig/hammer.i

 %module hammer
+%begin %{
+#define SWIG_PYTHON_STRICT_BYTE_CHAR
+%}
 
 %nodefaultctor;
 
@@ -25,6 +28,20 @@
  }
 
 %pythoncode %{
+  try:
+      INTEGER_TYPES = (int, long)
+  except NameError:
+      INTEGER_TYPES = (int,)
+
+  try:
+      TEXT_TYPE = unicode
+      def bchr(i):
+          return chr(i)
+  except NameError:
+      TEXT_TYPE = str
+      def bchr(i):
+          return bytes([i])
+
   class Placeholder(object):
       """The python equivalent of TT_NONE"""
       def __str__(self):
@@ -69,11 +86,11 @@
     PyErr_SetString(PyExc_ValueError, "Expecting a string");
     return NULL;
   } else {
-    $1 = *(uint8_t*)PyString_AsString($input);
+    $1 = *(uint8_t*)PyBytes_AsString($input);
   }
  }
 %typemap(out) HBytes* {
-  $result = PyString_FromStringAndSize((char*)$1->token, $1->len);
+  $result = PyBytes_FromStringAndSize((char*)$1->token, $1->len);
  }
 %typemap(out) struct HCountedArray_* {
   int i;
@@ -173,7 +190,7 @@
       return PyObject_CallFunctionObjArgs(_helper_Placeholder, NULL);
       break;
     case TT_BYTES:
-      return PyString_FromStringAndSize((char*)token->token_data.bytes.token, token->token_data.bytes.len);
+      return PyBytes_FromStringAndSize((char*)token->token_data.bytes.token, token->token_data.bytes.len);
     case TT_SINT:
       // TODO: return PyINT if appropriate
       return PyLong_FromLong(token->token_data.sint);
@@ -250,36 +267,35 @@
 }
 
 %pythoncode %{
-
 def action(p, act):
     return _h_action(p, act)
 def attr_bool(p, pred):
     return _h_attr_bool(p, pred)
 
 def ch(ch):
-    if isinstance(ch, str) or isinstance(ch, unicode):
+    if isinstance(ch, (bytes, TEXT_TYPE)):
         return token(ch)
     else:
         return  _h_ch(ch)
 
 def ch_range(c1, c2):
-    dostr = isinstance(c1, str)
-    dostr2 = isinstance(c2, str)
-    if isinstance(c1, unicode) or isinstance(c2, unicode):
+    dostr = isinstance(c1, bytes)
+    dostr2 = isinstance(c2, bytes)
+    if isinstance(c1, TEXT_TYPE) or isinstance(c2, TEXT_TYPE):
         raise TypeError("ch_range only works on bytes")
     if dostr != dostr2:
         raise TypeError("Both arguments to ch_range must be the same type")
     if dostr:
-        return action(_h_ch_range(c1, c2), chr)
+        return action(_h_ch_range(c1, c2), bchr)
     else:
         return _h_ch_range(c1, c2)
 def epsilon_p(): return _h_epsilon_p()
 def end_p():
     return _h_end_p()
 def in_(charset):
-    return action(_h_in(charset), chr)
+    return action(_h_in(charset), bchr)
 def not_in(charset):
-    return action(_h_not_in(charset), chr)
+    return action(_h_not_in(charset), bchr)
 def not_(p): return _h_not(p)
 def int_range(p, i1, i2):
     return _h_int_range(p, i1, i2)

src/bitreader.c

@@ -108,3 +108,77 @@ int64_t h_read_bits(HInputStream* state, int count, char signed_p) {
   out <<= final_shift;
   return (out ^ msb) - msb; // perform sign extension
 }
+
+void h_skip_bits(HInputStream* stream, size_t count) {
+  size_t left;
+
+  if (count == 0)
+    return;
+
+  if (stream->overrun)
+    return;
+
+  if (stream->index == stream->length) {
+    stream->overrun = true;
+    return;
+  }
+
+  // consume from a partial byte?
+  left = 8 - stream->bit_offset - stream->margin;
+  if (count < left) {
+    stream->bit_offset += count;
+    return;
+  }
+  if (left < 8) {
+    stream->index += 1;
+    stream->bit_offset = 0;
+    stream->margin = 0;
+    count -= left;
+  }
+  assert(stream->bit_offset == 0);
+  assert(stream->margin == 0);
+
+  // consume full bytes
+  left = stream->length - stream->index;
+  if (count / 8 <= left) {
+    stream->index += count / 8;
+    count = count % 8;
+  } else {
+    stream->index = stream->length;
+    stream->overrun = true;
+    return;
+  }
+  assert(count < 8);
+
+  // final partial byte
+  if (count > 0 && stream->index == stream->length)
+    stream->overrun = true;
+  else
+    stream->bit_offset = count;
+}
+
+void h_seek_bits(HInputStream* stream, size_t pos) {
+  size_t pos_index = pos / 8;
+  size_t pos_offset = pos % 8;
+
+  /* seek within the current byte? */
+  if (pos_index == stream->index) {
+    stream->bit_offset = pos_offset;
+    return;
+  }
+
+  stream->margin = 0;
+
+  /* seek past the end? */
+  if ((pos_index > stream->length) ||
+      (pos_index == stream->length && pos_offset > 0)) {
+    stream->index = stream->length;
+    stream->bit_offset = 0;
+    stream->overrun = true;
+    return;
+  }
+
+  stream->index = pos_index;
+  stream->bit_offset = pos_offset;
+  stream->margin = 0;
+}

src/cfgrammar.h

@@ -8,15 +8,15 @@ typedef struct HCFGrammar_ {
   HHashSet    *nts;     // HCFChoices, each representing the alternative
                         // productions for one nonterminal
   HHashSet    *geneps;  // set of NTs that can generate the empty string
-  HHashTable  **first;  // memoized first sets of the grammar's symbols
-  HHashTable  **follow; // memoized follow sets of the grammar's NTs
-  size_t      kmax;     // maximum lookahead depth allocated
+  HHashTable  *first;   // memoized first sets of the grammar's symbols
+  HHashTable  *follow;  // memoized follow sets of the grammar's NTs
   HArena      *arena;
   HAllocator  *mm__;
 
-  // constant set containing only the empty string.
-  // this is only a member of HCFGrammar because it needs a pointer to arena.
+  // constant sets containing only the empty string or end symbol.
+  // these are only members of HCFGrammar because they need a pointer to arena.
   const struct HStringMap_ *singleton_epsilon;
+  const struct HStringMap_ *singleton_end;
 } HCFGrammar;
 
 
@@ -37,6 +37,7 @@ typedef struct HStringMap_ {
   void *end_branch;             // points to leaf value
   HHashTable *char_branches;    // maps to inner nodes (HStringMaps)
   HArena *arena;
+  bool taint;                   // for use by h_follow() and h_first()
 } HStringMap;
 
 HStringMap *h_stringmap_new(HArena *a);
@@ -52,6 +53,7 @@ void *h_stringmap_get_lookahead(const HStringMap *m, HInputStream lookahead);
 bool h_stringmap_present(const HStringMap *m, const uint8_t *str, size_t n, bool end);
 bool h_stringmap_present_epsilon(const HStringMap *m);
 bool h_stringmap_empty(const HStringMap *m);
+bool h_stringmap_equal(const HStringMap *a, const HStringMap *b);
 
 static inline HStringMap *h_stringmap_get_char(const HStringMap *m, const uint8_t c)
  { return h_hashtable_get(m->char_branches, (void *)char_key(c)); }