src/bindings/ruby/test/autogen_test.rb

 require 'bundler/setup'
 require 'minitest/autorun'
-require 'hammer'
+require 'hammer-parser'
 class TestToken < Minitest::Test
   def setup
     super

src/bindings/ruby/test/parser_test.rb

 # -*- coding: utf-8 -*-
 require 'bundler/setup'
-require 'hammer'
+require 'hammer-parser'
 require 'minitest/autorun'
 
 class ParserTest < Minitest::Test

src/bindings/swig/hammer.i

 %module hammer
+%begin %{
+#define SWIG_PYTHON_STRICT_BYTE_CHAR
+%}
 
 %nodefaultctor;
 
@@ -25,6 +28,20 @@
  }
 
 %pythoncode %{
+  try:
+      INTEGER_TYPES = (int, long)
+  except NameError:
+      INTEGER_TYPES = (int,)
+
+  try:
+      TEXT_TYPE = unicode
+      def bchr(i):
+          return chr(i)
+  except NameError:
+      TEXT_TYPE = str
+      def bchr(i):
+          return bytes([i])
+
   class Placeholder(object):
       """The python equivalent of TT_NONE"""
       def __str__(self):
@@ -69,11 +86,11 @@
     PyErr_SetString(PyExc_ValueError, "Expecting a string");
     return NULL;
   } else {
-    $1 = *(uint8_t*)PyString_AsString($input);
+    $1 = *(uint8_t*)PyBytes_AsString($input);
   }
  }
 %typemap(out) HBytes* {
-  $result = PyString_FromStringAndSize((char*)$1->token, $1->len);
+  $result = PyBytes_FromStringAndSize((char*)$1->token, $1->len);
  }
 %typemap(out) struct HCountedArray_* {
   int i;
@@ -173,7 +190,7 @@
       return PyObject_CallFunctionObjArgs(_helper_Placeholder, NULL);
       break;
     case TT_BYTES:
-      return PyString_FromStringAndSize((char*)token->token_data.bytes.token, token->token_data.bytes.len);
+      return PyBytes_FromStringAndSize((char*)token->token_data.bytes.token, token->token_data.bytes.len);
     case TT_SINT:
       // TODO: return PyINT if appropriate
       return PyLong_FromLong(token->token_data.sint);
@@ -250,36 +267,35 @@
 }
 
 %pythoncode %{
-
 def action(p, act):
     return _h_action(p, act)
 def attr_bool(p, pred):
     return _h_attr_bool(p, pred)
 
 def ch(ch):
-    if isinstance(ch, str) or isinstance(ch, unicode):
+    if isinstance(ch, (bytes, TEXT_TYPE)):
         return token(ch)
     else:
         return  _h_ch(ch)
 
 def ch_range(c1, c2):
-    dostr = isinstance(c1, str)
-    dostr2 = isinstance(c2, str)
-    if isinstance(c1, unicode) or isinstance(c2, unicode):
+    dostr = isinstance(c1, bytes)
+    dostr2 = isinstance(c2, bytes)
+    if isinstance(c1, TEXT_TYPE) or isinstance(c2, TEXT_TYPE):
         raise TypeError("ch_range only works on bytes")
     if dostr != dostr2:
         raise TypeError("Both arguments to ch_range must be the same type")
     if dostr:
-        return action(_h_ch_range(c1, c2), chr)
+        return action(_h_ch_range(c1, c2), bchr)
     else:
         return _h_ch_range(c1, c2)
 def epsilon_p(): return _h_epsilon_p()
 def end_p():
     return _h_end_p()
 def in_(charset):
-    return action(_h_in(charset), chr)
+    return action(_h_in(charset), bchr)
 def not_in(charset):
-    return action(_h_not_in(charset), chr)
+    return action(_h_not_in(charset), bchr)
 def not_(p): return _h_not(p)
 def int_range(p, i1, i2):
     return _h_int_range(p, i1, i2)

src/bitreader.c

@@ -108,3 +108,77 @@ int64_t h_read_bits(HInputStream* state, int count, char signed_p) {
   out <<= final_shift;
   return (out ^ msb) - msb; // perform sign extension
 }
+
+void h_skip_bits(HInputStream* stream, size_t count) {
+  size_t left;
+
+  if (count == 0)
+    return;
+
+  if (stream->overrun)
+    return;
+
+  if (stream->index == stream->length) {
+    stream->overrun = true;
+    return;
+  }
+
+  // consume from a partial byte?
+  left = 8 - stream->bit_offset - stream->margin;
+  if (count < left) {
+    stream->bit_offset += count;
+    return;
+  }
+  if (left < 8) {
+    stream->index += 1;
+    stream->bit_offset = 0;
+    stream->margin = 0;
+    count -= left;
+  }
+  assert(stream->bit_offset == 0);
+  assert(stream->margin == 0);
+
+  // consume full bytes
+  left = stream->length - stream->index;
+  if (count / 8 <= left) {
+    stream->index += count / 8;
+    count = count % 8;
+  } else {
+    stream->index = stream->length;
+    stream->overrun = true;
+    return;
+  }
+  assert(count < 8);
+
+  // final partial byte
+  if (count > 0 && stream->index == stream->length)
+    stream->overrun = true;
+  else
+    stream->bit_offset = count;
+}
+
+void h_seek_bits(HInputStream* stream, size_t pos) {
+  size_t pos_index = pos / 8;
+  size_t pos_offset = pos % 8;
+
+  /* seek within the current byte? */
+  if (pos_index == stream->index) {
+    stream->bit_offset = pos_offset;
+    return;
+  }
+
+  stream->margin = 0;
+
+  /* seek past the end? */
+  if ((pos_index > stream->length) ||
+      (pos_index == stream->length && pos_offset > 0)) {
+    stream->index = stream->length;
+    stream->bit_offset = 0;
+    stream->overrun = true;
+    return;
+  }
+
+  stream->index = pos_index;
+  stream->bit_offset = pos_offset;
+  stream->margin = 0;
+}

src/bitwriter.c

@@ -12,10 +12,8 @@
 HBitWriter *h_bit_writer_new(HAllocator* mm__) {
   HBitWriter *writer = h_new(HBitWriter, 1);
   memset(writer, 0, sizeof(*writer));
-  writer->buf = mm__->alloc(mm__, writer->capacity = 8);
-  if (!writer) {
-    return NULL;
-  }
+  writer->buf = h_alloc(mm__, writer->capacity = 8);
+  assert(writer != NULL);
   memset(writer->buf, 0, writer->capacity);
   writer->mm__ = mm__;
   writer->flags = BYTE_BIG_ENDIAN | BIT_BIG_ENDIAN;

src/cfgrammar.h

@@ -8,15 +8,15 @@ typedef struct HCFGrammar_ {
   HHashSet    *nts;     // HCFChoices, each representing the alternative
                         // productions for one nonterminal
   HHashSet    *geneps;  // set of NTs that can generate the empty string
-  HHashTable  **first;  // memoized first sets of the grammar's symbols
-  HHashTable  **follow; // memoized follow sets of the grammar's NTs
-  size_t      kmax;     // maximum lookahead depth allocated
+  HHashTable  *first;   // memoized first sets of the grammar's symbols
+  HHashTable  *follow;  // memoized follow sets of the grammar's NTs
   HArena      *arena;
   HAllocator  *mm__;
 
-  // constant set containing only the empty string.
-  // this is only a member of HCFGrammar because it needs a pointer to arena.
+  // constant sets containing only the empty string or end symbol.
+  // these are only members of HCFGrammar because they need a pointer to arena.
   const struct HStringMap_ *singleton_epsilon;
+  const struct HStringMap_ *singleton_end;
 } HCFGrammar;
 
 
@@ -37,6 +37,7 @@ typedef struct HStringMap_ {
   void *end_branch;             // points to leaf value
   HHashTable *char_branches;    // maps to inner nodes (HStringMaps)
   HArena *arena;
+  bool taint;                   // for use by h_follow() and h_first()
 } HStringMap;
 
 HStringMap *h_stringmap_new(HArena *a);
@@ -52,6 +53,7 @@ void *h_stringmap_get_lookahead(const HStringMap *m, HInputStream lookahead);
 bool h_stringmap_present(const HStringMap *m, const uint8_t *str, size_t n, bool end);
 bool h_stringmap_present_epsilon(const HStringMap *m);
 bool h_stringmap_empty(const HStringMap *m);
+bool h_stringmap_equal(const HStringMap *a, const HStringMap *b);
 
 static inline HStringMap *h_stringmap_get_char(const HStringMap *m, const uint8_t c)
  { return h_hashtable_get(m->char_branches, (void *)char_key(c)); }

src/parsers/ch.c

@@ -8,6 +8,9 @@ static HParseResult* parse_ch(void* env, HParseState *state) {
   if (c == r) {
     HParsedToken *tok = a_new(HParsedToken, 1);    
     tok->token_type = TT_UINT; tok->uint = r;
+    tok->index = 0;
+    tok->bit_length = 0;
+    tok->bit_offset = 0;
     return make_result(state->arena, tok);
   } else {
     return NULL;