.travis.yml

@@ -2,6 +2,9 @@ language: c
 compiler:
   - gcc
   - clang
+before_install:
+  - sudo apt-get update -qq
+  - sudo apt-get install -qq swig php5-dev
 script: 
   - scons
 notifications:

src/SConscript

 # -*- python -*-
 Import('env')
 
-bindings = []
+bindings = ['php']
 
 dist_headers = [
     "hammer.h",

src/allocator.h

@@ -29,7 +29,11 @@ typedef struct HAllocator_ {
 typedef struct HArena_ HArena ; // hidden implementation
 
 HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for default...
+#ifndef SWIG
 void* h_arena_malloc(HArena *arena, size_t count) __attribute__(( malloc, alloc_size(2) ));
+#else
+void* h_arena_malloc(HArena *arena, size_t count);
+#endif
 void h_arena_free(HArena *arena, void* ptr); // For future expansion, with alternate memory managers.
 void h_delete_arena(HArena *arena);
 

src/bindings/desugar-header.pl

+#!/usr/bin/perl -w
+
+
+my $arg = qr/[^,]*/;
+
+while(<>) {
+  chomp;
+  if (/^HAMMER_FN_DECL_NOARG\(([^,]*), ([^,]*)\);/) {
+    print "$1 $2(void);\n";
+    print "$1 $2__m(HAllocator* mm__);\n";
+  } elsif (/^HAMMER_FN_DECL\(([^,]*), ([^,]*), ([^)]*)\);/) {
+    print "$1 $2($3);\n";
+    print "$1 $2__m(HAllocator* mm__, $3);\n";
+  } elsif (/^HAMMER_FN_DECL_VARARGS_ATTR\((__attribute__\(\([^)]*\)\)), ([^,]*), ([^,]*), ([^)]*)\);/) {
+    print "$2 $3($4, ...);\n";
+    print "$2 $3__m(HAllocator *mm__, $4, ...);\n";
+    print "$2 $3__a(void* args);\n";
+    print "$2 $3__ma(HAllocator* mm__, void* args);\n";
+  } elsif (/^HAMMER_FN_DECL/) {
+    print "\e[1;31m!!!\e[0m " . $_ . "\n";
+  }
+}

src/bindings/php/SConscript

+# -*- python -*-
+Import('env')
+
+phpenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0)
+
+phpenv.Append(CPPPATH = ['../../', '/usr/include/php5', '/usr/include/php5/main', '/usr/include/php5/Zend', '/usr/include/php5/TSRM'])
+phpenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error'])
+phpenv.Append(LIBS = ['hammer'])
+phpenv.Append(LIBPATH = ['../../']) 
+phpenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-php'])
+
+phpenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE"))
+
+swig = ['hammer.i']
+
+libhammer_php = phpenv.SharedLibrary('hammer', swig)

src/bindings/python/hammer.py

+from cffi import FFI
+import threading
+import sys
+
+_ffi = FFI()
+
+# {{{ Types
+
+_ffi.cdef("typedef struct HAllocator_ HAllocator;")
+_ffi.cdef("typedef struct HArena_ HArena;")
+_ffi.cdef("typedef int bool;")
+_ffi.cdef("typedef struct HParseState_ HParseState;")
+_ffi.cdef("""
+typedef enum HParserBackend_ {
+  PB_MIN = 0,
+  PB_PACKRAT = 0, // PB_MIN is always the default.
+  PB_REGULAR,
+  PB_LLk,
+  PB_LALR,
+  PB_GLR
+// TODO: support PB_MAX
+} HParserBackend;
+""")
+_ffi.cdef("""
+typedef enum HTokenType_ {
+  // Before you change the explicit values of these, think of the poor bindings ;_;
+  TT_NONE = 1,
+  TT_BYTES = 2,
+  TT_SINT = 4,
+  TT_UINT = 8,
+  TT_SEQUENCE = 16,
+  TT_RESERVED_1, // reserved for backend-specific internal use
+  TT_ERR = 32,
+  TT_USER = 64,
+  TT_MAX
+} HTokenType;
+""")
+_ffi.cdef("""
+typedef struct HCountedArray_ {
+  size_t capacity;
+  size_t used;
+  HArena * arena;
+  struct HParsedToken_ **elements;
+} HCountedArray;
+""")
+_ffi.cdef("""
+typedef struct HBytes_ {
+  const uint8_t *token;
+  size_t len;
+} HBytes;
+""")
+_ffi.cdef("""
+typedef struct HParsedToken_ {
+  HTokenType token_type;
+  union {
+    HBytes bytes;
+    int64_t sint;
+    uint64_t uint;
+    double dbl;
+    float flt;
+    HCountedArray *seq; // a sequence of HParsedToken's
+    void *user;
+  };
+  size_t index;
+  char bit_offset;
+} HParsedToken;
+""")
+_ffi.cdef("""
+typedef struct HParseResult_ {
+  const HParsedToken *ast;
+  long long bit_length;
+  HArena * arena;
+} HParseResult;
+""")
+
+_ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""")
+_ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""")
+_ffi.cdef("""
+typedef struct HCFChoice_ HCFChoice;
+typedef struct HRVMProg_ HRVMProg;
+typedef struct HParserVtable_ HParserVtable;
+""")
+
+_ffi.cdef("typedef struct HParser_ HParser;")
+_ffi.cdef("""
+typedef struct HParserTestcase_ {
+  unsigned char* input;
+  size_t length;
+  char* output_unambiguous;
+} HParserTestcase;
+
+typedef struct HCaseResult_ {
+  bool success;
+  union {
+    const char* actual_results; // on failure, filled in with the results of h_write_result_unamb
+    size_t parse_time; // on success, filled in with time for a single parse, in nsec
+  };
+} HCaseResult;
+
+typedef struct HBackendResults_ {
+  HParserBackend backend;
+  bool compile_success;
+  size_t n_testcases;
+  size_t failed_testcases; // actually a count...
+  HCaseResult *cases;
+} HBackendResults;
+
+typedef struct HBenchmarkResults_ {
+  size_t len;
+  HBackendResults *results;
+} HBenchmarkResults;
+""")
+
+# }}}
+# {{{ Arena functions
+_ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);")
+_ffi.cdef("void h_arena_free(HArena *arena, void* ptr);")
+# }}}
+# {{{ cdefs
+## The following section was generated by
+## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/'
+_ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);")
+_ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);")
+_ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);")
+_ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);")
+_ffi.cdef("HParser* h_ch(const uint8_t c);")
+_ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);")
+_ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);")
+_ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);")
+_ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);")
+_ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);")
+_ffi.cdef("HParser* h_bits(size_t len, bool sign);")
+_ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);")
+_ffi.cdef("HParser* h_int64(void);")
+_ffi.cdef("HParser* h_int64__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_int32(void);")
+_ffi.cdef("HParser* h_int32__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_int16(void);")
+_ffi.cdef("HParser* h_int16__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_int8(void);")
+_ffi.cdef("HParser* h_int8__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_uint64(void);")
+_ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_uint32(void);")
+_ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_uint16(void);")
+_ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_uint8(void);")
+_ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_whitespace(const HParser* p);")
+_ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);")
+_ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);")
+_ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);")
+_ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);")
+_ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);")
+_ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);")
+_ffi.cdef("HParser* h_action(const HParser* p, const HAction a);")
+_ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);")
+_ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);")
+_ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);")
+_ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);")
+_ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);")
+_ffi.cdef("HParser* h_end_p(void);")
+_ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_nothing_p(void);")
+_ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_sequence(HParser* p, ...);")
+_ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);")
+_ffi.cdef("HParser* h_sequence__a(void* args);")
+_ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);")
+_ffi.cdef("HParser* h_choice(HParser* p, ...);")
+_ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);")
+_ffi.cdef("HParser* h_choice__a(void* args);")
+_ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);")
+_ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_many(const HParser* p);")
+_ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_many1(const HParser* p);")
+_ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);")
+_ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);")
+_ffi.cdef("HParser* h_optional(const HParser* p);")
+_ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_ignore(const HParser* p);")
+_ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);")
+_ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);")
+_ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);")
+_ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);")
+_ffi.cdef("HParser* h_epsilon_p(void);")
+_ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);")
+_ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);")
+_ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);")
+_ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);")
+_ffi.cdef("HParser* h_and(const HParser* p);")
+_ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_not(const HParser* p);")
+_ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_indirect(void);")
+_ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);")
+_ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);")
+_ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);")
+_ffi.cdef("void h_parse_result_free(HParseResult *result);")
+_ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);")
+_ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);")
+_ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);")
+_ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);")
+_ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);")
+_ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);")
+
+_lib = _ffi.verify("#include <hammer/hammer.h>",
+                 libraries=['hammer'])
+
+_lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45
+# }}}
+class _DynamicScopeHolder(threading.local):
+    """A dynamically-scoped holder of python objects, which may or may not
+    otherwise appear in the object graph. Intended for use with CFFI """
+    def __init__(self):
+        self._ctxstack = []
+    def __enter__(self):
+        self._ctxstack.append([])
+    def __exit__(self, exc_type, exc_value, traceback):
+        self._ctxstack.pop()
+        return False
+    def stash(self, *objs):
+        if len(self._ctxstack) < 1:
+            raise Exception("Not in any dynamic scope")
+        for obj in objs:
+            self._ctxstack[-1].append(obj)
+def _fromHParsedToken(cobj):
+    # TODO: Free the toplevel parser
+    tt = cobj.token_type
+
+    if cobj.token_type == _lib.TT_BYTES:
+        return _ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:]
+    elif cobj.token_type == _lib.TT_ERR:
+        # I have no idea what this is for
+        pass
+    elif cobj.token_type == _lib.TT_NONE:
+        return None
+    elif cobj.token_type == _lib.TT_SEQUENCE:
+        return [_fromHParsedToken(cobj.seq.elements[i])
+                for i in range(cobj.seq.used)]
+    elif cobj.token_type == _lib.TT_SINT:
+        return cobj.sint
+    elif cobj.token_type == _lib.TT_UINT:
+        return cobj.uint
+    elif cobj.token_type == _lib.TT_PYTHON:
+        return _ffi.from_handle(cobj.user)
+
+_parser_result_holder = _DynamicScopeHolder()
+def _toHParsedToken(arena, pyobj):
+    if pyobj is None:
+        return _ffi.NULL
+    cobj = _ffi.new_handle(pyobj)
+    _parser_result_holder.stash(cobj)
+
+    hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(arena, _ffi.sizeof("HParsedToken")))
+    hpt.token_type = _lib.TT_PYTHON
+    hpt.user = cobj
+    hpt.bit_offset = chr(127)
+    hpt.index = 0
+    return hpt
+
+def _fromParseResult(cobj):
+    ret = _fromHParsedToken(cobj.ast)
+    _lib.h_parse_result_free(cobj)
+    return ret
+
+def _to_haction(fn):
+    """Turn a function that transforms a parsed value into an HAction"""
+    def action(parse_result):
+        res = _toHParsedToken(parse_result.arena,  fn(_fromParseResult(parse_result)))
+        if res != _ffi.NULL and parse_result.ast != _ffi.NULL:
+            res.index = parse_result.ast.index
+            res.bit_offset = parse_result.ast.bit_offset
+        return res
+    return _ffi.callback("HParsedToken*(HParseResult*)", action)
+
+def _to_hpredicate(fn):
+    """Turn a function that transforms a parsed value into an HAction"""
+    def predicate(parse_result):
+        res = fn(_fromParseResult(parse_result))
+        # TODO: Handle exceptions; parse should fail.
+        if type(res) != bool:
+            raise TypeError("Predicates should return a bool")
+        return res
+    return _ffi.callback("bool(HParseResult*)", predicate)
+
+class Parser(object):
+    # TODO: Map these to individually garbage-collected blocks of
+    # memory. Perhaps with an arena allocator with block size of 1?
+    # There has to be something more efficient than that, though.
+
+    # TODO: How do we handle encodings? By default, we're using UTF-8
+    def __init__(self, internal, deps):
+        """Create a new parser from an FFI object. Not for user code"""
+        self._parser = internal
+        self._deps = deps
+
+    def parse(self, string):
+        with _parser_result_holder:
+            pres = _lib.h_parse(self._parser, string, len(string))
+            if pres:
+                return _fromParseResult(pres)
+            else:
+                return None
+
+    def __mul__(self, count):
+        return repeat_n(self, count)
+
+
+                
+class IndirectParser(Parser):
+    def bind(self, inner):
+        _lib.h_bind_indirect(self._parser, inner._parser)
+        self._deps = (inner,)
+
+class BitsParser(Parser):
+    pass
+
+def token(token):
+    # TODO: Does not clone argument.
+    if isinstance(token, unicode):
+        token = token.encode("utf-8")
+    return Parser(_lib.h_token(token, len(token)), ())
+
+def ch(char):
+    """Returns either a token or an int, depending on the type of the
+    argument"""
+    if isinstance(char, int):
+        return Parser(_lib.h_ch(char), ())
+    else:
+        return token(char)
+
+def ch_range(chr1, chr2):
+    if not isinstance(chr1, str) or not isinstance(chr2, str):
+        raise TypeError("ch_range can't handle unicode")
+    def my_action(pr):
+        # print "In action: ", pr
+        return pr
+    return action(Parser(_lib.h_ch_range(ord(chr1), ord(chr2)), ()), my_action)
+
+def int_range(parser, i1, i2):
+    if type(parser) != BitsParser:
+        raise TypeError("int_range is only valid when used with a bits parser")
+    return Parser(_lib.h_int_range(parser._parser, i1, i2), (parser,))
+
+def bits(length, signedp):
+    return BitsParser(_lib.h_bits(length, signedp), ())
+
+def int64(): return bits(64, True)
+def int32(): return bits(32, True)
+def int16(): return bits(16, True)
+def int8 (): return bits(8,  True)
+def uint64(): return bits(64, False)
+def uint32(): return bits(32, False)
+def uint16(): return bits(16, False)
+def uint8 (): return bits(8,  False)
+
+def whitespace(p):
+    return Parser(_lib.h_whitespace(p._parser), (p,))
+def left(p1, p2):
+    return Parser(_lib.h_left(p1._parser, p2._parser), (p1, p2))
+def right(p1, p2):
+    return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2))
+def middle(p1, p2, p3):
+    return Parser(_lib.h_middle(p1._parser, p2._parser, p3._parser), (p1, p2, p3))
+def action(parser, action):
+    caction = _to_haction(action)
+    return Parser(_lib.h_action(parser._parser, caction), (parser, caction))
+
+def in_(charset):
+    if not isinstance(charset, str):
+        # TODO/Python3: change str to bytes
+        raise TypeError("in_ can't deal with unicode")
+    return Parser(_lib.h_in(charset, len(charset)), ())
+def not_in(charset):
+    if not isinstance(charset, str):
+        # TODO/Python3: change str to bytes
+        raise TypeError("in_ can't deal with unicode")
+    return Parser(_lib.h_not_in(charset, len(charset)), ())
+def end_p():
+    return Parser(_lib.h_end_p(), ())
+def nothing_p():
+    return Parser(_lib.h_nothing_p(), ())
+def sequence(*parsers):
+    plist = [p._parser for p in parsers]
+    plist.append(_ffi.NULL)
+    return Parser(_lib.h_sequence(*plist), (plist,))
+def choice(*parsers):
+    plist = [p._parser for p in parsers]
+    plist.append(_ffi.NULL)
+    return Parser(_lib.h_choice(*plist), (plist,))
+def butnot(p1, p2):
+    return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2))
+def difference(p1, p2):
+    return Parser(_lib.h_difference(p1._parser, p2._parser), (p1, p2))
+def xor(p1, p2):
+    return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2))
+def many(p1):
+    return Parser(_lib.h_many(p1._parser), (p1,))
+def many1(p1):
+    return Parser(_lib.h_many1(p1._parser), (p1,))
+def repeat_n(p1, n):
+    return Parser(_lib.h_repeat_n(p1._parser, n), (p1,))
+def optional(p1):
+    return Parser(_lib.h_optional(p1._parser), (p1,))
+def ignore(p1):
+    return Parser(_lib.h_ignore(p1._parser), (p1,))
+def sepBy(p, sep):
+    return Parser(_lib.h_sepBy(p._parser, sep._parser), (p, sep))
+def sepBy1(p, sep):
+    return Parser(_lib.h_sepBy1(p._parser, sep._parser), (p, sep))
+def epsilon_p():
+    return Parser(_lib.h_epsilon_p(), ())
+def length_value(p_len, p_value):
+    return Parser(_lib.h_length_value(p_len._parser, p_value._parser), (p_len, p_value))
+def attr_bool(parser, predicate):
+    cpredicate = _to_hpredicate(predicate)
+    return Parser(_lib.h_attr_bool(parser._parser, cpredicate), (parser, cpredicate))
+def and_(parser):
+    return Parser(_lib.h_and(parser._parser), (parser,))
+def not_(parser):
+    return Parser(_lib.h_not(parser._parser), (parser,))
+def indirect():
+    return IndirectParser(_lib.h_indirect(), ())
+def bind_indirect(indirect, inner):
+    indirect.bind(inner)
+
+def parse(parser):
+    return parser.parse()
+
+# Unfortunately, "in", "and", and "not" are keywords. This makes them
+# show up in the module namespace for the use of automated tools. Do
+# not attempt to use them by hand; only use the mangled forms (with
+# the '_')
+sys.modules[__name__].__dict__["in"] = in_
+sys.modules[__name__].__dict__["and"] = and_
+sys.modules[__name__].__dict__["not"] = not_
+
+def run_test():
+    p_test = sepBy1(choice(ch('1'),
+                           ch('2'),
+                           ch('3')),
+                    ch(','))
+    return p_test.parse("1,2,3")
+
+# {{{ Automatic parser construction... python specific
+
+# TODO: Implement Parsable metaclass, which requires the existence of
+# a "parse" method.
+
+# This is expected to be extended by user code. As a general rule,
+# only provide auto-parsers for your own types.
+AUTO_PARSERS = {
+    str: token,
+    unicode: token,
+}
+
+def _auto_seq(lst):
+    return sequence(*(auto_1(p, default_method=_auto_choice)
+                      for p in lst))
+        
+def _auto_choice(lst):
+    return choice(*(auto_1(p, default_method=_auto_seq)
+                    for p in lst))
+
+def auto_1(arg, default_method=_auto_choice):
+    if isinstance(arg, Parser):
+        return arg
+    elif type(arg) in AUTO_PARSERS:
+        return AUTO_PARSERS[type(arg)](arg)
+    else:
+        return default_method(arg)
+
+def auto(*args):
+    return auto_1(args, default_method=_auto_choice)
+
+# }}}

src/bindings/python/hammer_tests.py

+import unittest
+import hammer as h
+
+class TestTokenParser(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.token("95\xa2")
+    def test_success(self):
+        self.assertEqual(self.parser.parse("95\xa2"), "95\xa2")
+    def test_partial_fails(self):
+        self.assertEqual(self.parser.parse("95"), None)
+
+class TestChParser(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser_int = h.ch(0xa2)
+        cls.parser_chr = h.ch("\xa2")
+    def test_success(self):
+        self.assertEqual(self.parser_int.parse("\xa2"), 0xa2)
+        self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2")
+    def test_failure(self):
+        self.assertEqual(self.parser_int.parse("\xa3"), None)
+        self.assertEqual(self.parser_chr.parse("\xa3"), None)
+
+class TestChRange(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.ch_range("a", "c")
+### this segfaults
+#    def test_success(self):
+#        self.assertEqual(self.parser.parse("b"), "b")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("d"), None)
+
+class TestInt64(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.int64()
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None)
+
+class TestInt32(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.int32()
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000)
+        self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("\xff\xfe\x00"), None)
+        self.assertEqual(self.parser.parse("\x00\x02\x00"), None)
+
+class TestInt16(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.int16()
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\xfe\x00"), -0x200)
+        self.assertEqual(self.parser.parse("\x02\x00"), 0x200)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("\xfe"), None)
+        self.assertEqual(self.parser.parse("\x02"), None)
+
+class TestInt8(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.int8()
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\x88"), -0x78)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse(""), None)
+
+class TestUint64(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.uint64()
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None)
+
+class TestUint32(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.uint32()
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("\x00\x02\x00"), None)
+
+class TestUint16(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.uint16()
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\x02\x00"), 0x200)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("\x02"), None)
+
+class TestUint8(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.uint8()
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\x78"), 0x78)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse(""), None)
+        
+class TestIntRange(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.int_range(h.uint8(), 3, 10)
+    def test_success(self):
+        self.assertEqual(self.parser.parse("\x05"), 5)
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("\x0b"), None)
+
+class TestWhitespace(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.whitespace(h.ch("a"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), "a")
+        self.assertEqual(self.parser.parse(" a"), "a")
+        self.assertEqual(self.parser.parse("  a"), "a")
+        self.assertEqual(self.parser.parse("\ta"), "a")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("_a"), None)
+
+class TestWhitespaceEnd(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.whitespace(h.end_p())
+### this segfaults
+#    def test_success(self):
+#        self.assertEqual(self.parser.parse(""), "")
+#        self.assertEqual(self.parser.parse("  "), "")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("  x"), None)
+
+class TestLeft(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.left(h.ch("a"), h.ch(" "))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a "), "a")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a"), None)
+        self.assertEqual(self.parser.parse(" "), None)
+        self.assertEqual(self.parser.parse("ab"), None)
+
+class TestRight(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.right(h.ch(" "), h.ch("a"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse(" a"), "a")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a"), None)
+        self.assertEqual(self.parser.parse(" "), None)
+        self.assertEqual(self.parser.parse("ba"), None)
+
+class TestMiddle(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" "))
+    def test_success(self):
+        self.assertEqual(self.parser.parse(" a "), "a")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a"), None)
+        self.assertEqual(self.parser.parse(" "), None)
+        self.assertEqual(self.parser.parse(" a"), None)
+        self.assertEqual(self.parser.parse("a "), None)
+        self.assertEqual(self.parser.parse(" b "), None)
+        self.assertEqual(self.parser.parse("ba "), None)
+        self.assertEqual(self.parser.parse(" ab"), None)
+
+class TestAction(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")), h.choice(h.ch("b"), h.ch("B"))), lambda x: [y.upper() for y in x])
+### fails with "corrupted double-linked list"
+#    def test_success(self):
+#        self.assertEqual(self.parser.parse("ab"), ["A", "B"])
+#        self.assertEqual(self.parser.parse("AB"), ["A", "B"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("XX"), None)
+
+class TestIn(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.in_("abc")
+    def test_success(self):
+        self.assertEqual(self.parser.parse("b"), "b")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("d"), None)
+
+class TestNotIn(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.not_in("abc")
+    def test_success(self):
+        self.assertEqual(self.parser.parse("d"), "d")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a"), None)
+
+class TestEndP(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.end_p())
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), ["a"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("aa"), None)
+
+class TestNothingP(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.nothing_p()
+    def test_success(self):
+        pass
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a"), None)
+
+class TestSequence(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.ch("b"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("ab"), ["a", "b"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a"), None)
+        self.assertEqual(self.parser.parse("b"), None)
+
+class TestSequenceWhitespace(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b")))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("ab"), ["a", "b"])
+        self.assertEqual(self.parser.parse("a b"), ["a", "b"])
+        self.assertEqual(self.parser.parse("a  b"), ["a", "b"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a  c"), None)
+
+class TestChoice(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.choice(h.ch("a"), h.ch("b"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), "a")
+        self.assertEqual(self.parser.parse("b"), "b")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("c"), None)
+
+class TestButNot(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.butnot(h.ch("a"), h.token("ab"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), "a")
+        self.assertEqual(self.parser.parse("aa"), "a")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("ab"), None)
+
+### fails with malloc() memory corruption
+#class TestButNotRange(unittest.TestCase):
+#    @classmethod
+#    def setUpClass(cls):
+#        cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6"))
+#    def test_success(self):
+#        self.assertEqual(self.parser.parse("4"), "4")
+### this segfaults
+#    def test_failure(self):
+#        self.assertEqual(self.parser.parse("6"), None)
+
+class TestDifference(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.difference(h.token("ab"), h.ch("a"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("ab"), "ab")
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a"), None)
+
+#class TestXor(unittest.TestCase):
+#    @classmethod
+#    def setUpClass(cls):
+#        cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9"))
+### this segfaults
+#    def test_success(self):
+#        self.assertEqual(self.parser.parse("0"), "0")
+#        self.assertEqual(self.parser.parse("9"), "9")
+### fails with "malloc(): smallbin double linked list corrupted"
+#    def test_failure(self):
+#        self.assertEqual(self.parser.parse("5"), None)
+#        self.assertEqual(self.parser.parse("a"), None)
+
+class TestMany(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.many(h.choice(h.ch("a"), h.ch("b")))
+    def test_success(self):
+        self.assertEqual(self.parser.parse(""), [])
+        self.assertEqual(self.parser.parse("a"), ["a"])
+        self.assertEqual(self.parser.parse("b"), ["b"])
+        self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"])
+    def test_failure(self):
+        pass
+
+class TestMany1(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b")))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), ["a"])
+        self.assertEqual(self.parser.parse("b"), ["b"])
+        self.assertEqual(self.parser.parse("aabbaba"), ["a", "a", "b", "b", "a", "b", "a"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse(""), None)
+        self.assertEqual(self.parser.parse("daabbabadef"), None)
+
+class TestRepeatN(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2)
+    def test_success(self):
+        self.assertEqual(self.parser.parse("abdef"), ["a", "b"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("adef"), None)
+        self.assertEqual(self.parser.parse("dabdef"), None)
+
+class TestOptional(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("abd"), ["a", "b", "d"])
+        self.assertEqual(self.parser.parse("acd"), ["a", "c", "d"])
+        self.assertEqual(self.parser.parse("ad"), ["a", None, "d"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("aed"), None)
+        self.assertEqual(self.parser.parse("ab"), None)
+        self.assertEqual(self.parser.parse("ac"), None)
+
+class TestIgnore(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("abc"), ["a", "c"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("ac"), None)
+
+class TestSepBy(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(","))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"])
+        self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"])
+        self.assertEqual(self.parser.parse("1,3"), ["1", "3"])
+        self.assertEqual(self.parser.parse("3"), ["3"])
+        self.assertEqual(self.parser.parse(""), [])
+    def test_failure(self):
+        pass
+
+class TestSepBy1(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(","))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("1,2,3"), ["1", "2", "3"])
+        self.assertEqual(self.parser.parse("1,3,2"), ["1", "3", "2"])
+        self.assertEqual(self.parser.parse("1,3"), ["1", "3"])
+        self.assertEqual(self.parser.parse("3"), ["3"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse(""), None)
+
+class TestEpsilonP1(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("ab"), ["a", "b"])
+    def test_failure(self):
+        pass
+
+class TestEpsilonP2(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.epsilon_p(), h.ch("a"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), ["a"])
+    def test_failure(self):
+        pass
+
+class TestEpsilonP3(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.epsilon_p())
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), ["a"])
+    def test_failure(self):
+        pass
+
+# this has a double-free problem
+#class TestAttrBool(unittest.TestCase):
+#    @classmethod
+#    def setUpClass(cls):
+#        cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))), lambda x: x[0] == x[1])
+#    def test_success(self):
+#        self.assertEqual(self.parser.parse("aa"), ["a", "a"])
+#        self.assertEqual(self.parser.parse("bb"), ["b", "b"])
+#    def test_failure(self):
+#        self.assertEqual(self.parser.parse("ab"), None)
+
+class TestAnd1(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("0"), ["0"])
+    def test_failure(self):
+        pass
+
+class TestAnd2(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1"))
+    def test_success(self):
+        pass
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("0"), None)
+
+class TestAnd3(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2")))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("12"), ["1"])
+    def test_failure(self):
+        pass
+
+class TestNot1(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.choice(h.ch("+"), h.token("++")), h.ch("b"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a+b"), ["a", "+", "b"])
+    def test_failure(self):
+        self.assertEqual(self.parser.parse("a++b"), None)
+
+class TestNot2(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))), h.token("++")), h.ch("b"))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a+b"), ["a", ["+"], "b"])
+        self.assertEqual(self.parser.parse("a++b"), ["a", "++", "b"])
+    def test_failure(self):
+        pass
+
+class TestLeftrec(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.indirect()
+        a = h.ch("a")
+        h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), "a")
+        self.assertEqual(self.parser.parse("aa"), ["a", "a"])
+        self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"])
+    def test_failure(self):
+        pass
+
+class TestRightrec(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = h.indirect()
+        a = h.ch("a")
+        h.bind_indirect(cls.parser, h.choice(h.sequence(a, cls.parser), h.epsilon_p()))
+    def test_success(self):
+        self.assertEqual(self.parser.parse("a"), ["a"])
+        self.assertEqual(self.parser.parse("aa"), ["a", ["a"]])
+        self.assertEqual(self.parser.parse("aaa"), ["a", ["a", ["a"]]])
+    def test_failure(self):
+        pass
+
+#class TestAmbiguous(unittest.TestCase):
+#    @classmethod
+#    def setUpClass(cls):
+#        cls.parser = h.indirect()
+#        d = h.ch("d")
+#        p = h.ch("+")
+#        h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d))
+#        # this is supposed to be flattened
+#    def test_success(self):
+#        self.assertEqual(self.parser.parse("d"), ["d"])
+#        self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"])
+#        self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"])
+#    def test_failure(self):
+#        self.assertEqual(self.parser.parse("d+"), None)
+

src/bindings/swig/hammer.i

+%module hammer
+%nodefaultctor;
+
+%include "stdint.i"
+
+#if defined(SWIGPHP)
+%ignore HCountedArray_;
+%typemap(in) uint8_t* {
+
+ }
+%typemap(out) uint8_t* {
+
+ }
+%typemap(in) void*[] {
+
+ }
+%typemap(in) uint8_t {
+
+ }
+%typemap(out) HBytes* {
+
+ }
+%typemap(out) struct HCountedArray_* {
+
+ }
+#else
+  #warning no Hammer typemaps defined
+#endif
+
+ // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind.
+%{
+#include "allocator.h"
+#include "hammer.h"
+#include "internal.h"
+%}
+%include "allocator.h"
+%include "hammer.h"
+

src/hammer.c

@@ -62,6 +62,10 @@ HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t*
   return backends[parser->backend]->parse(mm__, parser, &input_stream);
 }
 
+void h_parse_result_free__m(HAllocator *alloc, HParseResult *result) {
+  h_parse_result_free(result);
+}
+
 void h_parse_result_free(HParseResult *result) {
   if(result == NULL) return;
   h_delete_arena(result->arena);

src/hammer.h

@@ -29,7 +29,9 @@
 #define BIT_LITTLE_ENDIAN 0x0
 #define BYTE_LITTLE_ENDIAN 0x0
 
+#ifndef HAMMER_INTERNAL__NO_STDARG_H
 typedef int bool;
+#endif // HAMMER_INTERNAL__NO_STDARG_H
 
 typedef struct HParseState_ HParseState;
 
@@ -68,8 +70,21 @@ typedef struct HBytes_ {
   size_t len;
 } HBytes;
 
+#ifdef SWIG
+typedef union {
+  HBytes bytes;
+  int64_t sint;
+  uint64_t uint;
+  double dbl;
+  float flt;
+  HCountedArray *seq;
+  void *user;
+} HTokenData;
+#endif
+
 typedef struct HParsedToken_ {
   HTokenType token_type;
+#ifndef SWIG
   union {
     HBytes bytes;
     int64_t sint;
@@ -79,6 +94,9 @@ typedef struct HParsedToken_ {
     HCountedArray *seq; // a sequence of HParsedToken's
     void *user;
   };
+#else
+  HTokenData token_data;
+#endif
   size_t index;
   char bit_offset;
 } HParsedToken;
@@ -126,6 +144,7 @@ typedef struct HCFChoice_ HCFChoice;
 typedef struct HRVMProg_ HRVMProg;
 typedef struct HParserVtable_ HParserVtable;
 
+// TODO: Make this internal
 typedef struct HParser_ {
   const HParserVtable *vtable;
   HParserBackend backend;
@@ -141,12 +160,23 @@ typedef struct HParserTestcase_ {
   char* output_unambiguous;
 } HParserTestcase;
 
+#ifdef SWIG
+typedef union {
+  const char* actual_results;
+  size_t parse_time;
+} HResultTiming;
+#endif
+
 typedef struct HCaseResult_ {
   bool success;
+#ifndef SWIG
   union {
     const char* actual_results; // on failure, filled in with the results of h_write_result_unamb
     size_t parse_time; // on success, filled in with time for a single parse, in nsec
   };
+#else
+  HResultTiming timestamp;
+#endif
 } HCaseResult;
 
 typedef struct HBackendResults_ {
@@ -176,7 +206,7 @@ typedef struct HBenchmarkResults_ {
   rtype_t name(__VA_ARGS__) attr;					\
   rtype_t name##__m(HAllocator* mm__, __VA_ARGS__) attr
 
-#ifndef HAMMER_INTERNAL__NO_STDARG_H
+#ifndef SWIG
 #define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...)			\
   rtype_t name(__VA_ARGS__, ...);					\
   rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...);		\
@@ -194,17 +224,17 @@ typedef struct HBenchmarkResults_ {
   rtype_t name##__a(void *args[]);					\
   rtype_t name##__ma(HAllocator *mm__, void *args[])
 #else
-#define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...)			\
-  rtype_t name(__VA_ARGS__, ...);					\
-  rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...);		\
-  rtype_t name##__a(void *args[]);					\
+#define HAMMER_FN_DECL_VARARGS(rtype_t, name, params...)  \
+  rtype_t name(params, ...);				  \
+  rtype_t name##__m(HAllocator* mm__, params, ...);    	  \
+  rtype_t name##__a(void *args[]);			 \
   rtype_t name##__ma(HAllocator *mm__, void *args[])
 
 // Note: this drops the attributes on the floor for the __v versions
-#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, ...)		\
-  rtype_t name(__VA_ARGS__, ...) attr;					\
-  rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...) attr;		\
-  rtype_t name##__a(void *args[]);					\
+#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, params...)		\
+  rtype_t name(params, ...);				\
+  rtype_t name##__m(HAllocator* mm__, params, ...);       	\
+  rtype_t name##__a(void *args[]);				\
   rtype_t name##__ma(HAllocator *mm__, void *args[])
 #endif // HAMMER_INTERNAL__NO_STDARG_H
 // }}}
@@ -586,7 +616,7 @@ char* h_write_result_unamb(const HParsedToken* tok);
  * Format token to the given output stream. Indent starting at
  * [indent] spaces, with [delta] spaces between levels.
  */
-HAMMER_FN_DECL(void, h_pprint, FILE* stream, const HParsedToken* tok, int indent, int delta);
+void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);
 
 /**
  * Build parse tables for the given parser backend. See the
@@ -630,7 +660,7 @@ HParsedToken *h_act_ignore(const HParseResult *p);
 // {{{ Benchmark functions
 HAMMER_FN_DECL(HBenchmarkResults *, h_benchmark, HParser* parser, HParserTestcase* testcases);
 void h_benchmark_report(FILE* stream, HBenchmarkResults* results);
-void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results);
+//void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results);
 // }}}
 
 // {{{ Token type registry

src/parsers/indirect.c

@@ -21,6 +21,10 @@ static const HParserVtable indirect_vt = {
   .compile_to_rvm = h_not_regular,
 };
 
+void h_bind_indirect__m(HAllocator *mm__, HParser* indirect, const HParser* inner) {
+  h_bind_indirect(indirect, inner);
+}
+
 void h_bind_indirect(HParser* indirect, const HParser* inner) {
   assert_message(indirect->vtable == &indirect_vt, "You can only bind an indirect parser");
   indirect->env = (void*)inner;

src/parsers/token.c

@@ -79,6 +79,8 @@ HParser* h_token(const uint8_t *str, const size_t len) {
 }
 HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len) { 
   HToken *t = h_new(HToken, 1);
-  t->str = (uint8_t*)str, t->len = len;
+  uint8_t *str_cpy = h_new(uint8_t, len);
+  memcpy(str_cpy, str, len);
+  t->str = str_cpy, t->len = len;
   return h_new_parser(mm__, &token_vt, t);
 }