.gitignore

@@ -23,3 +23,6 @@ build/
 .sconsign.dblite
 *.os
 *.pyc
+*.ll
+libhammer.pc
+TestResult.xml

.travis.yml

-sudo: required
 dist: trusty
+sudo: required
 language: c
 compiler:
   - gcc
   - clang
 env:
-  - BINDINGS=none
+  global:
+    - LLVM_CONFIG=llvm-config-3.8
+  matrix:
+    - BINDINGS=none LLVM_BACKEND=0
+    - BINDINGS=none LLVM_BACKEND=1
 matrix:
   include:
     - compiler: gcc
       language: ruby
-      rvm: ruby-1.9.3-p551
-      env: BINDINGS=ruby
-    - compiler: clang
-      language: ruby
-      rvm: ruby-1.9.3-p551
-      env: BINDINGS=ruby CC=clang
+      rvm: ruby-2.2.5
+      env: BINDINGS=ruby LLVM_BACKEND=0
     - compiler: gcc
       language: ruby
-      rvm: ruby-2.0.0-p647
-      env: BINDINGS=ruby
+      rvm: ruby-2.2.5
+      env: BINDINGS=ruby LLVM_BACKEND=1
     - compiler: clang
       language: ruby
-      rvm: ruby-2.0.0-p647
-      env: BINDINGS=ruby CC=clang
-    - compiler: gcc
-      language: ruby
-      rvm: ruby-2.1.7
-      env: BINDINGS=ruby
+      rvm: ruby-2.2.5
+      env: BINDINGS=ruby CC=clang LLVM_BACKEND=0
     - compiler: clang
       language: ruby
-      rvm: ruby-2.1.7
-      env: BINDINGS=ruby CC=clang
+      rvm: ruby-2.2.5
+      env: BINDINGS=ruby CC=clang LLVM_BACKEND=1
+    - compiler: gcc
+      language: ruby
+      rvm: ruby-2.3.1
+      env: BINDINGS=ruby LLVM_BACKEND=0
     - compiler: gcc
       language: ruby
-      rvm: ruby-2.2.3
-      env: BINDINGS=ruby
+      rvm: ruby-2.3.1
+      env: BINDINGS=ruby LLVM_BACKEND=1
+    - compiler: clang
+      language: ruby
+      rvm: ruby-2.3.1
+      env: BINDINGS=ruby CC=clang LLVM_BACKEND=0
     - compiler: clang
       language: ruby
-      rvm: ruby-2.2.3
-      env: BINDINGS=ruby CC=clang
+      rvm: ruby-2.3.1
+      env: BINDINGS=ruby CC=clang LLVM_BACKEND=1
     - compiler: gcc
       language: python
       python: "2.7.10"
-      env: BINDINGS=python
+      env: BINDINGS=python LLVM_BACKEND=0
+    - compiler: gcc
+      language: python
+      python: "2.7.10"
+      env: BINDINGS=python LLVM_BACKEND=1
     - compiler: clang
       language: python
       python: "2.7.10"
-      env: BINDINGS=python CC=clang
+      env: BINDINGS=python CC=clang LLVM_BACKEND=0
+    - compiler: clang
+      language: python
+      python: "2.7.10"
+      env: BINDINGS=python CC=clang LLVM_BACKEND=1
     - compiler: gcc
       language: perl
       perl: "5.18"
-      env: BINDINGS=perl
+      env: BINDINGS=perl LLVM_BACKEND=0
+    - compiler: gcc
+      language: perl
+      perl: "5.18"
+      env: BINDINGS=perl LLVM_BACKEND=1
     - compiler: clang
       language: perl
       perl: "5.18"
-      env: BINDINGS=perl CC=clang
+      env: BINDINGS=perl CC=clang LLVM_BACKEND=0
+    - compiler: clang
+      language: perl
+      perl: "5.18"
+      env: BINDINGS=perl CC=clang LLVM_BACKEND=1
+    - compiler: gcc
+      language: perl
+      perl: "5.14"
+      env: BINDINGS=perl LLVM_BACKEND=0
     - compiler: gcc
       language: perl
       perl: "5.14"
-      env: BINDINGS=perl
+      env: BINDINGS=perl LLVM_BACKEND=1
+    - compiler: clang
+      language: perl
+      perl: "5.14"
+      env: BINDINGS=perl CC=clang LLVM_BACKEND=0
     - compiler: clang
       language: perl
       perl: "5.14"
-      env: BINDINGS=perl CC=clang
+      env: BINDINGS=perl CC=clang LLVM_BACKEND=1
+    - compiler: gcc
+      language: perl
+      perl: "5.10"
+      env: BINDINGS=perl LLVM_BACKEND=0
     - compiler: gcc
       language: perl
       perl: "5.10"
-      env: BINDINGS=perl
+      env: BINDINGS=perl LLVM_BACKEND=1
+    - compiler: clang
+      language: perl
+      perl: "5.10"
+      env: BINDINGS=perl CC=clang LLVM_BACKEND=0
     - compiler: clang
       language: perl
       perl: "5.10"
-      env: BINDINGS=perl CC=clang
+      env: BINDINGS=perl CC=clang LLVM_BACKEND=1
     - compiler: gcc
       language: php
       php: "5.5"
-      env: BINDINGS=php
+      env: BINDINGS=php LLVM_BACKEND=0
+    - compiler: gcc
+      language: php
+      php: "5.5"
+      env: BINDINGS=php LLVM_BACKEND=1
     - compiler: clang
       language: php
       php: "5.5"
-      env: BINDINGS=php CC=clang
+      env: BINDINGS=php CC=clang LLVM_BACKEND=0
+    - compiler: clang
+      language: php
+      php: "5.5"
+      env: BINDINGS=php CC=clang LLVM_BACKEND=1
     - compiler: gcc
       language: php
       php: "5.4"
-      env: BINDINGS=php
+      env: BINDINGS=php LLVM_BACKEND=0
+    - compiler: gcc
+      language: php
+      php: "5.4"
+      env: BINDINGS=php LLVM_BACKEND=1
     - compiler: clang
       language: php
       php: "5.4"
-      env: BINDINGS=php CC=clang
+      env: BINDINGS=php CC=clang LLVM_BACKEND=0
+    - compiler: clang
+      language: php
+      php: "5.4"
+      env: BINDINGS=php CC=clang LLVM_BACKEND=1
+    - compiler: gcc
+      language: dotnet
+      env: BINDINGS=dotnet LLVM_BACKEND=0
     - compiler: gcc
       language: dotnet
-      env: BINDINGS=dotnet
+      env: BINDINGS=dotnet LLVM_BACKEND=1
     - compiler: clang
       language: dotnet
-      env: BINDINGS=dotnet CC=clang
+      env: BINDINGS=dotnet CC=clang LLVM_BACKEND=0
+    - compiler: clang
+      language: dotnet
+      env: BINDINGS=dotnet CC=clang LLVM_BACKEND=1
+    - compiler: gcc
+      language: cpp
+      env: BINDINGS=cpp LLVM_BACKEND=0
     - compiler: gcc
       language: cpp
-      env: BINDINGS=cpp
+      env: BINDINGS=cpp LLVM_BACKEND=1
+    - compiler: clang
+      language: cpp
+      env: BINDINGS=cpp CC=clang LLVM_BACKEND=0
     - compiler: clang
       language: cpp
-      env: BINDINGS=cpp CC=clang
+      env: BINDINGS=cpp CC=clang LLVM_BACKEND=1
 before_install:
   - sudo apt-get update -qq
   - sudo apt-get install -y lcov
   - gem install coveralls-lcov
-  - if [ "$BINDINGS" != "none" ]; then sudo sh -c 'echo "deb http://archive.ubuntu.com/ubuntu trusty-backports main restricted universe multiverse" >> /etc/apt/sources.list'; sudo apt-get update -qq; sudo apt-get install -yqq swig3.0/trusty-backports; fi
-  - if [ "$BINDINGS" == "python" ]; then sudo apt-get install -yqq python-dev; fi
-  - if [ "$BINDINGS" == "dotnet" ]; then sudo add-apt-repository ppa:directhex/monoxide -y; sudo apt-get update -qq; sudo apt-get install -yqq mono-devel mono-mcs nunit nunit-console; mozroots --import --sync; fi
+  - if [ "$CC" == "gcc" ]; then sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y; sudo apt-get update -qq; sudo apt-get install gcc-5; fi
+  - sudo apt-get install llvm-3.8 llvm-3.8-dev clang-3.8
+  - if [ "$BINDINGS" != "none" ]; then sudo apt-get install -qq swig; sudo apt-get install -t trusty-backports swig3.0; fi
+  - if [ "$BINDINGS" == "python" ]; then sudo apt-get install -qq python-dev; fi
+  - if [ "$BINDINGS" == "dotnet" ]; then sudo apt-get install -qq mono-devel mono-mcs nunit nunit-console; fi
 install: true
 before_script:
   - if [ "$BINDINGS" == "php" ]; then phpenv config-add src/bindings/php/hammer.ini; fi
 script:
-  - if [ "$BINDINGS" == "none" ]; then scons test --variant=debug --coverage; else scons bindings=$BINDINGS test; fi
+  - if [ "$BINDINGS" == "none" ]; then if [ "$LLVM_BACKEND" == "1" ]; then LLVM_FLAG="--enable-llvm-backend"; else LLVM_FLAG="--disable-llvm-backend"; fi; scons test --variant=debug $LLVM_FLAG --coverage; else scons $LLVM_FLAG bindings=$BINDINGS test; fi
 after_success:
   - if [ "$BINDINGS" == "none" ]; then if [ "$CC" == "clang" ]; then llvm-cov gcov -o coverage.info build/debug/src/test_suite.gcda; else lcov --capture --directory build/debug/src --output-file coverage.info; fi; fi
   - coveralls-lcov coverage.info

SConstruct

@@ -3,11 +3,16 @@ import os
 import os.path
 import platform
 import sys
+from distutils.version import LooseVersion
+import re
+import subprocess
 
 
 vars = Variables(None, ARGUMENTS)
 vars.Add(PathVariable('DESTDIR', "Root directory to install in (useful for packaging scripts)", None, PathVariable.PathIsDirCreate))
 vars.Add(PathVariable('prefix', "Where to install in the FHS", "/usr/local", PathVariable.PathAccept))
+vars.Add(PathVariable('libdir', "Where to install libraries", None, PathVariable.PathAccept))
+vars.Add(PathVariable('includedir', "Where to install headers", None, PathVariable.PathAccept))
 vars.Add(ListVariable('bindings', 'Language bindings to build', 'none', ['cpp', 'dotnet', 'perl', 'php', 'python', 'ruby']))
 
 tools = ['default', 'scanreplace']
@@ -41,13 +46,20 @@ if 'DESTDIR' in env:
         print >>sys.stderr, "--!!-- you want; files will be installed in"
         print >>sys.stderr, "--!!--    %s" % (calcInstallPath("$prefix"),)
 
-
-env['libpath'] = calcInstallPath("$prefix", "lib")
-env['incpath'] = calcInstallPath("$prefix", "include", "hammer")
-env['parsersincpath'] = calcInstallPath("$prefix", "include", "hammer", "parsers")
-env['backendsincpath'] = calcInstallPath("$prefix", "include", "hammer", "backends")
-env['pkgconfigpath'] = calcInstallPath("$prefix", "lib", "pkgconfig")
-env.ScanReplace('libhammer.pc.in')
+if 'includedir' in env:
+    env['incpath'] = calcInstallPath("$includedir", "hammer")
+else:
+    env['includedir'] = os.path.abspath(os.path.join(*map(env.subst, ["$prefix", "include"])))
+    env['incpath'] = calcInstallPath("$prefix", "include", "hammer")
+if 'libdir' in env:
+    env['libpath'] = calcInstallPath("$libdir")
+    env['pkgconfigpath'] = calcInstallPath("$libdir", "pkgconfig")
+else:
+    env['libpath'] = calcInstallPath("$prefix", "lib")
+    env['pkgconfigpath'] = calcInstallPath("$prefix", "lib", "pkgconfig")
+    env['libdir'] = os.path.abspath(os.path.join(*map(env.subst, ["$prefix", "lib"])))
+env['parsersincpath'] = calcInstallPath("$includedir", "hammer", "parsers")
+env['backendsincpath'] = calcInstallPath("$includedir", "hammer", "backends")
 
 env.MergeFlags("-std=gnu11 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-attributes -Wno-unused-variable")
 
@@ -78,9 +90,19 @@ AddOption("--in-place",
           action="store_true",
           help="Build in-place, rather than in the build/<variant> tree")
 
+AddOption("--disable-llvm-backend",
+          dest="use_llvm",
+          default=False,
+          action="store_false",
+          help="Disable the LLVM backend (and don't require LLVM library dependencies)")
+AddOption("--enable-llvm-backend",
+          dest="use_llvm",
+          default=False,
+          action="store_true",
+          help="Enable the LLVM backend (and require LLVM library dependencies)")
 
 dbg = env.Clone(VARIANT='debug')
-dbg.Append(CCFLAGS=['-g'])
+dbg.MergeFlags("-g -O0")
 
 opt = env.Clone(VARIANT='opt')
 opt.Append(CCFLAGS=["-O3"])
@@ -93,6 +115,12 @@ else:
 env["CC"] = os.getenv("CC") or env["CC"]
 env["CXX"] = os.getenv("CXX") or env["CXX"]
 
+if GetOption("use_llvm"):
+    # Overridable default path to llvm-config
+    env['LLVM_CONFIG'] = "llvm-config"
+    env["LLVM_CONFIG"] = os.getenv("LLVM_CONFIG") or env["LLVM_CONFIG"]
+    env.MergeFlags("-DHAMMER_LLVM_BACKEND")
+
 if GetOption("coverage"):
     env.Append(CFLAGS=["--coverage"],
                CXXFLAGS=["--coverage"],
@@ -100,7 +128,11 @@ if GetOption("coverage"):
     if env["CC"] == "gcc":
         env.Append(LIBS=['gcov'])
     else:
-        env.ParseConfig('llvm-config --ldflags')
+        # XXX Why do need this with --coverage when we're doing it anyway?
+        if GetOption("use_llvm"):
+            env.ParseConfig('%s --cflags --ldflags --libs core executionengine mcjit analysis x86codegen x86info' % \
+                            env["LLVM_CONFIG"])
+
 
 if os.getenv("CC") == "clang" or env['PLATFORM'] == 'darwin':
     env.Replace(CC="clang",
@@ -111,6 +143,126 @@ env["ENV"].update(x for x in os.environ.items() if x[0].startswith("CCC_"))
 #rootpath = env['ROOTPATH'] = os.path.abspath('.')
 #env.Append(CPPPATH=os.path.join('#', "hammer"))
 
+if GetOption("use_llvm"):
+# Set up LLVM config stuff to export
+
+# some llvm versions are old and will not work; some require --system-libs
+# with llvm-config, and some will break if given it
+    llvm_config_version = subprocess.Popen('%s --version' % env["LLVM_CONFIG"], \
+                                           shell=True, \
+                                           stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate()
+    if LooseVersion(llvm_config_version[0]) < LooseVersion("3.6"):
+        print "This LLVM version %s is too old" % llvm_config_version[0].strip()
+        Exit(1)
+
+    if LooseVersion(llvm_config_version[0]) < LooseVersion("3.9") and \
+        LooseVersion(llvm_config_version[0]) >= LooseVersion("3.5"):
+        llvm_system_libs_flag = "--system-libs"
+    else:
+        llvm_system_libs_flag = ""
+
+    # Only keep one copy of this
+    llvm_required_components = "core executionengine mcjit analysis x86codegen x86info"
+    # Stubbing this out so we can implement static-only mode if needed later
+    llvm_use_shared = True
+    # Can we ask for shared/static from llvm-config?
+    if LooseVersion(llvm_config_version[0]) < LooseVersion("3.9"):
+        # Nope
+        llvm_linkage_type_flag = ""
+        llvm_use_computed_shared_lib_name = True
+    else:
+        # Woo, they finally fixed the dumb
+        llvm_use_computed_shared_lib_name = False
+        if llvm_use_shared:
+            llvm_linkage_type_flag = "--link-shared"
+        else:
+            llvm_linkage_type_flag = "--link-static"
+
+    if llvm_use_computed_shared_lib_name:
+        # Okay, pull out the major and minor version numbers (barf barf)
+        p = re.compile("^(\d+)\.(\d+).*$")
+        m = p.match(llvm_config_version[0])
+        if m:
+            llvm_computed_shared_lib_name = "LLVM-%d.%d" % ((int)(m.group(1)), (int)(m.group(2)))
+        else:
+            print "Couldn't compute shared library name from LLVM version '%s', but needed to" % \
+                llvm_config_version[0]
+            Exit(1)
+    else:
+        # We won't be needing it
+        llvm_computed_shared_lib_name = None
+
+    # llvm-config 'helpfully' supplies -g and -O flags; educate it with this
+    # custom ParseConfig function arg; make it a class with a method so we can
+    # pass it around with scons export/import
+
+    class LLVMConfigSanitizer:
+        def sanitize(self, env, cmd, unique=1):
+            # cmd is output from llvm-config
+            flags = cmd.split()
+            # match -g or -O flags
+            p = re.compile("^-[gO].*$")
+            filtered_flags = [flag for flag in flags if not p.match(flag)]
+            filtered_cmd = ' '.join(filtered_flags)
+            # print "llvm_config_sanitize: \"%s\" => \"%s\"" % (cmd, filtered_cmd)
+            env.MergeFlags(filtered_cmd, unique)
+    llvm_config_sanitizer = LLVMConfigSanitizer()
+
+    # LLVM defines, which the python bindings need
+    try:
+        llvm_config_cflags = subprocess.Popen('%s --cflags' % env["LLVM_CONFIG"], \
+                                              shell=True, \
+                                              stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate()
+        flags = llvm_config_cflags[0].split()
+        # get just the -D ones
+        p = re.compile("^-D(.*)$")
+        llvm_defines = [p.match(flag).group(1) for flag in flags if p.match(flag)]
+    except:
+        print "%s failed. Make sure you have LLVM and clang installed." % env["LLVM_CONFIG"]
+        Exit(1)
+
+    # Get the llvm includedir, which the python bindings need
+    try:
+        llvm_config_includes = subprocess.Popen('%s --includedir' % env["LLVM_CONFIG"], \
+                                                shell=True, \
+                                                stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate()
+        llvm_includes = llvm_config_includes[0].splitlines()
+    except:
+        print "%s failed. Make sure you have LLVM and clang installed." % env["LLVM_CONFIG"]
+        Exit(1)
+
+    # This goes here so we already know all the LLVM crap
+    # Make a fresh environment to parse the config into, to read out just LLVM stuff
+    llvm_dummy_env = Environment()
+    # Get LLVM stuff into LIBS/LDFLAGS
+    llvm_dummy_env.ParseConfig('%s --ldflags %s %s %s' % \
+                               (env["LLVM_CONFIG"], llvm_system_libs_flag, llvm_linkage_type_flag, \
+                                llvm_required_components), \
+                               function=llvm_config_sanitizer.sanitize)
+    # Get the right -l lines in
+    if llvm_use_shared:
+        if llvm_use_computed_shared_lib_name:
+            llvm_dummy_env.Append(LIBS=[llvm_computed_shared_lib_name, ])
+        else:
+            llvm_dummy_env.ParseConfig('%s %s --libs %s' % \
+                                       (env["LLVM_CONFIG"], llvm_linkage_type_flag, llvm_required_components), \
+                                       function=llvm_config_sanitizer.sanitize)
+    llvm_dummy_env.Append(LIBS=['stdc++', ], )
+#endif GetOption("use_llvm")
+
+# The .pc.in file has substs for llvm_lib_flags and llvm_libdir_flags, so if
+# we aren't using LLVM, set them to the empty string
+if GetOption("use_llvm"):
+    env['llvm_libdir_flags'] = llvm_dummy_env.subst('$_LIBDIRFLAGS')
+    env['llvm_lib_flags'] = llvm_dummy_env.subst('$_LIBFLAGS')
+else:
+    env['llvm_libdir_flags'] = ""
+    env['llvm_lib_flags'] = ""
+
+pkgconfig = env.ScanReplace('libhammer.pc.in')
+Default(pkgconfig)
+env.Install("$pkgconfigpath", pkgconfig)
+
 testruns = []
 
 targets = ["$libpath",
@@ -122,6 +274,18 @@ targets = ["$libpath",
 Export('env')
 Export('testruns')
 Export('targets')
+# LLVM-related flags
+if GetOption("use_llvm"):
+    Export('llvm_computed_shared_lib_name')
+    Export('llvm_config_sanitizer')
+    Export('llvm_config_version')
+    Export('llvm_defines')
+    Export('llvm_includes')
+    Export('llvm_linkage_type_flag')
+    Export('llvm_required_components')
+    Export('llvm_system_libs_flag')
+    Export('llvm_use_computed_shared_lib_name')
+    Export('llvm_use_shared')
 
 if not GetOption("in_place"):
     env['BUILD_BASE'] = 'build/$VARIANT'

libhammer.pc.in

 prefix=${prefix}
 exec_prefix=${prefix}
 includedir=${prefix}/include
-libdir=${exec_prefix}/lib
+libdir=${libdir}
 
 Name: libhammer
 Description: The Hammer parsing library
 Version: 0.9.0
 Cflags: -I${includedir}
 Libs: -L${libdir} -lhammer
+Libs.private: ${llvm_libdir_flags} ${llvm_lib_flags}

src/SConscript

 # -*- python -*-
 import os.path
+from distutils.version import LooseVersion
+import re
+import subprocess
+
 Import('env testruns')
+# LLVM-related flags
+if GetOption("use_llvm"):
+    Import('llvm_computed_shared_lib_name')
+    Import('llvm_config_sanitizer')
+    Import('llvm_config_version')
+    Import('llvm_linkage_type_flag')
+    Import('llvm_required_components')
+    Import('llvm_system_libs_flag')
+    Import('llvm_use_computed_shared_lib_name')
+    Import('llvm_use_shared')
 
 dist_headers = [
     "hammer.h",
@@ -18,6 +32,7 @@ parsers_headers = [
 backends_headers = [
     "backends/regex.h",
     "backends/contextfree.h"
+    "backends/missing.h"
 ]
 
 parsers = ['parsers/%s.c'%s for s in
@@ -48,10 +63,15 @@ parsers = ['parsers/%s.c'%s for s in
             'unimplemented',
             'whitespace',
             'xor',
-            'value']] 
+            'value']]
 
 backends = ['backends/%s.c' % s for s in
-            ['packrat', 'llk', 'regex', 'glr', 'lalr', 'lr', 'lr0']]
+            ['packrat', 'llk', 'regex', 'glr', 'lalr', 'lr', 'lr0', "missing"]]
+
+# Add LLVM backend if enabled
+if GetOption("use_llvm"):
+    llvm_backend_files = ['llvm.c', 'llvm_charset.c']
+    backends = backends + ['backends/llvm/%s' % s for s in llvm_backend_files]
 
 misc_hammer_parts = [
     'allocator.c',
@@ -76,19 +96,66 @@ ctests = ['t_benchmark.c',
           't_misc.c',
 	  't_regression.c']
 
-libhammer_shared = env.SharedLibrary('hammer', parsers + backends + misc_hammer_parts)
+if GetOption("use_llvm"):
+    env.ParseConfig('%s --cflags --ldflags' % env["LLVM_CONFIG"], function=llvm_config_sanitizer.sanitize)
 libhammer_static = env.StaticLibrary('hammer', parsers + backends + misc_hammer_parts)
+
+# Use a cloned env for the shared library so we can have library dependencies
+shared_env = env.Clone()
+if GetOption("use_llvm"):
+    # Get LLVM stuff into LIBS/LDFLAGS
+    shared_env.ParseConfig('%s --ldflags %s %s %s' % \
+                           (env["LLVM_CONFIG"], llvm_system_libs_flag, \
+                            llvm_linkage_type_flag, llvm_required_components), \
+                           function=llvm_config_sanitizer.sanitize)
+    # Get the right -l lines in
+    if llvm_use_shared:
+        if llvm_use_computed_shared_lib_name:
+            shared_env.Append(LIBS=[llvm_computed_shared_lib_name, ])
+        else:
+            shared_env.ParseConfig('%s %s --libs %s' % \
+                                   (env["LLVM_CONFIG"], llvm_linkage_type_flag, llvm_required_components), \
+                                   function=llvm_config_sanitizer.sanitize)
+    else:
+        # Just grab the statics regardless of version
+        shared_env.ParseConfig('%s %s --libs %s' % \
+                               (env["LLVM_CONFIG"], llvm_linkage_type_flag, llvm_required_components), \
+                               function=llvm_config_sanitizer.sanitize)
+    shared_env.Append(LIBS=['stdc++', ], LIBPATH=['.'])
+
+libhammer_shared = shared_env.SharedLibrary('hammer', parsers + backends + misc_hammer_parts)
+
 Default(libhammer_shared, libhammer_static)
 
 env.Install("$libpath", [libhammer_static, libhammer_shared])
 env.Install("$incpath", dist_headers)
 env.Install("$parsersincpath", parsers_headers)
 env.Install("$backendsincpath", backends_headers)
-env.Install("$pkgconfigpath", "../../../libhammer.pc")
 
 testenv = env.Clone()
-testenv.ParseConfig('pkg-config --cflags --libs glib-2.0')
 testenv.Append(LIBS=['hammer'], LIBPATH=['.'])
+testenv.ParseConfig('pkg-config --cflags --libs glib-2.0')
+if GetOption("use_llvm"):
+    # Get LLVM stuff into LIBS/LDFLAGS
+    testenv.ParseConfig('%s --ldflags %s %s %s' % \
+                        (env["LLVM_CONFIG"], llvm_system_libs_flag, \
+                         llvm_linkage_type_flag, llvm_required_components), \
+                        function=llvm_config_sanitizer.sanitize)
+    # Get the right -l lines in
+    if llvm_use_shared:
+        if llvm_use_computed_shared_lib_name:
+            testenv.Append(LIBS=[llvm_computed_shared_lib_name, ])
+        else:
+            testenv.ParseConfig('%s %s --libs %s' % \
+                                (env["LLVM_CONFIG"], llvm_linkage_type_flag, llvm_required_components), \
+                                function=llvm_config_sanitizer.sanitize)
+    else:
+        # Just grab the statics regardless of version
+        testenv.ParseConfig('%s %s --libs %s' % \
+                            (env["LLVM_CONFIG"], llvm_linkage_type_flag, llvm_required_components), \
+                            function=llvm_config_sanitizer.sanitize)
+    testenv.Append(LIBS=['stdc++'], LIBPATH=['.'])
+
 ctestexec = testenv.Program('test_suite', ctests + ['test_suite.c'], LINKFLAGS="--coverage" if testenv.GetOption("coverage") else None)
 ctest = Alias('testc', [ctestexec], "".join(["env LD_LIBRARY_PATH=", os.path.dirname(ctestexec[0].path), " ", ctestexec[0].path]))
 AlwaysBuild(ctest)

src/backends/llvm/llvm.c

+#ifdef HAMMER_LLVM_BACKEND
+
+#include <llvm-c/Analysis.h>
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpedantic"
+#include <llvm-c/Core.h>
+#pragma GCC diagnostic pop
+#include <llvm-c/ExecutionEngine.h>
+#include "../../internal.h"
+#include "llvm.h"
+
+typedef struct HLLVMParser_ {
+  LLVMModuleRef mod;
+  LLVMValueRef func;
+  LLVMExecutionEngineRef engine;
+  LLVMBuilderRef builder;
+} HLLVMParser;
+
+HParseResult* make_result(HArena *arena, HParsedToken *tok) {
+  HParseResult *ret = h_arena_malloc(arena, sizeof(HParseResult));
+  ret->ast = tok;
+  ret->arena = arena;
+  ret->bit_length = 0; // This way it gets overridden in h_do_parse
+  return ret;
+}
+
+void h_llvm_declare_common(LLVMModuleRef mod) {
+  llvm_inputstream = LLVMStructCreateNamed(LLVMGetGlobalContext(), "struct.HInputStream_");
+  LLVMTypeRef llvm_inputstream_struct_types[] = {
+    LLVMPointerType(LLVMInt8Type(), 0),
+    LLVMInt64Type(),
+    LLVMInt64Type(),
+    LLVMInt64Type(),
+    LLVMInt8Type(),
+    LLVMInt8Type(),
+    LLVMInt8Type(),
+    LLVMInt8Type(),
+    LLVMInt8Type()
+  };
+  LLVMStructSetBody(llvm_inputstream, llvm_inputstream_struct_types, 9, 0);
+  llvm_inputstreamptr = LLVMPointerType(llvm_inputstream, 0);
+  llvm_arena = LLVMStructCreateNamed(LLVMGetGlobalContext(), "struct.HArena_");
+  llvm_arenaptr = LLVMPointerType(llvm_arena, 0);
+  llvm_parsedtoken = LLVMStructCreateNamed(LLVMGetGlobalContext(), "struct.HParsedToken_");
+  LLVMTypeRef llvm_parsedtoken_struct_types[] = {
+    LLVMInt32Type(), // actually an enum value
+    LLVMInt64Type(), // actually this is a union; the largest thing in it is 64 bits
+    LLVMInt64Type(), // FIXME sizeof(size_t) will be 32 bits on 32-bit platforms
+    LLVMInt64Type(), // FIXME ditto
+    LLVMInt8Type()
+  };
+  LLVMStructSetBody(llvm_parsedtoken, llvm_parsedtoken_struct_types, 5, 0);
+  llvm_parsedtokenptr = LLVMPointerType(llvm_parsedtoken, 0);
+  llvm_parseresult = LLVMStructCreateNamed(LLVMGetGlobalContext(), "struct.HParseResult_");
+  LLVMTypeRef llvm_parseresult_struct_types[] = {
+    llvm_parsedtokenptr,
+    LLVMInt64Type(),
+    llvm_arenaptr
+  };
+  LLVMStructSetBody(llvm_parseresult, llvm_parseresult_struct_types, 3, 0);
+  llvm_parseresultptr = LLVMPointerType(llvm_parseresult, 0);
+  LLVMTypeRef readbits_pt[] = {
+    llvm_inputstreamptr,
+    LLVMInt32Type(),
+    LLVMInt8Type()
+  };
+  LLVMTypeRef readbits_ret = LLVMFunctionType(LLVMInt64Type(), readbits_pt, 3, 0);
+  LLVMAddFunction(mod, "h_read_bits", readbits_ret);
+
+  LLVMTypeRef amalloc_pt[] = {
+    llvm_arenaptr,
+    LLVMInt32Type()
+  };
+  LLVMTypeRef amalloc_ret = LLVMFunctionType(LLVMPointerType(LLVMVoidType(), 0), amalloc_pt, 2, 0);
+  LLVMAddFunction(mod, "h_arena_malloc", amalloc_ret);
+
+  LLVMTypeRef makeresult_pt[] = {
+    llvm_arenaptr,
+    llvm_parsedtokenptr
+  };
+  LLVMTypeRef makeresult_ret = LLVMFunctionType(llvm_parseresultptr, makeresult_pt, 2, 0);
+  LLVMAddFunction(mod, "make_result", makeresult_ret);
+}
+
+int h_llvm_compile(HAllocator* mm__, HParser* parser, const void* params) {
+  // Boilerplate to set up a translation unit, aka a module.
+  const char* name = params ? (const char*)params : "parse";
+  LLVMModuleRef mod = LLVMModuleCreateWithName(name);
+  h_llvm_declare_common(mod);
+  // Boilerplate to set up the parser function to add to the module. It takes an HInputStream* and
+  // returns an HParseResult.
+  LLVMTypeRef param_types[] = {
+    llvm_inputstreamptr,
+    llvm_arenaptr
+  };
+  LLVMTypeRef ret_type = LLVMFunctionType(llvm_parseresultptr, param_types, 2, 0);
+  LLVMValueRef parse_func = LLVMAddFunction(mod, name, ret_type);
+  // Parse function is now declared; time to define it
+  LLVMBuilderRef builder = LLVMCreateBuilder();
+  // Translate the contents of the children of `parser` into their LLVM instruction equivalents
+  if (parser->vtable->llvm(mm__, builder, parse_func, mod, parser->env)) {
+    // But first, verification
+    char *error = NULL;
+    LLVMVerifyModule(mod, LLVMAbortProcessAction, &error);
+    LLVMDisposeMessage(error);
+    error = NULL;
+    // OK, link that sonofabitch
+    LLVMLinkInMCJIT();
+    LLVMInitializeNativeTarget();
+    LLVMInitializeNativeAsmPrinter();
+    LLVMExecutionEngineRef engine = NULL;
+    LLVMCreateExecutionEngineForModule(&engine, mod, &error);
+    if (error) {
+      fprintf(stderr, "error: %s\n", error);
+      LLVMDisposeMessage(error);
+      return -1;
+    }
+    char* dump = LLVMPrintModuleToString(mod);
+    fprintf(stderr, "\n\n%s\n\n", dump);
+    // Package up the pointers that comprise the module and stash it in the original HParser
+    HLLVMParser *llvm_parser = h_new(HLLVMParser, 1);
+    llvm_parser->mod = mod;
+    llvm_parser->func = parse_func;
+    llvm_parser->engine = engine;
+    llvm_parser->builder = builder;
+    parser->backend_data = llvm_parser;
+    return 0;
+  } else {
+    return -1;
+  }
+}
+
+void h_llvm_free(HParser *parser) {
+  HLLVMParser *llvm_parser = parser->backend_data;
+  LLVMModuleRef mod_out;
+  char *err_out;
+
+  llvm_parser->func = NULL;
+  LLVMRemoveModule(llvm_parser->engine, llvm_parser->mod, &mod_out, &err_out);
+  LLVMDisposeExecutionEngine(llvm_parser->engine);
+  llvm_parser->engine = NULL;
+
+  LLVMDisposeBuilder(llvm_parser->builder);
+  llvm_parser->builder = NULL;
+
+  LLVMDisposeModule(llvm_parser->mod);
+  llvm_parser->mod = NULL;
+}
+
+/*
+ * Construct LLVM IR to allocate a token of type TT_SINT or TT_UINT
+ *
+ * Parameters:
+ *  - mod [in]: an LLVMModuleRef
+ *  - builder [in]: an LLVMBuilderRef, positioned appropriately
+ *  - stream [in]: a value ref to an llvm_inputstreamptr, for the input stream
+ *  - arena [in]: a value ref to an llvm_arenaptr to be used for the malloc
+ *  - r [in]: a value ref to the value to be used to this token
+ *  - mr_out [out]: the return value from make_result()
+ *
+ * TODO actually support TT_SINT, inputs other than 8 bit
+ */
+
+void h_llvm_make_tt_suint(LLVMModuleRef mod, LLVMBuilderRef builder,
+                          LLVMValueRef stream, LLVMValueRef arena,
+                          LLVMValueRef r, LLVMValueRef *mr_out) {
+  /* Set up call to h_arena_malloc() for a new HParsedToken */
+  LLVMValueRef tok_size = LLVMConstInt(LLVMInt32Type(), sizeof(HParsedToken), 0);
+  LLVMValueRef amalloc_args[] = { arena, tok_size };
+  /* %h_arena_malloc = call void* @h_arena_malloc(%struct.HArena_.1* %1, i32 48) */
+  LLVMValueRef amalloc = LLVMBuildCall(builder, LLVMGetNamedFunction(mod, "h_arena_malloc"),
+      amalloc_args, 2, "h_arena_malloc");
+  /* %tok = bitcast void* %h_arena_malloc to %struct.HParsedToken_.2* */
+  LLVMValueRef tok = LLVMBuildBitCast(builder, amalloc, llvm_parsedtokenptr, "tok");
+
+  /*
+   * tok->token_type = TT_UINT;
+   *
+   * %token_type = getelementptr inbounds %struct.HParsedToken_.2, %struct.HParsedToken_.2* %3, i32 0, i32 0
+   *
+   * TODO if we handle TT_SINT too, adjust here and the zero-ext below
+   */
+  LLVMValueRef toktype = LLVMBuildStructGEP(builder, tok, 0, "token_type");
+  /* store i32 8, i32* %token_type */
+  LLVMBuildStore(builder, LLVMConstInt(LLVMInt32Type(), 8, 0), toktype);
+
+  /*
+   * tok->uint = r;
+   *
+   * %token_data = getelementptr inbounds %struct.HParsedToken_.2, %struct.HParsedToken_.2* %3, i32 0, i32 1
+   */
+  LLVMValueRef tokdata = LLVMBuildStructGEP(builder, tok, 1, "token_data");
+  /*
+   * TODO
+   *
+   * This is where we'll need to adjust to handle other types (sign vs. zero extend, omit extend if
+   * r is 64-bit already
+   */
+  LLVMBuildStore(builder, LLVMBuildZExt(builder, r, LLVMInt64Type(), "r"), tokdata);
+  /*
+   * Store the index from the stream into the token
+   */
+  /* %t_index = getelementptr inbounds %struct.HParsedToken_.2, %struct.HParsedToken_.2* %3, i32 0, i32 2 */
+  LLVMValueRef tokindex = LLVMBuildStructGEP(builder, tok, 2, "t_index");
+  /* %s_index = getelementptr inbounds %struct.HInputStream_.0, %struct.HInputStream_.0* %0, i32 0, i32 2 */
+  LLVMValueRef streamindex = LLVMBuildStructGEP(builder, stream, 2, "s_index");
+  /* %4 = load i64, i64* %s_index */
+  /* store i64 %4, i64* %t_index */
+  LLVMBuildStore(builder, LLVMBuildLoad(builder, streamindex, ""), tokindex);
+  /* Store the bit length into the token */
+  LLVMValueRef tokbitlen = LLVMBuildStructGEP(builder, tok, 3, "bit_length");
+  /* TODO handle multiple bit lengths */
+  LLVMBuildStore(builder, LLVMConstInt(LLVMInt64Type(), 8, 0), tokbitlen);
+
+  /*
+   * Now call make_result()
+   *
+   * %make_result = call %struct.HParseResult_.3* @make_result(%struct.HArena_.1* %1, %struct.HParsedToken_.2* %3)
+   */
+  LLVMValueRef result_args[] = { arena, tok };
+  LLVMValueRef mr = LLVMBuildCall(builder, LLVMGetNamedFunction(mod, "make_result"),
+      result_args, 2, "make_result");
+
+  *mr_out = mr;
+}
+
+HParseResult *h_llvm_parse(HAllocator* mm__, const HParser* parser, HInputStream *input_stream) {
+  const HLLVMParser *llvm_parser = parser->backend_data;
+  HArena *arena = h_new_arena(mm__, 0);
+
+  // LLVMRunFunction only supports certain signatures for dumb reasons; it's this hack with
+  // memcpy and function pointers, or writing a shim in LLVM IR.
+  //
+  // LLVMGenericValueRef args[] = {
+  //   LLVMCreateGenericValueOfPointer(input_stream),
+  //   LLVMCreateGenericValueOfPointer(arena)
+  // };
+  // LLVMGenericValueRef res = LLVMRunFunction(llvm_parser->engine, llvm_parser->func, 2, args);
+  // HParseResult *ret = (HParseResult*)LLVMGenericValueToPointer(res);
+
+  void *parse_func_ptr_v;
+  HParseResult * (*parse_func_ptr)(HInputStream *input_stream, HArena *arena);
+  parse_func_ptr_v = LLVMGetPointerToGlobal(llvm_parser->engine, llvm_parser->func);
+  memcpy(&parse_func_ptr, &parse_func_ptr_v, sizeof(parse_func_ptr));
+  HParseResult *ret = parse_func_ptr(input_stream, arena);
+  if (ret) {
+    ret->arena = arena;
+    if (!input_stream->overrun) {
+      size_t bit_length = h_input_stream_pos(input_stream);
+      if (ret->bit_length == 0) {
+	ret->bit_length = bit_length;
+      }
+      if (ret->ast && ret->ast->bit_length != 0) {
+	((HParsedToken*)(ret->ast))->bit_length = bit_length;
+      }
+    } else {
+      ret->bit_length = 0;
+    }
+  } else {
+    ret = NULL;
+  }
+  if (input_stream->overrun) {
+    return NULL; // overrun is always failure.
+  }
+  return ret;
+}
+
+HParserBackendVTable h__llvm_backend_vtable = {
+  .compile = h_llvm_compile,
+  .parse = h_llvm_parse,
+  .free = h_llvm_free
+};
+
+#endif /* defined(HAMMER_LLVM_BACKEND) */

src/backends/llvm/llvm.h

+#ifdef HAMMER_LLVM_BACKEND
+
+#ifndef HAMMER_LLVM__H
+#define HAMMER_LLVM__H
+
+#include "../../internal.h"
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpedantic"
+#include <llvm-c/Core.h>
+#pragma GCC diagnostic pop
+
+LLVMTypeRef llvm_inputstream, llvm_inputstreamptr, llvm_arena, llvm_arenaptr;
+LLVMTypeRef llvm_parsedtoken, llvm_parsedtokenptr, llvm_parseresult, llvm_parseresultptr;
+
+bool h_llvm_make_charset_membership_test(HAllocator* mm__,
+                                         LLVMModuleRef mod, LLVMValueRef func, LLVMBuilderRef builder,
+                                         LLVMValueRef r, HCharset cs,
+                                         LLVMBasicBlockRef yes, LLVMBasicBlockRef no);
+void h_llvm_make_tt_suint(LLVMModuleRef mod, LLVMBuilderRef builder,
+                          LLVMValueRef stream, LLVMValueRef arena, 
+                          LLVMValueRef r, LLVMValueRef *mr_out);
+
+#endif // #ifndef HAMMER_LLVM__H
+
+#endif /* defined(HAMMER_LLVM_BACKEND) */

src/backends/missing.c

+#include "missing.h"
+
+int h_missing_compile(HAllocator* mm__, HParser* parser, const void* params) {
+  /* Always fail */
+
+  return -1;
+}
+
+HParseResult *h_missing_parse(HAllocator* mm__, const HParser* parser, HInputStream* stream) {
+  /* Always fail */
+
+  return NULL;
+}
+
+void h_missing_free(HParser *parser) {
+  /* No-op */
+}
+
+HParserBackendVTable h__missing_backend_vtable = {
+  .compile = h_missing_compile, /* TODO */
+  .parse = h_missing_parse, /* TODO */
+  .free = h_missing_free, /* TODO */
+};

src/backends/missing.h

+#ifndef HAMMER_BACKENDS_MISSING__H
+#define HAMMER_BACKENDS_MISSING__H
+
+#include "../hammer.h"
+#include "../internal.h"
+
+#endif /* !defined(HAMMER_BACKENDS_MISSING__H) */

src/backends/regex.c

+#ifndef _GNU_SOURCE
 #define _GNU_SOURCE
+#endif
 #include <string.h>
 #include <assert.h>
 #include "../internal.h"

src/bindings/cpp/SConscript

@@ -6,7 +6,7 @@ cppenv = env.Clone()
 cppenv.Append(CPPPATH=[".", "vendor/gtest-1.7.0/fused-src", "../.."],
               LIBS=["hammer"],
               LIBPATH=["../../"])
-cppenv.MergeFlags("-Wno-missing-field-initializers -DGTEST_HAS_PTHREAD=0")
+cppenv.MergeFlags("-std=c++11 -Wno-missing-field-initializers -DGTEST_HAS_PTHREAD=0")
 
 
 gtest = cppenv.Object("vendor/gtest-1.7.0/fused-src/gtest/gtest-all.cc")

src/bindings/cpp/cpp_tests.cpp

@@ -356,7 +356,7 @@ namespace {
     EXPECT_TRUE(ParsesTo(p, "aaa", "(u0x61 (u0x61 (u0x61)))"));
   }
 
-};
+}
 
 int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);

src/bindings/cpp/hammer/hammer.hpp

@@ -93,7 +93,7 @@ namespace hammer {
   static inline Parser Token(const uint8_t *buf, size_t len) {
     return Parser(h_token(buf, len));
   }
-  static inline Parser Ch(char ch) {
+  static inline Parser Ch(uint8_t ch) {
     return Parser(h_ch(ch));
   }
   static inline Parser ChRange(uint8_t lower, uint8_t upper) {

src/bindings/dotnet/SConscript

@@ -11,6 +11,7 @@ dotnetenv.Append(CCFLAGS=["-fpic", '-DSWIG', '-Wno-all',
               LIBS=['hammer'],
               LIBPATH=["../.."],
               SWIGFLAGS=["-DHAMMER_INTERNAL__NO_STDARG_H",
+                         "-DSWIG2_CSHARP",
                          "-Isrc/", "-csharp",
                          "-dllimport","hammer_dotnet",
                          "-namespace", "Hammer.Internal"])

src/bindings/python/SConscript

 # -*- python -*-
 import os, os.path
 Import('env libhammer_shared testruns targets')
+# LLVM-related flags
+if GetOption("use_llvm"):
+    Import('llvm_defines')
+    Import('llvm_includes')
 
 pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0)
 
 swig = pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE"))
 setup = ['setup.py']
 pydir = os.path.join(env['BUILD_BASE'], 'src/bindings/python')
-libhammer_python = pythonenv.Command(['hammer.py', 'hammer_wrap.c'], [swig, setup], 'python ' + os.path.join(pydir, 'setup.py') + ' build_ext --swig=swig3.0 --inplace')
+if GetOption("use_llvm"):
+    define_list = ','.join(llvm_defines + ['HAMMER_LLVM_BACKEND'])
+    inc_list = ' '.join(['-I' + e for e in llvm_includes])
+else:
+    define_list = None
+    inc_list = None
+
+swig_opt_list = '-DHAMMER_INTERNAL__NO_STDARG_H -I../../'
+arg_list = 'python ' + os.path.join(pydir, 'setup.py') + ' build_ext --swig=swig3.0 --inplace '
+if inc_list:
+    arg_list = arg_list + inc_list
+    swig_opt_list = swig_opt_list + ' ' + inc_list
+if define_list:
+    arg_list = arg_list + ' --define=\"' + define_list + '\"'
+arg_list = arg_list + ' --swig-opts=\"' + swig_opt_list + '\"'
+
+libhammer_python = pythonenv.Command(['hammer.py', 'hammer_wrap.c'], [swig, setup], arg_list)
+
 Default(libhammer_python)
 
 pytestenv = pythonenv.Clone()

src/hammer.c

@@ -31,6 +31,23 @@ static HParserBackendVTable *backends[PB_MAX + 1] = {
   &h__llk_backend_vtable,
   &h__lalr_backend_vtable,
   &h__glr_backend_vtable,
+  /*
+   * Brittleness warning!
+   *
+   * We're using an enum as an index into this array (don't blame me...)
+   * so it's important that this array have the same size and order as
+   * the corresponding enum values in HParserBackend of src/hammer.h.
+   * Since callers use those enums as numeric constants to select a
+   * backend, dropping/reordering them breaks binary compatibility.
+   * If anyone adds any more optional backends in the future, don't
+   * #ifdef out those enum values in hammer.h, and do provide the
+   * 'missing' stub backend as an alternative here.
+   */
+#ifdef HAMMER_LLVM_BACKEND
+  &h__llvm_backend_vtable,
+#else
+  &h__missing_backend_vtable,
+#endif
 };
 
 

src/hammer.h

@@ -46,7 +46,12 @@ typedef enum HParserBackend_ {
   PB_LLk,
   PB_LALR,
   PB_GLR,
-  PB_MAX = PB_GLR
+  /*
+   * PB_LLVM stays even if no LLVM backend compiled in, since these constants
+   * are exposed to callers.
+   */
+  PB_LLVM,
+  PB_MAX = PB_LLVM
 } HParserBackend;
 
 typedef enum HTokenType_ {
@@ -177,7 +182,7 @@ typedef HParser* (*HContinuation)(HAllocator *mm__, const HParsedToken *x, void
 typedef struct HParserTestcase_ {
   unsigned char* input;
   size_t length;
-  char* output_unambiguous;
+  const char* output_unambiguous;
 } HParserTestcase;
 
 #ifdef SWIG

src/internal.h

@@ -24,7 +24,14 @@
 #define HAMMER_INTERNAL__H
 #include <stdint.h>
 #include <assert.h>
+#include <limits.h>
 #include <string.h>
+#ifdef HAMMER_LLVM_BACKEND
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpedantic"
+#include <llvm-c/Core.h>
+#pragma GCC diagnostic pop
+#endif
 #include "hammer.h"
 #include "platform.h"
 
@@ -152,23 +159,90 @@ static inline void h_sarray_clear(HSArray *arr) {
 
 // }}}
 
-typedef unsigned int *HCharset;
+typedef unsigned int HCharsetWord;
+#define CHARSET_WHOLE_WORD_MASK UINT_MAX
+
+typedef HCharsetWord *HCharset;
+
+#define CHARSET_BITS_PER_WORD (sizeof(HCharsetWord) * 8)
+#define CHARSET_WORDS (256 / CHARSET_BITS_PER_WORD)
+#define CHARSET_SIZE (CHARSET_WORDS * sizeof(HCharsetWord))
+#define CHARSET_BIT_IDX_TO_WORD(idx) \
+  (((unsigned int)(idx)) / CHARSET_BITS_PER_WORD)
+#define CHARSET_BIT_IDX_TO_BIT_IN_WORD(idx) \
+  (((unsigned int)(idx)) % CHARSET_BITS_PER_WORD)
+#define CHART_WORD_AND_BIT_TO_BIT_IDX(word,bit) \
+    ((uint8_t)(CHARSET_BITS_PER_WORD * ((unsigned int)(word)) + \
+      ((unsigned int)(bit))))
+#define CHARSET_BIT_POS_IN_WORD_MASK(bit) \
+  ((((HCharsetWord)(1)) << (bit)) & CHARSET_WHOLE_WORD_MASK)
+/* Mask for all bits below a position */
+#define CHARSET_BIT_MASK_UP_TO_POS(bit) \
+  ((CHARSET_BIT_POS_IN_WORD_MASK((bit)) - 1) & CHARSET_WHOLE_WORD_MASK)
+/* Mask off all bits above and including a position */
+#define CHARSET_BIT_MASK_FROM_POS(bit) \
+  ((~CHARSET_BIT_MASK_UP_TO_POS((bit))) & CHARSET_WHOLE_WORD_MASK)
+
+static inline HCharset copy_charset(HAllocator *mm__, HCharset in) {
+  HCharset cs = h_new(HCharsetWord, CHARSET_WORDS);
+  memcpy(cs, in, CHARSET_SIZE);
+  return cs;
+}
 
 static inline HCharset new_charset(HAllocator* mm__) {
-  HCharset cs = h_new(unsigned int, 256 / (sizeof(unsigned int) * 8));
-  memset(cs, 0, 32);  // 32 bytes = 256 bits
+  HCharset cs = h_new(HCharsetWord, CHARSET_WORDS);
+  memset(cs, 0, CHARSET_SIZE);
   return cs;
 }
 
+static inline void charset_complement(HCharset cs) {
+  for (unsigned int i = 0; i < CHARSET_WORDS; ++i) cs[i] = ~(cs[i]);
+}
+
 static inline int charset_isset(HCharset cs, uint8_t pos) {
-  return !!(cs[pos / (sizeof(*cs)*8)] & (1 << (pos % (sizeof(*cs)*8))));
+  return !!(cs[CHARSET_BIT_IDX_TO_WORD(pos)] &
+      CHARSET_BIT_POS_IN_WORD_MASK(CHARSET_BIT_IDX_TO_BIT_IN_WORD(pos)));
+}
+
+static inline void charset_restrict_to_range(HCharset cs, uint8_t idx_start, uint8_t idx_end) {
+  HCharsetWord mask;
+
+  if (idx_end < idx_start) {
+    /* Range is empty, clear the charset */
+    memset(cs, 0, CHARSET_SIZE);
+  } else {
+    /* Clear below, if any */
+    if (CHARSET_BIT_IDX_TO_WORD(idx_start) > 0) {
+      memset(cs, 0, CHARSET_BIT_IDX_TO_WORD(idx_start) * sizeof(HCharsetWord));
+    }
+    /* Note this partial start/ending word code still works if they are the same word */
+    /* Mask partial starting word, if any */
+    if (CHARSET_BIT_IDX_TO_BIT_IN_WORD(idx_start) != 0) {
+      mask = CHARSET_BIT_MASK_FROM_POS(CHARSET_BIT_IDX_TO_BIT_IN_WORD(idx_start));
+      cs[CHARSET_BIT_IDX_TO_WORD(idx_start)] &= mask;
+    }
+    /* Mask partial ending word, if any */
+    if (CHARSET_BIT_IDX_TO_BIT_IN_WORD(idx_end) != CHARSET_BITS_PER_WORD - 1) {
+      mask = CHARSET_BIT_MASK_UP_TO_POS(CHARSET_BIT_IDX_TO_BIT_IN_WORD(idx_end));
+      mask |= CHARSET_BIT_POS_IN_WORD_MASK(CHARSET_BIT_IDX_TO_BIT_IN_WORD(idx_end));
+      cs[CHARSET_BIT_IDX_TO_WORD(idx_end)] &= mask;
+    }
+    /* Clear above, if any */
+    if (CHARSET_BIT_IDX_TO_WORD(idx_end) + 1 < CHARSET_WORDS) {
+      memset(cs + CHARSET_BIT_IDX_TO_WORD(idx_end) + 1, 0,
+             (CHARSET_WORDS - (CHARSET_BIT_IDX_TO_WORD(idx_end) + 1)) *
+             sizeof(HCharsetWord));
+    }
+  }
 }
 
 static inline void charset_set(HCharset cs, uint8_t pos, int val) {
-  cs[pos / (sizeof(*cs)*8)] =
+  cs[CHARSET_BIT_IDX_TO_WORD(pos)] =
     val
-    ? cs[pos / (sizeof(*cs)*8)] |  (1 << (pos % (sizeof(*cs)*8)))
-    : cs[pos / (sizeof(*cs)*8)] & ~(1 << (pos % (sizeof(*cs)*8)));
+    ? cs[CHARSET_BIT_IDX_TO_WORD(pos)] |  CHARSET_BIT_POS_IN_WORD_MASK(
+        CHARSET_BIT_IDX_TO_BIT_IN_WORD(pos))
+    : cs[CHARSET_BIT_IDX_TO_WORD(pos)] & ~CHARSET_BIT_POS_IN_WORD_MASK(
+        CHARSET_BIT_IDX_TO_BIT_IN_WORD(pos));
 }
 
 typedef unsigned int HHashValue;
@@ -322,6 +396,10 @@ extern HParserBackendVTable h__packrat_backend_vtable;
 extern HParserBackendVTable h__llk_backend_vtable;
 extern HParserBackendVTable h__lalr_backend_vtable;
 extern HParserBackendVTable h__glr_backend_vtable;
+extern HParserBackendVTable h__missing_backend_vtable;
+#ifdef HAMMER_LLVM_BACKEND
+extern HParserBackendVTable h__llvm_backend_vtable;
+#endif
 // }}}
 
 // TODO(thequux): Set symbol visibility for these functions so that they aren't exported.
@@ -419,6 +497,9 @@ struct HParserVtable_ {
   bool (*isValidCF)(void *env);
   bool (*compile_to_rvm)(HRVMProg *prog, void* env); // FIXME: forgot what the bool return value was supposed to mean.
   void (*desugar)(HAllocator *mm__, HCFStack *stk__, void *env);
+#ifdef HAMMER_LLVM_BACKEND
+  bool (*llvm)(HAllocator *mm__, LLVMBuilderRef builder, LLVMValueRef func, LLVMModuleRef mod, void *env);
+#endif
   bool higher; // false if primitive
 };
 

src/parsers/bind.c

@@ -21,7 +21,10 @@ static void *aa_alloc(HAllocator *allocator, size_t size)
 
 static void *aa_realloc(HAllocator *allocator, void *ptr, size_t size)
 {
+  #pragma GCC diagnostic push
+  #pragma GCC diagnostic ignored "-Wunused-variable"
     HArena *arena = ((ArenaAllocator *)allocator)->arena;
+  #pragma GCC diagnostic pop
     assert(((void)"XXX need realloc for arena allocator", 0));
     return NULL;
 }