diff --git a/.gitignore b/.gitignore
index e05a551621bdb4230eeabf1e31cf7b053a1ef90c..721dcf9377a334d6d4f75617ea31df6f7c2ee639 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,3 +21,5 @@ Session.vim
cscope.out
build/
.sconsign.dblite
+*.os
+*.pyc
diff --git a/.travis.yml b/.travis.yml
index 1458ecd6fd00f402532023e03eacb24b38eded0e..8c5af523ada62766b6fdc18fb47fd92f7a613964 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,7 +2,27 @@ language: c
compiler:
- gcc
- clang
+env:
+ - BINDINGS=none
+matrix:
+ include:
+ - compiler: gcc
+ language: python
+ python: 2.7
+ env: BINDINGS=python
+ - compiler: clang
+ language: python
+ python: 2.7
+ env: BINDINGS=python
+before_install:
+ - sudo apt-get update -qq
+ - if [ "$BINDINGS" == "python" ]; then sudo apt-get install -qq swig python-dev; fi
script:
- - scons
+ - scons bindings=$BINDINGS test
notifications:
- irc: "irc.upstandinghackers.com#hammer"
+ irc:
+ channels:
+ - "irc.upstandinghackers.com#hammer"
+ use_notice: true
+ skip_join: true
+
diff --git a/SConstruct b/SConstruct
index 623bc5d8ef2ca99fe5ad79c5307426104078c736..e30f6df284eb488fa55e7edb4ed8bcbe666705d5 100644
--- a/SConstruct
+++ b/SConstruct
@@ -7,8 +7,12 @@ import sys
vars = Variables(None, ARGUMENTS)
vars.Add(PathVariable('DESTDIR', "Root directory to install in (useful for packaging scripts)", None, PathVariable.PathIsDirCreate))
vars.Add(PathVariable('prefix', "Where to install in the FHS", "/usr/local", PathVariable.PathAccept))
+vars.Add(ListVariable('bindings', 'Language bindings to build', 'none', ['python']))
-env = Environment(ENV = {'PATH' : os.environ['PATH']}, variables = vars)
+env = Environment(ENV = {'PATH' : os.environ['PATH']}, variables = vars, tools=['default', 'scanreplace'], toolpath=['tools'])
+
+if not 'bindings' in env:
+ env['bindings'] = []
def calcInstallPath(*elements):
path = os.path.abspath(os.path.join(*map(env.subst, elements)))
@@ -28,11 +32,16 @@ if 'DESTDIR' in env:
env['libpath'] = calcInstallPath("$prefix", "lib")
env['incpath'] = calcInstallPath("$prefix", "include", "hammer")
-# TODO: Add pkgconfig
+env['parsersincpath'] = calcInstallPath("$prefix", "include", "hammer", "parsers")
+env['backendsincpath'] = calcInstallPath("$prefix", "include", "hammer", "backends")
+env['pkgconfigpath'] = calcInstallPath("$prefix", "lib", "pkgconfig")
+env.ScanReplace('libhammer.pc.in')
env.MergeFlags("-std=gnu99 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-attributes")
-if not env['PLATFORM'] == 'darwin':
+if env['PLATFORM'] == 'darwin':
+ env.Append(SHLINKFLAGS = ['-install_name', '$TARGET'])
+else:
env.MergeFlags("-lrt")
AddOption("--variant",
@@ -49,7 +58,12 @@ AddOption("--coverage",
action="store_true",
help="Build with coverage instrumentation")
-env['BUILDDIR'] = 'build/$VARIANT'
+AddOption("--in-place",
+ dest="in_place",
+ default=False,
+ action="store_true",
+ help="Build in-place, rather than in the build/<variant> tree")
+
dbg = env.Clone(VARIANT='debug')
dbg.Append(CCFLAGS=['-g'])
@@ -68,19 +82,38 @@ if GetOption("coverage"):
LDFLAGS=["-fprofile-arcs", "-ftest-coverage"],
LIBS=['gcov'])
+env["CC"] = os.getenv("CC") or env["CC"]
+env["CXX"] = os.getenv("CXX") or env["CXX"]
+
if os.getenv("CC") == "clang" or env['PLATFORM'] == 'darwin':
env.Replace(CC="clang",
CXX="clang++")
+env["ENV"].update(x for x in os.environ.items() if x[0].startswith("CCC_"))
+
#rootpath = env['ROOTPATH'] = os.path.abspath('.')
#env.Append(CPPPATH=os.path.join('#', "hammer"))
+testruns = []
+
Export('env')
+Export('testruns')
+
+if not GetOption("in_place"):
+ env['BUILD_BASE'] = 'build/$VARIANT'
+ lib = env.SConscript(["src/SConscript"], variant_dir='$BUILD_BASE/src')
+ env.Alias("examples", env.SConscript(["examples/SConscript"], variant_dir='$BUILD_BASE/examples'))
+else:
+ env['BUILD_BASE'] = '.'
+ lib = env.SConscript(["src/SConscript"])
+ env.Alias(env.SConscript(["examples/SConscript"]))
-env.SConscript(["src/SConscript"], variant_dir='build/$VARIANT/src')
-env.SConscript(["examples/SConscript"], variant_dir='build/$VARIANT/examples')
+#env.Command('test', '$BUILD_BASE/src/test_suite', 'env LD_LIBRARY_PATH=$BUILD_BASE/src $SOURCE')
-env.Command('test', 'build/$VARIANT/src/test_suite', 'env LD_LIBRARY_PATH=build/$VARIANT/src $SOURCE')
+env.Alias("test", testruns)
env.Alias("install", "$libpath")
env.Alias("install", "$incpath")
+env.Alias("install", "$parsersincpath")
+env.Alias("install", "$backendsincpath")
+env.Alias("install", "$pkgconfigpath")
diff --git a/examples/base64_sem1.c b/examples/base64_sem1.c
index 46e58da23dcf49a7189c7d89a92ba8fff2780eea..4da171f355c9f8b7312ae29e231d58592602b901 100644
--- a/examples/base64_sem1.c
+++ b/examples/base64_sem1.c
@@ -23,7 +23,7 @@
// They must be named act_<rulename>.
///
-HParsedToken *act_bsfdig(const HParseResult *p)
+HParsedToken *act_bsfdig(const HParseResult *p, void* user_data)
{
HParsedToken *res = H_MAKE_UINT(0);
@@ -54,7 +54,7 @@ H_ACT_APPLY(act_index0, h_act_index, 0);
#define act_document act_index0
// General-form action to turn a block of base64 digits into bytes.
-HParsedToken *act_base64_n(int n, const HParseResult *p)
+HParsedToken *act_base64_n(int n, const HParseResult *p, void* user_data)
{
HParsedToken *res = H_MAKE_SEQN(n);
@@ -83,7 +83,7 @@ H_ACT_APPLY(act_base64_3, act_base64_n, 3);
H_ACT_APPLY(act_base64_2, act_base64_n, 2);
H_ACT_APPLY(act_base64_1, act_base64_n, 1);
-HParsedToken *act_base64(const HParseResult *p)
+HParsedToken *act_base64(const HParseResult *p, void* user_data)
{
assert(p->ast->token_type == TT_SEQUENCE);
assert(p->ast->seq->used == 2);
diff --git a/examples/base64_sem2.c b/examples/base64_sem2.c
index 84155da23b7f09ca84a6487dedf031c27955e4e9..b7a2263efc1810a0140785f3494dc29a4d204f03 100644
--- a/examples/base64_sem2.c
+++ b/examples/base64_sem2.c
@@ -49,7 +49,7 @@ uint8_t bsfdig_value(const HParsedToken *p)
// helper: append a byte value to a sequence
#define seq_append_byte(res, b) h_seq_snoc(res, H_MAKE_UINT(b))
-HParsedToken *act_base64(const HParseResult *p)
+HParsedToken *act_base64(const HParseResult *p, void* user_data)
{
assert(p->ast->token_type == TT_SEQUENCE);
assert(p->ast->seq->used == 2);
diff --git a/examples/dns.c b/examples/dns.c
index 3f730b970cd6d59677b7d8a38dcec8ea7b10ef48..63df3a443377dfd93717f091d107ffbccb41ac05 100644
--- a/examples/dns.c
+++ b/examples/dns.c
@@ -15,7 +15,7 @@
// Validations
///
-bool validate_hdzero(HParseResult *p) {
+bool validate_hdzero(HParseResult *p, void* user_data) {
if (TT_UINT != p->ast->token_type)
return false;
return (0 == p->ast->uint);
@@ -25,7 +25,7 @@ bool validate_hdzero(HParseResult *p) {
* Every DNS message should have QDCOUNT entries in the question
* section, and ANCOUNT+NSCOUNT+ARCOUNT resource records.
*/
-bool validate_message(HParseResult *p) {
+bool validate_message(HParseResult *p, void* user_data) {
if (TT_SEQUENCE != p->ast->token_type)
return false;
@@ -86,7 +86,7 @@ void set_rdata(struct dns_rr *rr, HCountedArray *rdata) {
}
}
-HParsedToken* act_header(const HParseResult *p) {
+HParsedToken* act_header(const HParseResult *p, void* user_data) {
HParsedToken **fields = h_seq_elements(p->ast);
dns_header_t header_ = {
.id = H_CAST_UINT(fields[0]),
@@ -109,7 +109,7 @@ HParsedToken* act_header(const HParseResult *p) {
return H_MAKE(dns_header_t, header);
}
-HParsedToken* act_label(const HParseResult *p) {
+HParsedToken* act_label(const HParseResult *p, void* user_data) {
dns_label_t *r = H_ALLOC(dns_label_t);
r->len = h_seq_len(p->ast);
@@ -121,7 +121,7 @@ HParsedToken* act_label(const HParseResult *p) {
return H_MAKE(dns_label_t, r);
}
-HParsedToken* act_rr(const HParseResult *p) {
+HParsedToken* act_rr(const HParseResult *p, void* user_data) {
dns_rr_t *rr = H_ALLOC(dns_rr_t);
rr->name = *H_FIELD(dns_domain_t, 0);
@@ -136,7 +136,7 @@ HParsedToken* act_rr(const HParseResult *p) {
return H_MAKE(dns_rr_t, rr);
}
-HParsedToken* act_question(const HParseResult *p) {
+HParsedToken* act_question(const HParseResult *p, void* user_data) {
dns_question_t *q = H_ALLOC(dns_question_t);
HParsedToken **fields = h_seq_elements(p->ast);
@@ -153,7 +153,7 @@ HParsedToken* act_question(const HParseResult *p) {
return H_MAKE(dns_question_t, q);
}
-HParsedToken* act_message(const HParseResult *p) {
+HParsedToken* act_message(const HParseResult *p, void* user_data) {
h_pprint(stdout, p->ast, 0, 2);
dns_message_t *msg = H_ALLOC(dns_message_t);
diff --git a/examples/dns_common.c b/examples/dns_common.c
index 01dd8f0fb4630680174f8fff657c70f845775624..bf934d6184190d419495aa6011118d5a67e4f7fa 100644
--- a/examples/dns_common.c
+++ b/examples/dns_common.c
@@ -10,7 +10,7 @@ H_ACT_APPLY(act_index0, h_act_index, 0)
/**
* A label can't be more than 63 characters.
*/
-bool validate_label(HParseResult *p) {
+bool validate_label(HParseResult *p, void* user_data) {
if (TT_SEQUENCE != p->ast->token_type)
return false;
return (64 > p->ast->seq->used);
@@ -18,7 +18,7 @@ bool validate_label(HParseResult *p) {
#define act_label h_act_flatten
-HParsedToken* act_domain(const HParseResult *p) {
+HParsedToken* act_domain(const HParseResult *p, void* user_data) {
HParsedToken *ret = NULL;
char *arr = NULL;
diff --git a/examples/dns_common.h b/examples/dns_common.h
index 8af014b2f22c28da36e3312b56e355e1d5500e73..3e30770ad60a6acd4f375f2d36b5a9def5c8906a 100644
--- a/examples/dns_common.h
+++ b/examples/dns_common.h
@@ -7,6 +7,6 @@
HParser* init_domain();
HParser* init_character_string();
-HParsedToken* act_index0(const HParseResult *p);
+HParsedToken* act_index0(const HParseResult *p, void* user_data);
#endif
diff --git a/examples/rr.c b/examples/rr.c
index dd250637a7eaccf1074f9441bb6d273452cc130d..c179922d664986be6277748c55e9050d1bfcb687 100644
--- a/examples/rr.c
+++ b/examples/rr.c
@@ -11,13 +11,13 @@
// Validations and Semantic Actions
///
-bool validate_null(HParseResult *p) {
+bool validate_null(HParseResult *p, void* user_data) {
if (TT_SEQUENCE != p->ast->token_type)
return false;
return (65536 > p->ast->seq->used);
}
-HParsedToken *act_null(const HParseResult *p) {
+HParsedToken *act_null(const HParseResult *p, void* user_data) {
dns_rr_null_t *null = H_ALLOC(dns_rr_null_t);
size_t len = h_seq_len(p->ast);
@@ -28,7 +28,7 @@ HParsedToken *act_null(const HParseResult *p) {
return H_MAKE(dns_rr_null_t, null);
}
-HParsedToken *act_txt(const HParseResult *p) {
+HParsedToken *act_txt(const HParseResult *p, void* user_data) {
dns_rr_txt_t *txt = H_ALLOC(dns_rr_txt_t);
const HCountedArray *arr = H_CAST_SEQ(p->ast);
@@ -47,7 +47,7 @@ HParsedToken *act_txt(const HParseResult *p) {
return H_MAKE(dns_rr_txt_t, txt);
}
-HParsedToken* act_cstr(const HParseResult *p) {
+HParsedToken* act_cstr(const HParseResult *p, void* user_data) {
dns_cstr_t *cs = H_ALLOC(dns_cstr_t);
const HCountedArray *arr = H_CAST_SEQ(p->ast);
@@ -60,7 +60,7 @@ HParsedToken* act_cstr(const HParseResult *p) {
return H_MAKE(dns_cstr_t, cs);
}
-HParsedToken* act_soa(const HParseResult *p) {
+HParsedToken* act_soa(const HParseResult *p, void* user_data) {
dns_rr_soa_t *soa = H_ALLOC(dns_rr_soa_t);
soa->mname = *H_FIELD(dns_domain_t, 0);
@@ -74,7 +74,7 @@ HParsedToken* act_soa(const HParseResult *p) {
return H_MAKE(dns_rr_soa_t, soa);
}
-HParsedToken* act_wks(const HParseResult *p) {
+HParsedToken* act_wks(const HParseResult *p, void* user_data) {
dns_rr_wks_t *wks = H_ALLOC(dns_rr_wks_t);
wks->address = H_FIELD_UINT(0);
@@ -87,7 +87,7 @@ HParsedToken* act_wks(const HParseResult *p) {
return H_MAKE(dns_rr_wks_t, wks);
}
-HParsedToken* act_hinfo(const HParseResult *p) {
+HParsedToken* act_hinfo(const HParseResult *p, void* user_data) {
dns_rr_hinfo_t *hinfo = H_ALLOC(dns_rr_hinfo_t);
hinfo->cpu = *H_FIELD(dns_cstr_t, 0);
@@ -96,7 +96,7 @@ HParsedToken* act_hinfo(const HParseResult *p) {
return H_MAKE(dns_rr_hinfo_t, hinfo);
}
-HParsedToken* act_minfo(const HParseResult *p) {
+HParsedToken* act_minfo(const HParseResult *p, void* user_data) {
dns_rr_minfo_t *minfo = H_ALLOC(dns_rr_minfo_t);
minfo->rmailbx = *H_FIELD(dns_domain_t, 0);
@@ -105,7 +105,7 @@ HParsedToken* act_minfo(const HParseResult *p) {
return H_MAKE(dns_rr_minfo_t, minfo);
}
-HParsedToken* act_mx(const HParseResult *p) {
+HParsedToken* act_mx(const HParseResult *p, void* user_data) {
dns_rr_mx_t *mx = H_ALLOC(dns_rr_mx_t);
mx->preference = H_FIELD_UINT(0);
@@ -184,7 +184,7 @@ HParser* init_rdata(uint16_t type) {
for(uint16_t i = 0; i<RDATA_TYPE_MAX+1; i++) {
if(parsers[i]) {
parsers[i] = h_action(h_sequence(parsers[i], h_end_p(), NULL),
- act_index0);
+ act_index0, NULL);
}
}
diff --git a/libhammer.pc.in b/libhammer.pc.in
new file mode 100644
index 0000000000000000000000000000000000000000..97ded13f8806ab4e3146f9cb79924dfa52521cc7
--- /dev/null
+++ b/libhammer.pc.in
@@ -0,0 +1,10 @@
+prefix=/usr
+exec_prefix=${prefix}
+includedir=${prefix}/include
+libdir=${exec_prefix}/lib
+
+Name: libhammer
+Description: The Hammer parsing library
+Version: 0.9.0
+Cflags: -I${includedir}/hammer
+Libs: -L${libdir} -lhammer
diff --git a/src/SConscript b/src/SConscript
index 9b5c868c5f2966c9305139c4805245d8c1c85c26..038839320cc6f65b0ea5a6a0140442923bd0db00 100644
--- a/src/SConscript
+++ b/src/SConscript
@@ -1,12 +1,21 @@
# -*- python -*-
-Import('env')
-
-bindings = []
+import os.path
+Import('env testruns')
dist_headers = [
"hammer.h",
"allocator.h",
- "glue.h"
+ "glue.h",
+ "internal.h"
+]
+
+parsers_headers = [
+ "parsers/parser_internal.h"
+]
+
+backends_headers = [
+ "backends/regex.h",
+ "backends/contextfree.h"
]
parsers = ['parsers/%s.c'%s for s in
@@ -49,27 +58,35 @@ misc_hammer_parts = [
'glue.c',
'hammer.c',
'pprint.c',
+ 'registry.c',
'system_allocator.c']
-tests = ['t_benchmark.c',
- 't_bitreader.c',
- 't_bitwriter.c',
- 't_parser.c',
- 't_grammar.c',
- 't_misc.c']
+ctests = ['t_benchmark.c',
+ 't_bitreader.c',
+ 't_bitwriter.c',
+ 't_parser.c',
+ 't_grammar.c',
+ 't_misc.c']
libhammer_shared = env.SharedLibrary('hammer', parsers + backends + misc_hammer_parts)
libhammer_static = env.StaticLibrary('hammer', parsers + backends + misc_hammer_parts)
+Default(libhammer_shared, libhammer_static)
env.Install("$libpath", [libhammer_static, libhammer_shared])
env.Install("$incpath", dist_headers)
+env.Install("$parsersincpath", parsers_headers)
+env.Install("$backendsincpath", backends_headers)
+env.Install("$pkgconfigpath", "../../../libhammer.pc")
testenv = env.Clone()
testenv.ParseConfig('pkg-config --cflags --libs glib-2.0')
testenv.Append(LIBS=['hammer'], LIBPATH=['.'])
-testenv.Program('test_suite', tests + ['test_suite.c'])
+ctestexec = testenv.Program('test_suite', ctests + ['test_suite.c'])
+ctest = Alias('testc', [ctestexec], "".join(["env LD_LIBRARY_PATH=", os.path.dirname(ctestexec[0].path), " ", ctestexec[0].path]))
+AlwaysBuild(ctest)
+testruns.append(ctest)
Export("libhammer_static libhammer_shared")
-for b in bindings:
+for b in env['bindings']:
env.SConscript(["bindings/%s/SConscript" % b])
diff --git a/src/allocator.h b/src/allocator.h
index 2dfc14e689f825efabc0d7c46b515217ccd90abb..a02d97cd51873c7ae979c0dfa208bb3bd975f986 100644
--- a/src/allocator.h
+++ b/src/allocator.h
@@ -29,7 +29,11 @@ typedef struct HAllocator_ {
typedef struct HArena_ HArena ; // hidden implementation
HArena *h_new_arena(HAllocator* allocator, size_t block_size); // pass 0 for default...
+#ifndef SWIG
void* h_arena_malloc(HArena *arena, size_t count) __attribute__(( malloc, alloc_size(2) ));
+#else
+void* h_arena_malloc(HArena *arena, size_t count);
+#endif
void h_arena_free(HArena *arena, void* ptr); // For future expansion, with alternate memory managers.
void h_delete_arena(HArena *arena);
diff --git a/src/backends/contextfree.h b/src/backends/contextfree.h
index b387e55df21387d4be137d7ff159889de50985ba..ab04ab523064fba731d2cf292bcc1a7bd28a9faf 100644
--- a/src/backends/contextfree.h
+++ b/src/backends/contextfree.h
@@ -1,3 +1,8 @@
+/*
+ * NOTE: This is an internal header and installed for use by extensions. The
+ * API is not guaranteed stable.
+*/
+
// This is an internal header; it provides macros to make desugaring cleaner.
#include <assert.h>
#include "../internal.h"
diff --git a/src/backends/llk.c b/src/backends/llk.c
index c0cf6afef75aae37c9b9479cdc44223244326ab7..2bc39daf92b371b3b22b783623442eee36053bc0 100644
--- a/src/backends/llk.c
+++ b/src/backends/llk.c
@@ -372,13 +372,13 @@ HParseResult *h_llk_parse(HAllocator* mm__, const HParser* parser, HInputStream*
// perform token reshape if indicated
if(x->reshape)
- tok = (HParsedToken *)x->reshape(make_result(arena, tok));
+ tok = (HParsedToken *)x->reshape(make_result(arena, tok), x->user_data);
// call validation and semantic action, if present
- if(x->pred && !x->pred(make_result(tarena, tok)))
+ if(x->pred && !x->pred(make_result(tarena, tok), x->user_data))
goto no_parse; // validation failed -> no parse
if(x->action)
- tok = (HParsedToken *)x->action(make_result(arena, tok));
+ tok = (HParsedToken *)x->action(make_result(arena, tok), x->user_data);
// append to result sequence
h_carray_append(seq, tok);
diff --git a/src/backends/lr.c b/src/backends/lr.c
index d258e8a7831d11985e82d4ae76ba692e05bc6900..e7f237756361303102440700af4ceb5fcfb5abdf 100644
--- a/src/backends/lr.c
+++ b/src/backends/lr.c
@@ -307,13 +307,13 @@ bool h_lrengine_step(HLREngine *engine, const HLRAction *action)
// perform token reshape if indicated
if(symbol->reshape)
- value = (HParsedToken *)symbol->reshape(make_result(arena, value));
+ value = (HParsedToken *)symbol->reshape(make_result(arena, value), symbol->user_data);
// call validation and semantic action, if present
- if(symbol->pred && !symbol->pred(make_result(tarena, value)))
+ if(symbol->pred && !symbol->pred(make_result(tarena, value), symbol->user_data))
return false; // validation failed -> no parse; terminate
if(symbol->action)
- value = (HParsedToken *)symbol->action(make_result(arena, value));
+ value = (HParsedToken *)symbol->action(make_result(arena, value), symbol->user_data);
// this is LR, building a right-most derivation bottom-up, so no reduce can
// follow a reduce. we can also assume no conflict follows for GLR if we
diff --git a/src/backends/packrat.c b/src/backends/packrat.c
index 8aa1f8ed670502f4b59e9be6498d22eaa74723ad..87f166de95e8fb2b58bd8455622ca8f7cf007115 100644
--- a/src/backends/packrat.c
+++ b/src/backends/packrat.c
@@ -83,7 +83,7 @@ void setupLR(const HParser *p, HParseState *state, HLeftRec *rec_detect) {
some->eval_set = NULL;
rec_detect->head = some;
}
- assert(state->lr_stack->head != NULL);
+ //assert(state->lr_stack->head != NULL);
HSlistNode *head = state->lr_stack->head;
HLeftRec *lr;
while (head && (lr = head->elem)->rule != p) {
diff --git a/src/backends/regex.h b/src/backends/regex.h
index a84904d0b6bcd4ea18b8b0281b625c4c303e9a8d..4ea85a884b3fdec764f9ad94fc4c48880a0a6ea2 100644
--- a/src/backends/regex.h
+++ b/src/backends/regex.h
@@ -1,3 +1,8 @@
+/*
+ * NOTE: This is an internal header and installed for use by extensions. The
+ * API is not guaranteed stable.
+*/
+
// Internal defs
#ifndef HAMMER_BACKEND_REGEX__H
#define HAMMER_BACKEND_REGEX__H
diff --git a/src/bindings/desugar-header.pl b/src/bindings/desugar-header.pl
new file mode 100644
index 0000000000000000000000000000000000000000..5bdd11e665b86af623583a94002551795d7b9ade
--- /dev/null
+++ b/src/bindings/desugar-header.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl -w
+
+
+my $arg = qr/[^,]*/;
+
+while(<>) {
+ chomp;
+ if (/^HAMMER_FN_DECL_NOARG\(([^,]*), ([^,]*)\);/) {
+ print "$1 $2(void);\n";
+ print "$1 $2__m(HAllocator* mm__);\n";
+ } elsif (/^HAMMER_FN_DECL\(([^,]*), ([^,]*), ([^)]*)\);/) {
+ print "$1 $2($3);\n";
+ print "$1 $2__m(HAllocator* mm__, $3);\n";
+ } elsif (/^HAMMER_FN_DECL_VARARGS_ATTR\((__attribute__\(\([^)]*\)\)), ([^,]*), ([^,]*), ([^)]*)\);/) {
+ print "$2 $3($4, ...);\n";
+ print "$2 $3__m(HAllocator *mm__, $4, ...);\n";
+ print "$2 $3__a(void* args);\n";
+ print "$2 $3__ma(HAllocator* mm__, void* args);\n";
+ } elsif (/^HAMMER_FN_DECL/) {
+ print "\e[1;31m!!!\e[0m " . $_ . "\n";
+ }
+}
diff --git a/src/bindings/python/SConscript b/src/bindings/python/SConscript
new file mode 100644
index 0000000000000000000000000000000000000000..0f1e9a92e5215fcb03547a3e075411e36bf2a949
--- /dev/null
+++ b/src/bindings/python/SConscript
@@ -0,0 +1,29 @@
+# -*- python -*-
+import os, os.path
+Import('env libhammer_shared testruns')
+
+pythonenv = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0)
+
+pythonenv.Append(CPPPATH = ['../../'])
+pythonenv.Append(CCFLAGS = ['-fpic', '-DSWIG', '-Wno-all', '-Wno-extra', '-Wno-error'])
+pythonenv.ParseConfig("pkg-config --cflags python")
+pythonenv.Append(LIBS = ['hammer'])
+pythonenv.Append(LIBPATH = ['../../'])
+pythonenv.Append(SWIGFLAGS = ['-DHAMMER_INTERNAL__NO_STDARG_H', '-Isrc/', '-python'])
+
+pythonenv.Command("hammer.i", "../swig/hammer.i", Copy("$TARGET", "$SOURCE"))
+
+swig = ['hammer.i']
+
+libhammer_python = pythonenv.SharedLibrary('hammer', swig, SHLIBPREFIX='_')
+Default(libhammer_python)
+
+pytestenv = pythonenv.Clone()
+pytestenv['ENV']['LD_LIBRARY_PATH'] = os.path.dirname(str(libhammer_shared[0]))
+pytests = ['hammer_tests.py']
+pytestexec = pytestenv.Command(['hammer.pyc', 'hammer_tests.pyc'], pytests + libhammer_python, "nosetests -vv $SOURCE")
+pytest = Alias("testpython", [pytestexec], pytestexec)
+AlwaysBuild(pytest)
+testruns.append(pytest)
+
+
diff --git a/src/bindings/python/hammer.py b/src/bindings/python/hammer.py
new file mode 100644
index 0000000000000000000000000000000000000000..36b78c8c8d3408b68e7df1e92a70be53e159f877
--- /dev/null
+++ b/src/bindings/python/hammer.py
@@ -0,0 +1,488 @@
+from cffi import FFI
+import threading
+import sys
+
+_ffi = FFI()
+
+# {{{ Types
+
+_ffi.cdef("typedef struct HAllocator_ HAllocator;")
+_ffi.cdef("typedef struct HArena_ HArena;")
+_ffi.cdef("typedef int bool;")
+_ffi.cdef("typedef struct HParseState_ HParseState;")
+_ffi.cdef("""
+typedef enum HParserBackend_ {
+ PB_MIN = 0,
+ PB_PACKRAT = 0, // PB_MIN is always the default.
+ PB_REGULAR,
+ PB_LLk,
+ PB_LALR,
+ PB_GLR
+// TODO: support PB_MAX
+} HParserBackend;
+""")
+_ffi.cdef("""
+typedef enum HTokenType_ {
+ // Before you change the explicit values of these, think of the poor bindings ;_;
+ TT_NONE = 1,
+ TT_BYTES = 2,
+ TT_SINT = 4,
+ TT_UINT = 8,
+ TT_SEQUENCE = 16,
+ TT_RESERVED_1, // reserved for backend-specific internal use
+ TT_ERR = 32,
+ TT_USER = 64,
+ TT_MAX
+} HTokenType;
+""")
+_ffi.cdef("""
+typedef struct HCountedArray_ {
+ size_t capacity;
+ size_t used;
+ HArena * arena;
+ struct HParsedToken_ **elements;
+} HCountedArray;
+""")
+_ffi.cdef("""
+typedef struct HBytes_ {
+ const uint8_t *token;
+ size_t len;
+} HBytes;
+""")
+_ffi.cdef("""
+typedef struct HParsedToken_ {
+ HTokenType token_type;
+ union {
+ HBytes bytes;
+ int64_t sint;
+ uint64_t uint;
+ double dbl;
+ float flt;
+ HCountedArray *seq; // a sequence of HParsedToken's
+ void *user;
+ };
+ size_t index;
+ char bit_offset;
+} HParsedToken;
+""")
+_ffi.cdef("""
+typedef struct HParseResult_ {
+ const HParsedToken *ast;
+ long long bit_length;
+ HArena * arena;
+} HParseResult;
+""")
+
+_ffi.cdef("""typedef HParsedToken* (*HAction)(const HParseResult *p);""")
+_ffi.cdef("""typedef bool (*HPredicate)(HParseResult *p);""")
+_ffi.cdef("""
+typedef struct HCFChoice_ HCFChoice;
+typedef struct HRVMProg_ HRVMProg;
+typedef struct HParserVtable_ HParserVtable;
+""")
+
+_ffi.cdef("typedef struct HParser_ HParser;")
+_ffi.cdef("""
+typedef struct HParserTestcase_ {
+ unsigned char* input;
+ size_t length;
+ char* output_unambiguous;
+} HParserTestcase;
+
+typedef struct HCaseResult_ {
+ bool success;
+ union {
+ const char* actual_results; // on failure, filled in with the results of h_write_result_unamb
+ size_t parse_time; // on success, filled in with time for a single parse, in nsec
+ };
+} HCaseResult;
+
+typedef struct HBackendResults_ {
+ HParserBackend backend;
+ bool compile_success;
+ size_t n_testcases;
+ size_t failed_testcases; // actually a count...
+ HCaseResult *cases;
+} HBackendResults;
+
+typedef struct HBenchmarkResults_ {
+ size_t len;
+ HBackendResults *results;
+} HBenchmarkResults;
+""")
+
+# }}}
+# {{{ Arena functions
+_ffi.cdef("void* h_arena_malloc(HArena *arena, size_t count);")
+_ffi.cdef("void h_arena_free(HArena *arena, void* ptr);")
+# }}}
+# {{{ cdefs
+## The following section was generated by
+## $ perl ../desugar-header.pl <../../hammer.h |sed -e 's/.*/_ffi.cdef("&")/'
+_ffi.cdef("HParseResult* h_parse(const HParser* parser, const uint8_t* input, size_t length);")
+_ffi.cdef("HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t* input, size_t length);")
+_ffi.cdef("HParser* h_token(const uint8_t *str, const size_t len);")
+_ffi.cdef("HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len);")
+_ffi.cdef("HParser* h_ch(const uint8_t c);")
+_ffi.cdef("HParser* h_ch__m(HAllocator* mm__, const uint8_t c);")
+_ffi.cdef("HParser* h_ch_range(const uint8_t lower, const uint8_t upper);")
+_ffi.cdef("HParser* h_ch_range__m(HAllocator* mm__, const uint8_t lower, const uint8_t upper);")
+_ffi.cdef("HParser* h_int_range(const HParser *p, const int64_t lower, const int64_t upper);")
+_ffi.cdef("HParser* h_int_range__m(HAllocator* mm__, const HParser *p, const int64_t lower, const int64_t upper);")
+_ffi.cdef("HParser* h_bits(size_t len, bool sign);")
+_ffi.cdef("HParser* h_bits__m(HAllocator* mm__, size_t len, bool sign);")
+_ffi.cdef("HParser* h_int64(void);")
+_ffi.cdef("HParser* h_int64__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_int32(void);")
+_ffi.cdef("HParser* h_int32__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_int16(void);")
+_ffi.cdef("HParser* h_int16__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_int8(void);")
+_ffi.cdef("HParser* h_int8__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_uint64(void);")
+_ffi.cdef("HParser* h_uint64__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_uint32(void);")
+_ffi.cdef("HParser* h_uint32__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_uint16(void);")
+_ffi.cdef("HParser* h_uint16__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_uint8(void);")
+_ffi.cdef("HParser* h_uint8__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_whitespace(const HParser* p);")
+_ffi.cdef("HParser* h_whitespace__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_left(const HParser* p, const HParser* q);")
+_ffi.cdef("HParser* h_left__m(HAllocator* mm__, const HParser* p, const HParser* q);")
+_ffi.cdef("HParser* h_right(const HParser* p, const HParser* q);")
+_ffi.cdef("HParser* h_right__m(HAllocator* mm__, const HParser* p, const HParser* q);")
+_ffi.cdef("HParser* h_middle(const HParser* p, const HParser* x, const HParser* q);")
+_ffi.cdef("HParser* h_middle__m(HAllocator* mm__, const HParser* p, const HParser* x, const HParser* q);")
+_ffi.cdef("HParser* h_action(const HParser* p, const HAction a);")
+_ffi.cdef("HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a);")
+_ffi.cdef("HParser* h_in(const uint8_t *charset, size_t length);")
+_ffi.cdef("HParser* h_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);")
+_ffi.cdef("HParser* h_not_in(const uint8_t *charset, size_t length);")
+_ffi.cdef("HParser* h_not_in__m(HAllocator* mm__, const uint8_t *charset, size_t length);")
+_ffi.cdef("HParser* h_end_p(void);")
+_ffi.cdef("HParser* h_end_p__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_nothing_p(void);")
+_ffi.cdef("HParser* h_nothing_p__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_sequence(HParser* p, ...);")
+_ffi.cdef("HParser* h_sequence__m(HAllocator *mm__, HParser* p, ...);")
+_ffi.cdef("HParser* h_sequence__a(void* args);")
+_ffi.cdef("HParser* h_sequence__ma(HAllocator* mm__, void* args);")
+_ffi.cdef("HParser* h_choice(HParser* p, ...);")
+_ffi.cdef("HParser* h_choice__m(HAllocator *mm__, HParser* p, ...);")
+_ffi.cdef("HParser* h_choice__a(void* args);")
+_ffi.cdef("HParser* h_choice__ma(HAllocator* mm__, void* args);")
+_ffi.cdef("HParser* h_butnot(const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_butnot__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_difference(const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_difference__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_xor(const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_xor__m(HAllocator* mm__, const HParser* p1, const HParser* p2);")
+_ffi.cdef("HParser* h_many(const HParser* p);")
+_ffi.cdef("HParser* h_many__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_many1(const HParser* p);")
+_ffi.cdef("HParser* h_many1__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_repeat_n(const HParser* p, const size_t n);")
+_ffi.cdef("HParser* h_repeat_n__m(HAllocator* mm__, const HParser* p, const size_t n);")
+_ffi.cdef("HParser* h_optional(const HParser* p);")
+_ffi.cdef("HParser* h_optional__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_ignore(const HParser* p);")
+_ffi.cdef("HParser* h_ignore__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_sepBy(const HParser* p, const HParser* sep);")
+_ffi.cdef("HParser* h_sepBy__m(HAllocator* mm__, const HParser* p, const HParser* sep);")
+_ffi.cdef("HParser* h_sepBy1(const HParser* p, const HParser* sep);")
+_ffi.cdef("HParser* h_sepBy1__m(HAllocator* mm__, const HParser* p, const HParser* sep);")
+_ffi.cdef("HParser* h_epsilon_p(void);")
+_ffi.cdef("HParser* h_epsilon_p__m(HAllocator* mm__);")
+_ffi.cdef("HParser* h_length_value(const HParser* length, const HParser* value);")
+_ffi.cdef("HParser* h_length_value__m(HAllocator* mm__, const HParser* length, const HParser* value);")
+_ffi.cdef("HParser* h_attr_bool(const HParser* p, HPredicate pred);")
+_ffi.cdef("HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred);")
+_ffi.cdef("HParser* h_and(const HParser* p);")
+_ffi.cdef("HParser* h_and__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_not(const HParser* p);")
+_ffi.cdef("HParser* h_not__m(HAllocator* mm__, const HParser* p);")
+_ffi.cdef("HParser* h_indirect(void);")
+_ffi.cdef("HParser* h_indirect__m(HAllocator* mm__);")
+_ffi.cdef("void h_bind_indirect(HParser* indirect, const HParser* inner);")
+_ffi.cdef("void h_bind_indirect__m(HAllocator* mm__, HParser* indirect, const HParser* inner);")
+_ffi.cdef("void h_parse_result_free(HParseResult *result);")
+_ffi.cdef("void h_parse_result_free__m(HAllocator* mm__, HParseResult *result);")
+_ffi.cdef("void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);")
+_ffi.cdef("int h_compile(HParser* parser, HParserBackend backend, const void* params);")
+_ffi.cdef("int h_compile__m(HAllocator* mm__, HParser* parser, HParserBackend backend, const void* params);")
+_ffi.cdef("HBenchmarkResults * h_benchmark(HParser* parser, HParserTestcase* testcases);")
+_ffi.cdef("HBenchmarkResults * h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTestcase* testcases);")
+
+_lib = _ffi.verify("#include <hammer/hammer.h>",
+ libraries=['hammer'])
+
+_lib.TT_PYTHON = _lib.TT_USER # TODO: Use the token type allocator from #45
+# }}}
+class _DynamicScopeHolder(threading.local):
+ """A dynamically-scoped holder of python objects, which may or may not
+ otherwise appear in the object graph. Intended for use with CFFI """
+ def __init__(self):
+ self._ctxstack = []
+ def __enter__(self):
+ self._ctxstack.append([])
+ def __exit__(self, exc_type, exc_value, traceback):
+ self._ctxstack.pop()
+ return False
+ def stash(self, *objs):
+ if len(self._ctxstack) < 1:
+ raise Exception("Not in any dynamic scope")
+ for obj in objs:
+ self._ctxstack[-1].append(obj)
+def _fromHParsedToken(cobj):
+ # TODO: Free the toplevel parser
+ tt = cobj.token_type
+
+ if cobj.token_type == _lib.TT_BYTES:
+ return _ffi.buffer(cobj.bytes.token, cobj.bytes.len)[:]
+ elif cobj.token_type == _lib.TT_ERR:
+ # I have no idea what this is for
+ pass
+ elif cobj.token_type == _lib.TT_NONE:
+ return None
+ elif cobj.token_type == _lib.TT_SEQUENCE:
+ return [_fromHParsedToken(cobj.seq.elements[i])
+ for i in range(cobj.seq.used)]
+ elif cobj.token_type == _lib.TT_SINT:
+ return cobj.sint
+ elif cobj.token_type == _lib.TT_UINT:
+ return cobj.uint
+ elif cobj.token_type == _lib.TT_PYTHON:
+ return _ffi.from_handle(cobj.user)
+
+_parser_result_holder = _DynamicScopeHolder()
+def _toHParsedToken(arena, pyobj):
+ if pyobj is None:
+ return _ffi.NULL
+ cobj = _ffi.new_handle(pyobj)
+ _parser_result_holder.stash(cobj)
+
+ hpt = _ffi.cast("HParsedToken*", _lib.h_arena_malloc(arena, _ffi.sizeof("HParsedToken")))
+ hpt.token_type = _lib.TT_PYTHON
+ hpt.user = cobj
+ hpt.bit_offset = chr(127)
+ hpt.index = 0
+ return hpt
+
+def _fromParseResult(cobj):
+ ret = _fromHParsedToken(cobj.ast)
+ _lib.h_parse_result_free(cobj)
+ return ret
+
+def _to_haction(fn):
+ """Turn a function that transforms a parsed value into an HAction"""
+ def action(parse_result):
+ res = _toHParsedToken(parse_result.arena, fn(_fromParseResult(parse_result)))
+ if res != _ffi.NULL and parse_result.ast != _ffi.NULL:
+ res.index = parse_result.ast.index
+ res.bit_offset = parse_result.ast.bit_offset
+ return res
+ return _ffi.callback("HParsedToken*(HParseResult*)", action)
+
+def _to_hpredicate(fn):
+ """Turn a function that transforms a parsed value into an HAction"""
+ def predicate(parse_result):
+ res = fn(_fromParseResult(parse_result))
+ # TODO: Handle exceptions; parse should fail.
+ if type(res) != bool:
+ raise TypeError("Predicates should return a bool")
+ return res
+ return _ffi.callback("bool(HParseResult*)", predicate)
+
+class Parser(object):
+ # TODO: Map these to individually garbage-collected blocks of
+ # memory. Perhaps with an arena allocator with block size of 1?
+ # There has to be something more efficient than that, though.
+
+ # TODO: How do we handle encodings? By default, we're using UTF-8
+ def __init__(self, internal, deps):
+ """Create a new parser from an FFI object. Not for user code"""
+ self._parser = internal
+ self._deps = deps
+
+ def parse(self, string):
+ with _parser_result_holder:
+ pres = _lib.h_parse(self._parser, string, len(string))
+ if pres:
+ return _fromParseResult(pres)
+ else:
+ return None
+
+ def __mul__(self, count):
+ return repeat_n(self, count)
+
+
+
+class IndirectParser(Parser):
+ def bind(self, inner):
+ _lib.h_bind_indirect(self._parser, inner._parser)
+ self._deps = (inner,)
+
+class BitsParser(Parser):
+ pass
+
+def token(token):
+ # TODO: Does not clone argument.
+ if isinstance(token, unicode):
+ token = token.encode("utf-8")
+ return Parser(_lib.h_token(token, len(token)), ())
+
+def ch(char):
+ """Returns either a token or an int, depending on the type of the
+ argument"""
+ if isinstance(char, int):
+ return Parser(_lib.h_ch(char), ())
+ else:
+ return token(char)
+
+def ch_range(chr1, chr2):
+ if not isinstance(chr1, str) or not isinstance(chr2, str):
+ raise TypeError("ch_range can't handle unicode")
+ def my_action(pr):
+ # print "In action: ", pr
+ return pr
+ return action(Parser(_lib.h_ch_range(ord(chr1), ord(chr2)), ()), my_action)
+
+def int_range(parser, i1, i2):
+ if type(parser) != BitsParser:
+ raise TypeError("int_range is only valid when used with a bits parser")
+ return Parser(_lib.h_int_range(parser._parser, i1, i2), (parser,))
+
+def bits(length, signedp):
+ return BitsParser(_lib.h_bits(length, signedp), ())
+
+def int64(): return bits(64, True)
+def int32(): return bits(32, True)
+def int16(): return bits(16, True)
+def int8 (): return bits(8, True)
+def uint64(): return bits(64, False)
+def uint32(): return bits(32, False)
+def uint16(): return bits(16, False)
+def uint8 (): return bits(8, False)
+
+def whitespace(p):
+ return Parser(_lib.h_whitespace(p._parser), (p,))
+def left(p1, p2):
+ return Parser(_lib.h_left(p1._parser, p2._parser), (p1, p2))
+def right(p1, p2):
+ return Parser(_lib.h_right(p1._parser, p2._parser), (p1, p2))
+def middle(p1, p2, p3):
+ return Parser(_lib.h_middle(p1._parser, p2._parser, p3._parser), (p1, p2, p3))
+def action(parser, action):
+ caction = _to_haction(action)
+ return Parser(_lib.h_action(parser._parser, caction), (parser, caction))
+
+def in_(charset):
+ if not isinstance(charset, str):
+ # TODO/Python3: change str to bytes
+ raise TypeError("in_ can't deal with unicode")
+ return Parser(_lib.h_in(charset, len(charset)), ())
+def not_in(charset):
+ if not isinstance(charset, str):
+ # TODO/Python3: change str to bytes
+ raise TypeError("in_ can't deal with unicode")
+ return Parser(_lib.h_not_in(charset, len(charset)), ())
+def end_p():
+ return Parser(_lib.h_end_p(), ())
+def nothing_p():
+ return Parser(_lib.h_nothing_p(), ())
+def sequence(*parsers):
+ plist = [p._parser for p in parsers]
+ plist.append(_ffi.NULL)
+ return Parser(_lib.h_sequence(*plist), (plist,))
+def choice(*parsers):
+ plist = [p._parser for p in parsers]
+ plist.append(_ffi.NULL)
+ return Parser(_lib.h_choice(*plist), (plist,))
+def butnot(p1, p2):
+ return Parser(_lib.h_butnot(p1._parser, p2._parser), (p1, p2))
+def difference(p1, p2):
+ return Parser(_lib.h_difference(p1._parser, p2._parser), (p1, p2))
+def xor(p1, p2):
+ return Parser(_lib.h_xor(p1._parser, p2._parser), (p1, p2))
+def many(p1):
+ return Parser(_lib.h_many(p1._parser), (p1,))
+def many1(p1):
+ return Parser(_lib.h_many1(p1._parser), (p1,))
+def repeat_n(p1, n):
+ return Parser(_lib.h_repeat_n(p1._parser, n), (p1,))
+def optional(p1):
+ return Parser(_lib.h_optional(p1._parser), (p1,))
+def ignore(p1):
+ return Parser(_lib.h_ignore(p1._parser), (p1,))
+def sepBy(p, sep):
+ return Parser(_lib.h_sepBy(p._parser, sep._parser), (p, sep))
+def sepBy1(p, sep):
+ return Parser(_lib.h_sepBy1(p._parser, sep._parser), (p, sep))
+def epsilon_p():
+ return Parser(_lib.h_epsilon_p(), ())
+def length_value(p_len, p_value):
+ return Parser(_lib.h_length_value(p_len._parser, p_value._parser), (p_len, p_value))
+def attr_bool(parser, predicate):
+ cpredicate = _to_hpredicate(predicate)
+ return Parser(_lib.h_attr_bool(parser._parser, cpredicate), (parser, cpredicate))
+def and_(parser):
+ return Parser(_lib.h_and(parser._parser), (parser,))
+def not_(parser):
+ return Parser(_lib.h_not(parser._parser), (parser,))
+def indirect():
+ return IndirectParser(_lib.h_indirect(), ())
+def bind_indirect(indirect, inner):
+ indirect.bind(inner)
+
+def parse(parser):
+ return parser.parse()
+
+# Unfortunately, "in", "and", and "not" are keywords. This makes them
+# show up in the module namespace for the use of automated tools. Do
+# not attempt to use them by hand; only use the mangled forms (with
+# the '_')
+sys.modules[__name__].__dict__["in"] = in_
+sys.modules[__name__].__dict__["and"] = and_
+sys.modules[__name__].__dict__["not"] = not_
+
+def run_test():
+ p_test = sepBy1(choice(ch('1'),
+ ch('2'),
+ ch('3')),
+ ch(','))
+ return p_test.parse("1,2,3")
+
+# {{{ Automatic parser construction... python specific
+
+# TODO: Implement Parsable metaclass, which requires the existence of
+# a "parse" method.
+
+# This is expected to be extended by user code. As a general rule,
+# only provide auto-parsers for your own types.
+AUTO_PARSERS = {
+ str: token,
+ unicode: token,
+}
+
+def _auto_seq(lst):
+ return sequence(*(auto_1(p, default_method=_auto_choice)
+ for p in lst))
+
+def _auto_choice(lst):
+ return choice(*(auto_1(p, default_method=_auto_seq)
+ for p in lst))
+
+def auto_1(arg, default_method=_auto_choice):
+ if isinstance(arg, Parser):
+ return arg
+ elif type(arg) in AUTO_PARSERS:
+ return AUTO_PARSERS[type(arg)](arg)
+ else:
+ return default_method(arg)
+
+def auto(*args):
+ return auto_1(args, default_method=_auto_choice)
+
+# }}}
diff --git a/src/bindings/python/hammer_tests.py b/src/bindings/python/hammer_tests.py
new file mode 100644
index 0000000000000000000000000000000000000000..45a63f49ffe42384a1f3025b0c02eb85264462e5
--- /dev/null
+++ b/src/bindings/python/hammer_tests.py
@@ -0,0 +1,509 @@
+import unittest
+import hammer as h
+
+class TestTokenParser(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.token("95\xa2")
+ def test_success(self):
+ self.assertEqual(self.parser.parse("95\xa2"), "95\xa2")
+ def test_partial_fails(self):
+ self.assertEqual(self.parser.parse("95"), None)
+
+class TestChParser(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser_int = h.ch(0xa2)
+ cls.parser_chr = h.ch("\xa2")
+ def test_success(self):
+ self.assertEqual(self.parser_int.parse("\xa2"), 0xa2)
+ self.assertEqual(self.parser_chr.parse("\xa2"), "\xa2")
+ def test_failure(self):
+ self.assertEqual(self.parser_int.parse("\xa3"), None)
+ self.assertEqual(self.parser_chr.parse("\xa3"), None)
+
+class TestChRange(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.ch_range("a", "c")
+ def test_success(self):
+ self.assertEqual(self.parser.parse("b"), "b")
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("d"), None)
+
+class TestInt64(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.int64()
+ def test_success(self):
+ self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00\x00"), -0x200000000)
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("\xff\xff\xff\xfe\x00\x00\x00"), None)
+
+class TestInt32(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.int32()
+ def test_success(self):
+ self.assertEqual(self.parser.parse("\xff\xfe\x00\x00"), -0x20000)
+ self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000)
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("\xff\xfe\x00"), None)
+ self.assertEqual(self.parser.parse("\x00\x02\x00"), None)
+
+class TestInt16(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.int16()
+ def test_success(self):
+ self.assertEqual(self.parser.parse("\xfe\x00"), -0x200)
+ self.assertEqual(self.parser.parse("\x02\x00"), 0x200)
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("\xfe"), None)
+ self.assertEqual(self.parser.parse("\x02"), None)
+
+class TestInt8(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.int8()
+ def test_success(self):
+ self.assertEqual(self.parser.parse("\x88"), -0x78)
+ def test_failure(self):
+ self.assertEqual(self.parser.parse(""), None)
+
+class TestUint64(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.uint64()
+ def test_success(self):
+ self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00\x00"), 0x200000000)
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("\x00\x00\x00\x02\x00\x00\x00"), None)
+
+class TestUint32(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.uint32()
+ def test_success(self):
+ self.assertEqual(self.parser.parse("\x00\x02\x00\x00"), 0x20000)
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("\x00\x02\x00"), None)
+
+class TestUint16(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.uint16()
+ def test_success(self):
+ self.assertEqual(self.parser.parse("\x02\x00"), 0x200)
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("\x02"), None)
+
+class TestUint8(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.uint8()
+ def test_success(self):
+ self.assertEqual(self.parser.parse("\x78"), 0x78)
+ def test_failure(self):
+ self.assertEqual(self.parser.parse(""), None)
+
+class TestIntRange(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.int_range(h.uint8(), 3, 10)
+ def test_success(self):
+ self.assertEqual(self.parser.parse("\x05"), 5)
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("\x0b"), None)
+
+class TestWhitespace(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.whitespace(h.ch("a"))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("a"), "a")
+ self.assertEqual(self.parser.parse(" a"), "a")
+ self.assertEqual(self.parser.parse(" a"), "a")
+ self.assertEqual(self.parser.parse("\ta"), "a")
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("_a"), None)
+
+class TestWhitespaceEnd(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.whitespace(h.end_p())
+ def test_success(self):
+ self.assertEqual(self.parser.parse(""), None) # empty string
+ self.assertEqual(self.parser.parse(" "), None) # empty string
+ def test_failure(self):
+ self.assertEqual(self.parser.parse(" x"), None)
+
+class TestLeft(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.left(h.ch("a"), h.ch(" "))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("a "), "a")
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("a"), None)
+ self.assertEqual(self.parser.parse(" "), None)
+ self.assertEqual(self.parser.parse("ab"), None)
+
+class TestRight(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.right(h.ch(" "), h.ch("a"))
+ def test_success(self):
+ self.assertEqual(self.parser.parse(" a"), "a")
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("a"), None)
+ self.assertEqual(self.parser.parse(" "), None)
+ self.assertEqual(self.parser.parse("ba"), None)
+
+class TestMiddle(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.middle(h.ch(" "), h.ch("a"), h.ch(" "))
+ def test_success(self):
+ self.assertEqual(self.parser.parse(" a "), "a")
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("a"), None)
+ self.assertEqual(self.parser.parse(" "), None)
+ self.assertEqual(self.parser.parse(" a"), None)
+ self.assertEqual(self.parser.parse("a "), None)
+ self.assertEqual(self.parser.parse(" b "), None)
+ self.assertEqual(self.parser.parse("ba "), None)
+ self.assertEqual(self.parser.parse(" ab"), None)
+
+class TestAction(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.action(h.sequence(h.choice(h.ch("a"), h.ch("A")),
+ h.choice(h.ch("b"), h.ch("B"))),
+ lambda x: [y.upper() for y in x])
+ def test_success(self):
+ self.assertEqual(self.parser.parse("ab"), ["A", "B"])
+ self.assertEqual(self.parser.parse("AB"), ["A", "B"])
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("XX"), None)
+
+class TestIn(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.in_("abc")
+ def test_success(self):
+ self.assertEqual(self.parser.parse("b"), "b")
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("d"), None)
+
+class TestNotIn(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.not_in("abc")
+ def test_success(self):
+ self.assertEqual(self.parser.parse("d"), "d")
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("a"), None)
+
+class TestEndP(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.sequence(h.ch("a"), h.end_p())
+ def test_success(self):
+ self.assertEqual(self.parser.parse("a"), ("a",))
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("aa"), None)
+
+class TestNothingP(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.nothing_p()
+ def test_success(self):
+ pass
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("a"), None)
+
+class TestSequence(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.sequence(h.ch("a"), h.ch("b"))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("ab"), ('a','b'))
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("a"), None)
+ self.assertEqual(self.parser.parse("b"), None)
+
+class TestSequenceWhitespace(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.sequence(h.ch("a"), h.whitespace(h.ch("b")))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("ab"), ('a','b'))
+ self.assertEqual(self.parser.parse("a b"), ('a','b'))
+ self.assertEqual(self.parser.parse("a b"), ('a','b'))
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("a c"), None)
+
+class TestChoice(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.choice(h.ch("a"), h.ch("b"))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("a"), "a")
+ self.assertEqual(self.parser.parse("b"), "b")
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("c"), None)
+
+class TestButNot(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.butnot(h.ch("a"), h.token("ab"))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("a"), "a")
+ self.assertEqual(self.parser.parse("aa"), "a")
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("ab"), None)
+
+class TestButNotRange(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.butnot(h.ch_range("0", "9"), h.ch("6"))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("4"), "4")
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("6"), None)
+
+class TestDifference(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.difference(h.token("ab"), h.ch("a"))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("ab"), "ab")
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("a"), None)
+
+class TestXor(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.xor(h.ch_range("0", "6"), h.ch_range("5", "9"))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("0"), "0")
+ self.assertEqual(self.parser.parse("9"), "9")
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("5"), None)
+ self.assertEqual(self.parser.parse("a"), None)
+
+class TestMany(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.many(h.choice(h.ch("a"), h.ch("b")))
+ def test_success(self):
+ self.assertEqual(self.parser.parse(""), ())
+ self.assertEqual(self.parser.parse("a"), ('a',))
+ self.assertEqual(self.parser.parse("b"), ('b',))
+ self.assertEqual(self.parser.parse("aabbaba"), ('a','a','b','b','a','b','a'))
+ def test_failure(self):
+ pass
+
+class TestMany1(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.many1(h.choice(h.ch("a"), h.ch("b")))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("a"), ("a",))
+ self.assertEqual(self.parser.parse("b"), ("b",))
+ self.assertEqual(self.parser.parse("aabbaba"), ("a", "a", "b", "b", "a", "b", "a"))
+ def test_failure(self):
+ self.assertEqual(self.parser.parse(""), None)
+ self.assertEqual(self.parser.parse("daabbabadef"), None)
+
+class TestRepeatN(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.repeat_n(h.choice(h.ch("a"), h.ch("b")), 2)
+ def test_success(self):
+ self.assertEqual(self.parser.parse("abdef"), ('a', 'b'))
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("adef"), None)
+ self.assertEqual(self.parser.parse("dabdef"), None)
+
+class TestOptional(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.sequence(h.ch("a"), h.optional(h.choice(h.ch("b"), h.ch("c"))), h.ch("d"))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("abd"), ('a','b','d'))
+ self.assertEqual(self.parser.parse("acd"), ('a','c','d'))
+ self.assertEqual(self.parser.parse("ad"), ('a',h.Placeholder(), 'd'))
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("aed"), None)
+ self.assertEqual(self.parser.parse("ab"), None)
+ self.assertEqual(self.parser.parse("ac"), None)
+
+class TestIgnore(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.sequence(h.ch("a"), h.ignore(h.ch("b")), h.ch("c"))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("abc"), ("a","c"))
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("ac"), None)
+
+class TestSepBy(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.sepBy(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(","))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("1,2,3"), ('1','2','3'))
+ self.assertEqual(self.parser.parse("1,3,2"), ('1','3','2'))
+ self.assertEqual(self.parser.parse("1,3"), ('1','3'))
+ self.assertEqual(self.parser.parse("3"), ('3',))
+ self.assertEqual(self.parser.parse(""), ())
+ def test_failure(self):
+ pass
+
+class TestSepBy1(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.sepBy1(h.choice(h.ch("1"), h.ch("2"), h.ch("3")), h.ch(","))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("1,2,3"), ('1','2','3'))
+ self.assertEqual(self.parser.parse("1,3,2"), ('1','3','2'))
+ self.assertEqual(self.parser.parse("1,3"), ('1','3'))
+ self.assertEqual(self.parser.parse("3"), ('3',))
+ def test_failure(self):
+ self.assertEqual(self.parser.parse(""), None)
+
+class TestEpsilonP1(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.sequence(h.ch("a"), h.epsilon_p(), h.ch("b"))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("ab"), ("a", "b"))
+ def test_failure(self):
+ pass
+
+class TestEpsilonP2(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.sequence(h.epsilon_p(), h.ch("a"))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("a"), ("a",))
+ def test_failure(self):
+ pass
+
+class TestEpsilonP3(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.sequence(h.ch("a"), h.epsilon_p())
+ def test_success(self):
+ self.assertEqual(self.parser.parse("a"), ("a",))
+ def test_failure(self):
+ pass
+
+class TestAttrBool(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.attr_bool(h.many1(h.choice(h.ch("a"), h.ch("b"))),
+ lambda x: x[0] == x[1])
+ def test_success(self):
+ self.assertEqual(self.parser.parse("aa"), ("a", "a"))
+ self.assertEqual(self.parser.parse("bb"), ("b", "b"))
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("ab"), None)
+
+class TestAnd1(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("0"))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("0"), ("0",))
+ def test_failure(self):
+ pass
+
+class TestAnd2(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.sequence(h.and_(h.ch("0")), h.ch("1"))
+ def test_success(self):
+ pass
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("0"), None)
+
+class TestAnd3(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.sequence(h.ch("1"), h.and_(h.ch("2")))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("12"), ('1',))
+ def test_failure(self):
+ pass
+
+class TestNot1(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.sequence(h.ch("a"),
+ h.choice(h.ch("+"), h.token("++")),
+ h.ch("b"))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("a+b"), ("a", "+", "b"))
+ def test_failure(self):
+ self.assertEqual(self.parser.parse("a++b"), None)
+
+class TestNot2(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = h.sequence(h.ch("a"), h.choice(h.sequence(h.ch("+"), h.not_(h.ch("+"))),
+ h.token("++")),
+ h.ch("b"))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("a+b"), ('a', ('+',), 'b'))
+ self.assertEqual(self.parser.parse("a++b"), ('a', "++", 'b'))
+ def test_failure(self):
+ pass
+
+# ### this is commented out for packrat in C ...
+# #class TestLeftrec(unittest.TestCase):
+# # @classmethod
+# # def setUpClass(cls):
+# # cls.parser = h.indirect()
+# # a = h.ch("a")
+# # h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, a), a))
+# # def test_success(self):
+# # self.assertEqual(self.parser.parse("a"), "a")
+# # self.assertEqual(self.parser.parse("aa"), ["a", "a"])
+# # self.assertEqual(self.parser.parse("aaa"), ["a", "a", "a"])
+# # def test_failure(self):
+# # pass
+
+
+class TestRightrec(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ #raise unittest.SkipTest("Bind doesn't work right now")
+ cls.parser = h.indirect()
+ a = h.ch("a")
+ cls.parser.bind(h.choice(h.sequence(a, cls.parser),
+ h.epsilon_p()))
+ def test_success(self):
+ self.assertEqual(self.parser.parse("a"), ('a',))
+ self.assertEqual(self.parser.parse("aa"), ('a', ('a',)))
+ self.assertEqual(self.parser.parse("aaa"), ('a', ('a', ('a',))))
+ def test_failure(self):
+ pass
+
+# ### this is just for GLR
+# #class TestAmbiguous(unittest.TestCase):
+# # @classmethod
+# # def setUpClass(cls):
+# # cls.parser = h.indirect()
+# # d = h.ch("d")
+# # p = h.ch("+")
+# # h.bind_indirect(cls.parser, h.choice(h.sequence(cls.parser, p, cls.parser), d))
+# # # this is supposed to be flattened
+# # def test_success(self):
+# # self.assertEqual(self.parser.parse("d"), ["d"])
+# # self.assertEqual(self.parser.parse("d+d"), ["d", "+", "d"])
+# # self.assertEqual(self.parser.parse("d+d+d"), ["d", "+", "d", "+", "d"])
+# # def test_failure(self):
+# # self.assertEqual(self.parser.parse("d+"), None)
diff --git a/src/bindings/swig/hammer.i b/src/bindings/swig/hammer.i
new file mode 100644
index 0000000000000000000000000000000000000000..5ac8c3767b2c639e2a5444de532ca1e59c8a450a
--- /dev/null
+++ b/src/bindings/swig/hammer.i
@@ -0,0 +1,343 @@
+%module hammer
+
+%nodefaultctor;
+
+%include "stdint.i"
+
+#if defined(SWIGPYTHON)
+%ignore HCountedArray_;
+%apply (char *STRING, size_t LENGTH) {(uint8_t* str, size_t len)}
+%apply (uint8_t* str, size_t len) {(const uint8_t* input, size_t length)}
+%apply (uint8_t* str, size_t len) {(const uint8_t* str, const size_t len)}
+%apply (uint8_t* str, size_t len) {(const uint8_t* charset, size_t length)}
+
+
+%rename("_%s") "";
+// %rename(_h_ch) h_ch;
+
+%inline {
+ static PyObject *_helper_Placeholder = NULL, *_helper_ParseError = NULL;
+
+ static void register_helpers(PyObject* parse_error, PyObject *placeholder) {
+ _helper_ParseError = parse_error;
+ _helper_Placeholder = placeholder;
+ }
+ }
+
+%pythoncode %{
+ class Placeholder(object):
+ """The python equivalent of TT_NONE"""
+ def __str__(self):
+ return "Placeholder"
+ def __repr__(self):
+ return "Placeholder"
+ def __eq__(self, other):
+ return type(self) == type(other)
+ class ParseError(Exception):
+ """The parse failed; the message may have more information"""
+ pass
+
+ _hammer._register_helpers(ParseError,
+ Placeholder)
+ %}
+
+%typemap(in) void*[] {
+ if (PyList_Check($input)) {
+ Py_INCREF($input);
+ int size = PyList_Size($input);
+ int i = 0;
+ int res = 0;
+ $1 = (void**)malloc((size+1)*sizeof(HParser*));
+ for (i=0; i<size; i++) {
+ PyObject *o = PyList_GetItem($input, i);
+ res = SWIG_ConvertPtr(o, &($1[i]), SWIGTYPE_p_HParser_, 0 | 0);
+ if (!SWIG_IsOK(res)) {
+ SWIG_exception_fail(SWIG_ArgError(res), "that wasn't an HParser" );
+ }
+ }
+ $1[size] = NULL;
+ } else {
+ PyErr_SetString(PyExc_TypeError, "__a functions take lists of parsers as their argument");
+ return NULL;
+ }
+ }
+%typemap(in) uint8_t {
+ if (PyInt_Check($input)) {
+ $1 = PyInt_AsLong($input);
+ }
+ else if (!PyString_Check($input)) {
+ PyErr_SetString(PyExc_ValueError, "Expecting a string");
+ return NULL;
+ } else {
+ $1 = *(uint8_t*)PyString_AsString($input);
+ }
+ }
+%typemap(out) HBytes* {
+ $result = PyString_FromStringAndSize((char*)$1->token, $1->len);
+ }
+%typemap(out) struct HCountedArray_* {
+ int i;
+ $result = PyList_New($1->used);
+ for (i=0; i<$1->used; i++) {
+ HParsedToken *t = $1->elements[i];
+ PyObject *o = SWIG_NewPointerObj(SWIG_as_voidptr(t), SWIGTYPE_p_HParsedToken_, 0 | 0);
+ PyList_SetItem($result, i, o);
+ }
+ }
+%typemap(out) struct HParseResult_* {
+ if ($1 == NULL) {
+ // TODO: raise parse failure
+ Py_INCREF(Py_None);
+ $result = Py_None;
+ } else {
+ $result = hpt_to_python($1->ast);
+ }
+ }
+%typemap(newfree) struct HParseResult_* {
+ h_parse_result_free($input);
+ }
+%inline %{
+ static int h_tt_python;
+ %}
+%init %{
+ h_tt_python = h_allocate_token_type("com.upstandinghackers.hammer.python");
+ %}
+
+
+
+
+%typemap(in) (HPredicate pred, void* user_data) {
+ Py_INCREF($input);
+ $2 = $input;
+ $1 = call_predicate;
+ }
+
+%typemap(in) (const HAction a, void* user_data) {
+ Py_INCREF($input);
+ $2 = $input;
+ $1 = call_action;
+ }
+
+%inline %{
+
+ struct HParsedToken_;
+ struct HParseResult_;
+ static PyObject* hpt_to_python(const struct HParsedToken_ *token);
+
+ static struct HParsedToken_* call_action(const struct HParseResult_ *p, void* user_data);
+ static int call_predicate(const struct HParseResult_ *p, void* user_data);
+ %}
+#else
+ #warning no uint8_t* typemaps defined
+#endif
+
+ // All the include paths are relative to the build, i.e., ../../. If you need to build these manually (i.e., not with scons), keep that in mind.
+%{
+#include "allocator.h"
+#include "hammer.h"
+#include "internal.h"
+#include "glue.h"
+%}
+%include "allocator.h"
+%include "hammer.h"
+
+%extend HArena_ {
+ ~HArena_() {
+ h_delete_arena($self);
+ }
+ };
+%extend HParseResult_ {
+ ~HParseResult_() {
+ h_parse_result_free($self);
+ }
+};
+
+%newobject h_parse;
+%delobject h_parse_result_free;
+%newobject h_new_arena;
+%delobject h_delete_arena;
+
+#ifdef SWIGPYTHON
+%inline {
+ static PyObject* hpt_to_python(const HParsedToken *token) {
+ // Caller holds a reference to returned object
+ PyObject *ret;
+ if (token == NULL) {
+ Py_RETURN_NONE;
+ }
+ switch (token->token_type) {
+ case TT_NONE:
+ return PyObject_CallFunctionObjArgs(_helper_Placeholder, NULL);
+ break;
+ case TT_BYTES:
+ return PyString_FromStringAndSize((char*)token->token_data.bytes.token, token->token_data.bytes.len);
+ case TT_SINT:
+ // TODO: return PyINT if appropriate
+ return PyLong_FromLong(token->token_data.sint);
+ case TT_UINT:
+ // TODO: return PyINT if appropriate
+ return PyLong_FromUnsignedLong(token->token_data.uint);
+ case TT_SEQUENCE:
+ ret = PyTuple_New(token->token_data.seq->used);
+ for (int i = 0; i < token->token_data.seq->used; i++) {
+ PyTuple_SET_ITEM(ret, i, hpt_to_python(token->token_data.seq->elements[i]));
+ }
+ return ret;
+ default:
+ if (token->token_type == h_tt_python) {
+ ret = (PyObject*)token->token_data.user;
+ Py_INCREF(ret);
+ return ret;
+ } else {
+ return SWIG_NewPointerObj((void*)token, SWIGTYPE_p_HParsedToken_, 0 | 0);
+ // TODO: support registry
+ }
+
+ }
+ }
+ static struct HParsedToken_* call_action(const struct HParseResult_ *p, void* user_data) {
+ PyObject *callable = user_data;
+ PyObject *ret = PyObject_CallFunctionObjArgs(callable,
+ hpt_to_python(p->ast),
+ NULL);
+ if (ret == NULL) {
+ PyErr_Print();
+ assert(ret != NULL);
+ }
+ // TODO: add reference to ret to parse-local data
+ // For now, just hold onto reference
+ HParsedToken *tok = h_make(p->arena, h_tt_python, ret);
+ return tok;
+ }
+
+ static int call_predicate(const struct HParseResult_ *p, void* user_data) {
+ PyObject *callable = user_data;
+ PyObject *ret = PyObject_CallFunctionObjArgs(callable,
+ hpt_to_python(p->ast),
+ NULL);
+ int rret = 0;
+ if (ret == NULL) {
+ // TODO: throw exception
+ PyErr_Print();
+ assert(ret != NULL);
+ }
+ // TODO: add reference to ret to parse-local data
+ rret = PyObject_IsTrue(ret);
+ Py_DECREF(ret);
+ return rret;
+ }
+
+ }
+
+%rename("%s") "";
+
+%extend HParser_ {
+ HParseResult* parse(const uint8_t* input, size_t length) {
+ return h_parse($self, input, length);
+ }
+ bool compile(HParserBackend backend) {
+ return h_compile($self, backend, NULL) == 0;
+ }
+ PyObject* __dir__() {
+ PyObject* ret = PyList_New(2);
+ PyList_SET_ITEM(ret, 0, PyString_FromString("parse"));
+ PyList_SET_ITEM(ret, 1, PyString_FromString("compile"));
+ return ret;
+ }
+}
+
+%pythoncode %{
+
+def action(p, act):
+ return _h_action(p, act)
+def attr_bool(p, pred):
+ return _h_attr_bool(p, pred)
+
+def ch(ch):
+ if isinstance(ch, str) or isinstance(ch, unicode):
+ return token(ch)
+ else:
+ return _h_ch(ch)
+
+def ch_range(c1, c2):
+ dostr = isinstance(c1, str)
+ dostr2 = isinstance(c2, str)
+ if isinstance(c1, unicode) or isinstance(c2, unicode):
+ raise TypeError("ch_range only works on bytes")
+ if dostr != dostr2:
+ raise TypeError("Both arguments to ch_range must be the same type")
+ if dostr:
+ return action(_h_ch_range(c1, c2), chr)
+ else:
+ return _h_ch_range(c1, c2)
+def epsilon_p(): return _h_epsilon_p()
+def end_p():
+ return _h_end_p()
+def in_(charset):
+ return action(_h_in(charset), chr)
+def not_in(charset):
+ return action(_h_not_in(charset), chr)
+def not_(p): return _h_not(p)
+def int_range(p, i1, i2):
+ return _h_int_range(p, i1, i2)
+def token(string):
+ return _h_token(string)
+def whitespace(p):
+ return _h_whitespace(p)
+def xor(p1, p2):
+ return _h_xor(p1, p2)
+def butnot(p1, p2):
+ return _h_butnot(p1, p2)
+def and_(p1):
+ return _h_and(p1)
+def difference(p1, p2):
+ return _h_difference(p1, p2)
+
+def sepBy(p, sep): return _h_sepBy(p, sep)
+def sepBy1(p, sep): return _h_sepBy1(p, sep)
+def many(p): return _h_many(p)
+def many1(p): return _h_many1(p)
+def repeat_n(p, n): return _h_repeat_n(p, n)
+def choice(*args): return _h_choice__a(list(args))
+def sequence(*args): return _h_sequence__a(list(args))
+
+def optional(p): return _h_optional(p)
+def nothing_p(): return _h_nothing_p()
+def ignore(p): return _h_ignore(p)
+
+def left(p1, p2): return _h_left(p1, p2)
+def middle(p1, p2, p3): return _h_middle(p1, p2, p3)
+def right(p1, p2): return _h_right(p1, p2)
+
+
+class HIndirectParser(_HParser_):
+ def __init__(self):
+ # Shoves the guts of an _HParser_ into a HIndirectParser.
+ tret = _h_indirect()
+ self.__dict__.clear()
+ self.__dict__.update(tret.__dict__)
+
+ def __dir__(self):
+ return super(HIndirectParser, self).__dir__() + ['bind']
+ def bind(self, parser):
+ _h_bind_indirect(self, parser)
+
+def indirect():
+ return HIndirectParser()
+
+def bind_indirect(indirect, new_parser):
+ indirect.bind(new_parser)
+
+def uint8(): return _h_uint8()
+def uint16(): return _h_uint16()
+def uint32(): return _h_uint32()
+def uint64(): return _h_uint64()
+def int8(): return _h_int8()
+def int16(): return _h_int16()
+def int32(): return _h_int32()
+def int64(): return _h_int64()
+
+
+%}
+
+#endif
diff --git a/src/glue.c b/src/glue.c
index 5e3804d0e0ec266cf3185867c975dada86f785dc..c2d915aeaab8e8d58646569f33edc5402af43023 100644
--- a/src/glue.c
+++ b/src/glue.c
@@ -5,7 +5,7 @@
#include "parsers/parser_internal.h"
// Helper to build HAction's that pick one index out of a sequence.
-HParsedToken *h_act_index(int i, const HParseResult *p)
+HParsedToken *h_act_index(int i, const HParseResult *p, void* user_data)
{
if(!p) return NULL;
@@ -23,7 +23,7 @@ HParsedToken *h_act_index(int i, const HParseResult *p)
return tok->seq->elements[i];
}
-HParsedToken *h_act_first(const HParseResult *p) {
+HParsedToken *h_act_first(const HParseResult *p, void* user_data) {
assert(p->ast);
assert(p->ast->token_type == TT_SEQUENCE);
assert(p->ast->seq->used > 0);
@@ -31,7 +31,7 @@ HParsedToken *h_act_first(const HParseResult *p) {
return p->ast->seq->elements[0];
}
-HParsedToken *h_act_second(const HParseResult *p) {
+HParsedToken *h_act_second(const HParseResult *p, void* user_data) {
assert(p->ast);
assert(p->ast->token_type == TT_SEQUENCE);
assert(p->ast->seq->used > 0);
@@ -39,7 +39,7 @@ HParsedToken *h_act_second(const HParseResult *p) {
return p->ast->seq->elements[1];
}
-HParsedToken *h_act_last(const HParseResult *p) {
+HParsedToken *h_act_last(const HParseResult *p, void* user_data) {
assert(p->ast);
assert(p->ast->token_type == TT_SEQUENCE);
assert(p->ast->seq->used > 0);
@@ -59,7 +59,7 @@ static void act_flatten_(HCountedArray *seq, const HParsedToken *tok) {
}
}
-HParsedToken *h_act_flatten(const HParseResult *p) {
+HParsedToken *h_act_flatten(const HParseResult *p, void* user_data) {
HCountedArray *seq = h_carray_new(p->arena);
act_flatten_(seq, p->ast);
@@ -72,7 +72,7 @@ HParsedToken *h_act_flatten(const HParseResult *p) {
return res;
}
-HParsedToken *h_act_ignore(const HParseResult *p) {
+HParsedToken *h_act_ignore(const HParseResult *p, void* user_data) {
return NULL;
}
diff --git a/src/glue.h b/src/glue.h
index 54b598507927b656f9bd1d17a363cd3a7bd723ac..1fe6ce46f453e911339e5ea3090e2436283f106a 100644
--- a/src/glue.h
+++ b/src/glue.h
@@ -53,16 +53,27 @@
// H_AVRULE is like H_VARULE but the action is attached outside the validation,
// i.e. the validation receives the uninterpreted AST as input.
//
-
+// H_ADRULE, H_VDRULE, H_AVDRULE, and H_VADRULE are the same as the
+// equivalent non-D variants, except that they also allow you to uset
+// the user_data pointer. In cases where both an attr_bool and an
+// action are used, the same userdata pointer is given to both.
#define H_RULE(rule, def) HParser *rule = def
-#define H_ARULE(rule, def) HParser *rule = h_action(def, act_ ## rule)
+#define H_ARULE(rule, def) HParser *rule = h_action(def, act_ ## rule, NULL)
#define H_VRULE(rule, def) HParser *rule = \
- h_attr_bool(def, validate_ ## rule)
+ h_attr_bool(def, validate_ ## rule, NULL)
#define H_VARULE(rule, def) HParser *rule = \
- h_attr_bool(h_action(def, act_ ## rule), validate_ ## rule)
+ h_attr_bool(h_action(def, act_ ## rule, NULL), validate_ ## rule, NULL)
#define H_AVRULE(rule, def) HParser *rule = \
- h_action(h_attr_bool(def, validate_ ## rule), act_ ## rule)
+ h_action(h_attr_bool(def, validate_ ## rule, NULL), act_ ## rule, NULL)
+#define H_ADRULE(rule, def, data) HParser *rule = \
+ h_action(def, act_ ## rule, data)
+#define H_VDRULE(rule, def, data) HParser *rule = \
+ h_attr_bool(def, validate_ ## rule, data)
+#define H_VADRULE(rule, def, data) HParser *rule = \
+ h_attr_bool(h_action(def, act_ ## rule, data), validate_ ## rule, data)
+#define H_AVDRULE(rule, def, data) HParser *rule = \
+ h_action(h_attr_bool(def, validate_ ## rule, data), act_ ## rule, data)
//
@@ -88,18 +99,18 @@
// action such as h_act_index.
//
-HParsedToken *h_act_index(int i, const HParseResult *p);
-HParsedToken *h_act_first(const HParseResult *p);
-HParsedToken *h_act_second(const HParseResult *p);
-HParsedToken *h_act_last(const HParseResult *p);
-HParsedToken *h_act_flatten(const HParseResult *p);
-HParsedToken *h_act_ignore(const HParseResult *p);
+HParsedToken *h_act_index(int i, const HParseResult *p, void* user_data);
+HParsedToken *h_act_first(const HParseResult *p, void* user_data);
+HParsedToken *h_act_second(const HParseResult *p, void* user_data);
+HParsedToken *h_act_last(const HParseResult *p, void* user_data);
+HParsedToken *h_act_flatten(const HParseResult *p, void* user_data);
+HParsedToken *h_act_ignore(const HParseResult *p, void* user_data);
// Define 'myaction' as a specialization of 'paction' by supplying the leading
// parameters.
#define H_ACT_APPLY(myaction, paction, ...) \
- HParsedToken *myaction(const HParseResult *p) { \
- return paction(__VA_ARGS__, p); \
+ HParsedToken *myaction(const HParseResult *p, void* user_data) { \
+ return paction(__VA_ARGS__, p, user_data); \
}
diff --git a/src/hammer.c b/src/hammer.c
index 7fc80dba0c86ec76a2376d0d69914f235bf08afc..2456bdcedb7c9c7a0b4e374e8b8146bf19603179 100644
--- a/src/hammer.c
+++ b/src/hammer.c
@@ -62,6 +62,10 @@ HParseResult* h_parse__m(HAllocator* mm__, const HParser* parser, const uint8_t*
return backends[parser->backend]->parse(mm__, parser, &input_stream);
}
+void h_parse_result_free__m(HAllocator *alloc, HParseResult *result) {
+ h_parse_result_free(result);
+}
+
void h_parse_result_free(HParseResult *result) {
if(result == NULL) return;
h_delete_arena(result->arena);
diff --git a/src/hammer.h b/src/hammer.h
index 508653d8fcf213bf97e7dceec7460e3e55a2f1e7..6756c0cc584951c905b9abdc918d5587b3ced28e 100644
--- a/src/hammer.h
+++ b/src/hammer.h
@@ -29,7 +29,15 @@
#define BIT_LITTLE_ENDIAN 0x0
#define BYTE_LITTLE_ENDIAN 0x0
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef __cplusplus
+#ifndef HAMMER_INTERNAL__NO_STDARG_H
typedef int bool;
+#endif // HAMMER_INTERNAL__NO_STDARG_H
+#endif
typedef struct HParseState_ HParseState;
@@ -68,8 +76,21 @@ typedef struct HBytes_ {
size_t len;
} HBytes;
+#ifdef SWIG
+typedef union {
+ HBytes bytes;
+ int64_t sint;
+ uint64_t uint;
+ double dbl;
+ float flt;
+ HCountedArray *seq;
+ void *user;
+} HTokenData;
+#endif
+
typedef struct HParsedToken_ {
HTokenType token_type;
+#ifndef SWIG
union {
HBytes bytes;
int64_t sint;
@@ -79,6 +100,9 @@ typedef struct HParsedToken_ {
HCountedArray *seq; // a sequence of HParsedToken's
void *user;
};
+#else
+ HTokenData token_data;
+#endif
size_t index;
char bit_offset;
} HParsedToken;
@@ -113,19 +137,20 @@ typedef struct HBitWriter_ HBitWriter;
* say, structs) and stuff values for them into the void* in the
* tagged union in HParsedToken.
*/
-typedef HParsedToken* (*HAction)(const HParseResult *p);
+typedef HParsedToken* (*HAction)(const HParseResult *p, void* user_data);
/**
* Type of a boolean attribute-checking function, used in the
* attr_bool() parser. It can be any (user-defined) function that takes
* a HParseResult* and returns true or false.
*/
-typedef bool (*HPredicate)(HParseResult *p);
+typedef bool (*HPredicate)(HParseResult *p, void* user_data);
typedef struct HCFChoice_ HCFChoice;
typedef struct HRVMProg_ HRVMProg;
typedef struct HParserVtable_ HParserVtable;
+// TODO: Make this internal
typedef struct HParser_ {
const HParserVtable *vtable;
HParserBackend backend;
@@ -141,12 +166,23 @@ typedef struct HParserTestcase_ {
char* output_unambiguous;
} HParserTestcase;
+#ifdef SWIG
+typedef union {
+ const char* actual_results;
+ size_t parse_time;
+} HResultTiming;
+#endif
+
typedef struct HCaseResult_ {
bool success;
+#ifndef SWIG
union {
const char* actual_results; // on failure, filled in with the results of h_write_result_unamb
size_t parse_time; // on success, filled in with time for a single parse, in nsec
};
+#else
+ HResultTiming timestamp;
+#endif
} HCaseResult;
typedef struct HBackendResults_ {
@@ -176,7 +212,7 @@ typedef struct HBenchmarkResults_ {
rtype_t name(__VA_ARGS__) attr; \
rtype_t name##__m(HAllocator* mm__, __VA_ARGS__) attr
-#ifndef HAMMER_INTERNAL__NO_STDARG_H
+#ifndef SWIG
#define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...) \
rtype_t name(__VA_ARGS__, ...); \
rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...); \
@@ -194,17 +230,17 @@ typedef struct HBenchmarkResults_ {
rtype_t name##__a(void *args[]); \
rtype_t name##__ma(HAllocator *mm__, void *args[])
#else
-#define HAMMER_FN_DECL_VARARGS(rtype_t, name, ...) \
- rtype_t name(__VA_ARGS__, ...); \
- rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...); \
- rtype_t name##__a(void *args[]); \
+#define HAMMER_FN_DECL_VARARGS(rtype_t, name, params...) \
+ rtype_t name(params, ...); \
+ rtype_t name##__m(HAllocator* mm__, params, ...); \
+ rtype_t name##__a(void *args[]); \
rtype_t name##__ma(HAllocator *mm__, void *args[])
// Note: this drops the attributes on the floor for the __v versions
-#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, ...) \
- rtype_t name(__VA_ARGS__, ...) attr; \
- rtype_t name##__m(HAllocator* mm__, __VA_ARGS__, ...) attr; \
- rtype_t name##__a(void *args[]); \
+#define HAMMER_FN_DECL_VARARGS_ATTR(attr, rtype_t, name, params...) \
+ rtype_t name(params, ...); \
+ rtype_t name##__m(HAllocator* mm__, params, ...); \
+ rtype_t name##__a(void *args[]); \
rtype_t name##__ma(HAllocator *mm__, void *args[])
#endif // HAMMER_INTERNAL__NO_STDARG_H
// }}}
@@ -349,7 +385,7 @@ HAMMER_FN_DECL(HParser*, h_middle, const HParser* p, const HParser* x, const HPa
*
* Result token type: any
*/
-HAMMER_FN_DECL(HParser*, h_action, const HParser* p, const HAction a);
+HAMMER_FN_DECL(HParser*, h_action, const HParser* p, const HAction a, void* user_data);
/**
* Parse a single character in the given charset.
@@ -515,7 +551,7 @@ HAMMER_FN_DECL(HParser*, h_length_value, const HParser* length, const HParser* v
*
* Result token type: p's result type if pred succeeded, NULL otherwise.
*/
-HAMMER_FN_DECL(HParser*, h_attr_bool, const HParser* p, HPredicate pred);
+HAMMER_FN_DECL(HParser*, h_attr_bool, const HParser* p, HPredicate pred, void* user_data);
/**
* The 'and' parser asserts that a conditional syntax is satisfied,
@@ -586,7 +622,7 @@ char* h_write_result_unamb(const HParsedToken* tok);
* Format token to the given output stream. Indent starting at
* [indent] spaces, with [delta] spaces between levels.
*/
-HAMMER_FN_DECL(void, h_pprint, FILE* stream, const HParsedToken* tok, int indent, int delta);
+void h_pprint(FILE* stream, const HParsedToken* tok, int indent, int delta);
/**
* Build parse tables for the given parser backend. See the
@@ -621,16 +657,31 @@ void h_bit_writer_free(HBitWriter* w);
// General-purpose actions for use with h_action
// XXX to be consolidated with glue.h when merged upstream
-HParsedToken *h_act_first(const HParseResult *p);
-HParsedToken *h_act_second(const HParseResult *p);
-HParsedToken *h_act_last(const HParseResult *p);
-HParsedToken *h_act_flatten(const HParseResult *p);
-HParsedToken *h_act_ignore(const HParseResult *p);
+HParsedToken *h_act_first(const HParseResult *p, void* userdata);
+HParsedToken *h_act_second(const HParseResult *p, void* userdata);
+HParsedToken *h_act_last(const HParseResult *p, void* userdata);
+HParsedToken *h_act_flatten(const HParseResult *p, void* userdata);
+HParsedToken *h_act_ignore(const HParseResult *p, void* userdata);
// {{{ Benchmark functions
HAMMER_FN_DECL(HBenchmarkResults *, h_benchmark, HParser* parser, HParserTestcase* testcases);
void h_benchmark_report(FILE* stream, HBenchmarkResults* results);
-void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results);
+//void h_benchmark_dump_optimized_code(FILE* stream, HBenchmarkResults* results);
// }}}
+// {{{ Token type registry
+/// Allocate a new, unused (as far as this function knows) token type.
+int h_allocate_token_type(const char* name);
+
+/// Get the token type associated with name. Returns -1 if name is unkown
+int h_get_token_type_number(const char* name);
+
+/// Get the name associated with token_type. Returns NULL if the token type is unkown
+const char* h_get_token_type_name(int token_type);
+// }}}
+
+#ifdef __cplusplus
+}
+#endif
+
#endif // #ifndef HAMMER_HAMMER__H
diff --git a/src/internal.h b/src/internal.h
index c402da511fe75da9a4365266207634fbdc9e8798..3c639d02fcb7c1b949f8285e03ef25020dc0a1f4 100644
--- a/src/internal.h
+++ b/src/internal.h
@@ -15,6 +15,11 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
+/*
+ * NOTE: This is an internal header and installed for use by extensions. The
+ * API is not guaranteed stable.
+*/
+
#ifndef HAMMER_INTERNAL__H
#define HAMMER_INTERNAL__H
#include <stdint.h>
@@ -360,6 +365,7 @@ struct HCFChoice_ {
// to execute before action and pred are applied.
HAction action;
HPredicate pred;
+ void* user_data;
};
struct HCFSequence_ {
diff --git a/src/parsers/action.c b/src/parsers/action.c
index 52c9bc1e9fdb12c0274f350d546dcde4a73036f6..e254a894320099d930e6fe08ddaa30e8d39ff18c 100644
--- a/src/parsers/action.c
+++ b/src/parsers/action.c
@@ -4,6 +4,7 @@
typedef struct {
const HParser *p;
HAction action;
+ void* user_data;
} HParseAction;
static HParseResult* parse_action(void *env, HParseState *state) {
@@ -12,8 +13,8 @@ static HParseResult* parse_action(void *env, HParseState *state) {
HParseResult *tmp = h_do_parse(a->p, state);
//HParsedToken *tok = a->action(h_do_parse(a->p, state));
if(tmp) {
- const HParsedToken *tok = a->action(tmp);
- return make_result(state->arena, (HParsedToken*)tok);
+ const HParsedToken *tok = a->action(tmp, a->user_data);
+ return make_result(state->arena, (HParsedToken*)tok);
} else
return NULL;
} else // either the parser's missing or the action's missing
@@ -27,6 +28,7 @@ static void desugar_action(HAllocator *mm__, HCFStack *stk__, void *env) {
HCFS_BEGIN_SEQ() {
HCFS_DESUGAR(a->p);
} HCFS_END_SEQ();
+ HCFS_THIS_CHOICE->user_data = a->user_data;
HCFS_THIS_CHOICE->action = a->action;
HCFS_THIS_CHOICE->reshape = h_act_first;
} HCFS_END_CHOICE();
@@ -44,7 +46,7 @@ static bool action_isValidCF(void *env) {
static bool h_svm_action_action(HArena *arena, HSVMContext *ctx, void* arg) {
HParseResult res;
- HAction action = arg;
+ HParseAction *a = arg;
assert(ctx->stack_count >= 1);
if (ctx->stack[ctx->stack_count-1]->token_type != TT_MARK) {
assert(ctx->stack_count >= 2 && ctx->stack[ctx->stack_count-2]->token_type == TT_MARK);
@@ -56,7 +58,7 @@ static bool h_svm_action_action(HArena *arena, HSVMContext *ctx, void* arg) {
}
res.arena = arena;
- HParsedToken *tok = action(&res);
+ HParsedToken *tok = a->action(&res, a->user_data);
if (tok != NULL)
ctx->stack[ctx->stack_count-1] = tok;
else
@@ -69,7 +71,7 @@ static bool action_ctrvm(HRVMProg *prog, void* env) {
h_rvm_insert_insn(prog, RVM_PUSH, 0);
if (!h_compile_regex(prog, a->p))
return false;
- h_rvm_insert_insn(prog, RVM_ACTION, h_rvm_create_action(prog, h_svm_action_action, a->action));
+ h_rvm_insert_insn(prog, RVM_ACTION, h_rvm_create_action(prog, h_svm_action_action, a));
return true;
}
@@ -81,13 +83,14 @@ static const HParserVtable action_vt = {
.compile_to_rvm = action_ctrvm,
};
-HParser* h_action(const HParser* p, const HAction a) {
- return h_action__m(&system_allocator, p, a);
+HParser* h_action(const HParser* p, const HAction a, void* user_data) {
+ return h_action__m(&system_allocator, p, a, user_data);
}
-HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a) {
+HParser* h_action__m(HAllocator* mm__, const HParser* p, const HAction a, void* user_data) {
HParseAction *env = h_new(HParseAction, 1);
env->p = p;
env->action = a;
+ env->user_data = user_data;
return h_new_parser(mm__, &action_vt, env);
}
diff --git a/src/parsers/attr_bool.c b/src/parsers/attr_bool.c
index fc980b24f8eaff2fbf29da79cc630ac0a6cdd988..e8359ab03e06e681061dd75788d6ca6bb6e9b89b 100644
--- a/src/parsers/attr_bool.c
+++ b/src/parsers/attr_bool.c
@@ -4,13 +4,14 @@
typedef struct {
const HParser *p;
HPredicate pred;
+ void* user_data;
} HAttrBool;
static HParseResult* parse_attr_bool(void *env, HParseState *state) {
HAttrBool *a = (HAttrBool*)env;
HParseResult *res = h_do_parse(a->p, state);
if (res && res->ast) {
- if (a->pred(res))
+ if (a->pred(res, a->user_data))
return res;
else
return NULL;
@@ -42,12 +43,13 @@ static void desugar_ab(HAllocator *mm__, HCFStack *stk__, void *env) {
} HCFS_END_SEQ();
HCFS_THIS_CHOICE->pred = a->pred;
HCFS_THIS_CHOICE->reshape = h_act_first;
+ HCFS_THIS_CHOICE->user_data = a->user_data;
} HCFS_END_CHOICE();
}
static bool h_svm_action_attr_bool(HArena *arena, HSVMContext *ctx, void* arg) {
HParseResult res;
- HPredicate pred = arg;
+ HAttrBool *ab = arg;
assert(ctx->stack_count >= 1);
if (ctx->stack[ctx->stack_count-1]->token_type != TT_MARK) {
assert(ctx->stack_count >= 2 && ctx->stack[ctx->stack_count-2]->token_type == TT_MARK);
@@ -59,7 +61,7 @@ static bool h_svm_action_attr_bool(HArena *arena, HSVMContext *ctx, void* arg) {
res.ast = NULL;
}
res.arena = arena;
- return pred(&res);
+ return ab->pred(&res, ab->user_data);
}
static bool ab_ctrvm(HRVMProg *prog, void *env) {
@@ -67,7 +69,7 @@ static bool ab_ctrvm(HRVMProg *prog, void *env) {
h_rvm_insert_insn(prog, RVM_PUSH, 0);
if (!h_compile_regex(prog, ab->p))
return false;
- h_rvm_insert_insn(prog, RVM_ACTION, h_rvm_create_action(prog, h_svm_action_attr_bool, ab->pred));
+ h_rvm_insert_insn(prog, RVM_ACTION, h_rvm_create_action(prog, h_svm_action_attr_bool, ab));
return true;
}
@@ -80,12 +82,13 @@ static const HParserVtable attr_bool_vt = {
};
-HParser* h_attr_bool(const HParser* p, HPredicate pred) {
- return h_attr_bool__m(&system_allocator, p, pred);
+HParser* h_attr_bool(const HParser* p, HPredicate pred, void* user_data) {
+ return h_attr_bool__m(&system_allocator, p, pred, user_data);
}
-HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred) {
+HParser* h_attr_bool__m(HAllocator* mm__, const HParser* p, HPredicate pred, void* user_data) {
HAttrBool *env = h_new(HAttrBool, 1);
env->p = p;
env->pred = pred;
+ env->user_data = user_data;
return h_new_parser(mm__, &attr_bool_vt, env);
}
diff --git a/src/parsers/bits.c b/src/parsers/bits.c
index 93b4aef60e9bac7f5fa112d3087192197d3dd0ca..716524ce61a0a35cb0f9581646f72a0efa491c7f 100644
--- a/src/parsers/bits.c
+++ b/src/parsers/bits.c
@@ -17,7 +17,9 @@ static HParseResult* parse_bits(void* env, HParseState *state) {
return make_result(state->arena, result);
}
-static HParsedToken *reshape_bits(const HParseResult *p, bool signedp) {
+static HParsedToken *reshape_bits(const HParseResult *p, void* signedp_p) {
+ // signedp == NULL iff unsigned
+ bool signedp = (signedp_p != NULL);
// XXX works only for whole bytes
// XXX assumes big-endian
assert(p->ast);
@@ -45,12 +47,6 @@ static HParsedToken *reshape_bits(const HParseResult *p, bool signedp) {
return ret;
}
-static HParsedToken *reshape_bits_unsigned(const HParseResult *p) {
- return reshape_bits(p, false);
-}
-static HParsedToken *reshape_bits_signed(const HParseResult *p) {
- return reshape_bits(p, true);
-}
static void desugar_bits(HAllocator *mm__, HCFStack *stk__, void *env) {
struct bits_env *bits = (struct bits_env*)env;
@@ -67,9 +63,9 @@ static void desugar_bits(HAllocator *mm__, HCFStack *stk__, void *env) {
HCFS_ADD_CHARSET(match_all);
}
} HCFS_END_SEQ();
- HCFS_THIS_CHOICE->reshape = bits->signedp
- ? reshape_bits_signed
- : reshape_bits_unsigned;
+ HCFS_THIS_CHOICE->reshape = reshape_bits;
+ HCFS_THIS_CHOICE->user_data = bits->signedp ? HCFS_THIS_CHOICE : NULL; // HCFS_THIS_CHOICE is an arbitrary non-null pointer
+
} HCFS_END_CHOICE();
}
diff --git a/src/parsers/choice.c b/src/parsers/choice.c
index bb55fa8dd96c32323b5fe201625c2fc4ca70c9b5..bfc3f904f19a6d88ddc0bd77702b2c45f89c2b0f 100644
--- a/src/parsers/choice.c
+++ b/src/parsers/choice.c
@@ -143,7 +143,7 @@ HParser* h_choice__ma(HAllocator* mm__, void *args[]) {
s->len = len;
HParser *ret = h_new(HParser, 1);
- ret->vtable = &choice_vt;
+ ret->vtable = &choice_vt;
ret->env = (void*)s;
ret->backend = PB_MIN;
return ret;
diff --git a/src/parsers/indirect.c b/src/parsers/indirect.c
index 2217a202968f2a11306c60ccea34c9e3126186c1..c91eaab5905229b178fa0c888dce1fa056babd88 100644
--- a/src/parsers/indirect.c
+++ b/src/parsers/indirect.c
@@ -21,6 +21,10 @@ static const HParserVtable indirect_vt = {
.compile_to_rvm = h_not_regular,
};
+void h_bind_indirect__m(HAllocator *mm__, HParser* indirect, const HParser* inner) {
+ h_bind_indirect(indirect, inner);
+}
+
void h_bind_indirect(HParser* indirect, const HParser* inner) {
assert_message(indirect->vtable == &indirect_vt, "You can only bind an indirect parser");
indirect->env = (void*)inner;
diff --git a/src/parsers/optional.c b/src/parsers/optional.c
index c4282a914fb81ae82b1b2ec18bd5d91ec001b5c3..ff9fc15bbf465bd36a0edd4fd1906c1f6077c1b1 100644
--- a/src/parsers/optional.c
+++ b/src/parsers/optional.c
@@ -22,7 +22,7 @@ static bool opt_isValidCF(void *env) {
return p->vtable->isValidCF(p->env);
}
-static HParsedToken* reshape_optional(const HParseResult *p) {
+static HParsedToken* reshape_optional(const HParseResult *p, void* user_data) {
assert(p->ast);
assert(p->ast->token_type == TT_SEQUENCE);
@@ -52,6 +52,7 @@ static void desugar_optional(HAllocator *mm__, HCFStack *stk__, void *env) {
HCFS_BEGIN_SEQ() {
} HCFS_END_SEQ();
HCFS_THIS_CHOICE->reshape = reshape_optional;
+ HCFS_THIS_CHOICE->user_data = NULL;
} HCFS_END_CHOICE();
}
diff --git a/src/parsers/parser_internal.h b/src/parsers/parser_internal.h
index aeb202bf410a53b262e87cc3eadde603b9e11e8a..ec97dd1b0696fcb69f4a17bfc7d4078138f4d355 100644
--- a/src/parsers/parser_internal.h
+++ b/src/parsers/parser_internal.h
@@ -1,3 +1,8 @@
+/*
+ * NOTE: This is an internal header and installed for use by extensions. The
+ * API is not guaranteed stable.
+*/
+
#ifndef HAMMER_PARSE_INTERNAL__H
#define HAMMER_PARSE_INTERNAL__H
#include "../hammer.h"
diff --git a/src/parsers/sequence.c b/src/parsers/sequence.c
index bdea8c02a45c4346a7523de97a7f6238ba12c246..93c0cfb983200a33b7909fd1b2c73114711beac5 100644
--- a/src/parsers/sequence.c
+++ b/src/parsers/sequence.c
@@ -43,7 +43,7 @@ static bool sequence_isValidCF(void *env) {
return true;
}
-static HParsedToken *reshape_sequence(const HParseResult *p) {
+static HParsedToken *reshape_sequence(const HParseResult *p, void* user_data) {
assert(p->ast);
assert(p->ast->token_type == TT_SEQUENCE);
@@ -72,6 +72,7 @@ static void desugar_sequence(HAllocator *mm__, HCFStack *stk__, void *env) {
HCFS_DESUGAR(s->p_array[i]);
} HCFS_END_SEQ();
HCFS_THIS_CHOICE->reshape = reshape_sequence;
+ HCFS_THIS_CHOICE->user_data = NULL;
} HCFS_END_CHOICE();
}
@@ -159,8 +160,8 @@ HParser* h_sequence__ma(HAllocator* mm__, void *args[]) {
s->len = len;
HParser *ret = h_new(HParser, 1);
- ret->vtable = &sequence_vt;
- ret->env = (void*)s;
+ ret->vtable = &sequence_vt;
+ ret->env = (void*)s;
ret->backend = PB_MIN;
return ret;
}
diff --git a/src/parsers/token.c b/src/parsers/token.c
index 97886b7afaef94124028c927e354198a705352d1..d36ec54be4c07a35b729da71455c5bc3b3555cbc 100644
--- a/src/parsers/token.c
+++ b/src/parsers/token.c
@@ -20,7 +20,7 @@ static HParseResult* parse_token(void *env, HParseState *state) {
}
-static HParsedToken *reshape_token(const HParseResult *p) {
+static HParsedToken *reshape_token(const HParseResult *p, void* user_data) {
// fetch sequence of uints from p
assert(p->ast);
assert(p->ast->token_type == TT_SEQUENCE);
@@ -52,6 +52,7 @@ static void desugar_token(HAllocator *mm__, HCFStack *stk__, void *env) {
HCFS_ADD_CHAR(tok->str[i]);
} HCFS_END_SEQ();
HCFS_THIS_CHOICE->reshape = reshape_token;
+ HCFS_THIS_CHOICE->user_data = NULL;
} HCFS_END_CHOICE();
}
@@ -79,6 +80,8 @@ HParser* h_token(const uint8_t *str, const size_t len) {
}
HParser* h_token__m(HAllocator* mm__, const uint8_t *str, const size_t len) {
HToken *t = h_new(HToken, 1);
- t->str = (uint8_t*)str, t->len = len;
+ uint8_t *str_cpy = h_new(uint8_t, len);
+ memcpy(str_cpy, str, len);
+ t->str = str_cpy, t->len = len;
return h_new_parser(mm__, &token_vt, t);
}
diff --git a/src/registry.c b/src/registry.c
new file mode 100644
index 0000000000000000000000000000000000000000..c59b6ea9f6e0214c279d51b3997b5770278928bc
--- /dev/null
+++ b/src/registry.c
@@ -0,0 +1,87 @@
+/* Parser combinators for binary formats.
+ * Copyright (C) 2012 Meredith L. Patterson, Dan "TQ" Hirsch
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include <search.h>
+#include <stdlib.h>
+#include "hammer.h"
+#include "internal.h"
+
+typedef struct Entry_ {
+ const char* name;
+ int value;
+} Entry;
+
+static void *tt_registry = NULL;
+static Entry** tt_by_id = NULL;
+static int tt_by_id_sz = 0;
+#define TT_START TT_USER
+static int tt_next = TT_START;
+
+/*
+ // TODO: These are for the extension registry, which does not yet have a good name.
+static void *ext_registry = NULL;
+static Entry** ext_by_id = NULL;
+static int ext_by_id_sz = 0;
+static int ext_next = 0;
+*/
+
+
+static int compare_entries(const void* v1, const void* v2) {
+ const Entry *e1 = (Entry*)v1, *e2 = (Entry*)v2;
+ return strcmp(e1->name, e2->name);
+}
+
+int h_allocate_token_type(const char* name) {
+ Entry* new_entry = malloc(sizeof(*new_entry));
+ new_entry->name = name;
+ new_entry->value = -1;
+ Entry* probe = *(Entry**)tsearch(new_entry, &tt_registry, compare_entries);
+ if (probe->value != -1) {
+ // Token type already exists...
+ // TODO: treat this as a bug?
+ free(new_entry);
+ return probe->value;
+ } else {
+ // new value
+ probe->name = strdup(probe->name); // drop ownership of name
+ probe->value = tt_next++;
+ if ((probe->value - TT_START) >= tt_by_id_sz) {
+ if (tt_by_id_sz == 0)
+ tt_by_id = malloc(sizeof(*tt_by_id) * ((tt_by_id_sz = (tt_next - TT_START) * 16)));
+ else
+ tt_by_id = realloc(tt_by_id, sizeof(*tt_by_id) * ((tt_by_id_sz *= 2)));
+ }
+ assert(probe->value - TT_START < tt_by_id_sz);
+ tt_by_id[probe->value - TT_START] = probe;
+ return probe->value;
+ }
+}
+int h_get_token_type_number(const char* name) {
+ Entry e;
+ e.name = name;
+ Entry **ret = (Entry**)tfind(&e, &tt_registry, compare_entries);
+ if (ret == NULL)
+ return -1;
+ else
+ return (*ret)->value;
+}
+const char* h_get_token_type_name(int token_type) {
+ if (token_type >= tt_next || token_type < TT_START)
+ return NULL;
+ else
+ return tt_by_id[token_type - TT_START]->name;
+}
diff --git a/src/t_misc.c b/src/t_misc.c
index c762c07380bc36db4426c6877d784f329d2ac0bd..74a57ca9b438e75fcbf222b4bbbc79a951c39022 100644
--- a/src/t_misc.c
+++ b/src/t_misc.c
@@ -1,4 +1,5 @@
#include <glib.h>
+#include <string.h>
#include "test_suite.h"
#include "hammer.h"
@@ -11,6 +12,24 @@ static void test_tt_user(void) {
g_check_cmp_int32(TT_USER, >, TT_ERR);
}
+static void test_tt_registry(void) {
+ int id = h_allocate_token_type("com.upstandinghackers.test.token_type");
+ g_check_cmp_int32(id, >=, TT_USER);
+ int id2 = h_allocate_token_type("com.upstandinghackers.test.token_type_2");
+ g_check_cmp_int32(id2, !=, id);
+ g_check_cmp_int32(id2, >=, TT_USER);
+ g_check_cmp_int32(id, ==, h_get_token_type_number("com.upstandinghackers.test.token_type"));
+ g_check_cmp_int32(id2, ==, h_get_token_type_number("com.upstandinghackers.test.token_type_2"));
+ g_check_string("com.upstandinghackers.test.token_type", ==, h_get_token_type_name(id));
+ g_check_string("com.upstandinghackers.test.token_type_2", ==, h_get_token_type_name(id2));
+ if (h_get_token_type_name(0) != NULL) {
+ g_test_message("Unknown token type should not return a name");
+ g_test_fail();
+ }
+ g_check_cmp_int32(h_get_token_type_number("com.upstandinghackers.test.unkown_token_type"), ==, -1);
+}
+
void register_misc_tests(void) {
g_test_add_func("/core/misc/tt_user", test_tt_user);
+ g_test_add_func("/core/misc/tt_registry", test_tt_registry);
}
diff --git a/src/t_parser.c b/src/t_parser.c
index bba0148aecd5777a8f8199deb4e4b4df4de14bd5..e2eca978e9e96b341b04d55ba8a850d95dacae54 100644
--- a/src/t_parser.c
+++ b/src/t_parser.c
@@ -162,7 +162,7 @@ static void test_middle(gconstpointer backend) {
#include <ctype.h>
-HParsedToken* upcase(const HParseResult *p) {
+HParsedToken* upcase(const HParseResult *p, void* user_data) {
switch(p->ast->token_type) {
case TT_SEQUENCE:
{
@@ -202,7 +202,8 @@ static void test_action(gconstpointer backend) {
h_ch('B'),
NULL),
NULL),
- upcase);
+ upcase,
+ NULL);
g_check_parse_match(action_, (HParserBackend)GPOINTER_TO_INT(backend), "ab", 2, "(u0x41 u0x42)");
g_check_parse_match(action_, (HParserBackend)GPOINTER_TO_INT(backend), "AB", 2, "(u0x41 u0x42)");
@@ -364,7 +365,7 @@ static void test_epsilon_p(gconstpointer backend) {
g_check_parse_match(epsilon_p_3, (HParserBackend)GPOINTER_TO_INT(backend), "a", 1, "(u0x61)");
}
-bool validate_test_ab(HParseResult *p) {
+bool validate_test_ab(HParseResult *p, void* user_data) {
if (TT_SEQUENCE != p->ast->token_type)
return false;
if (TT_UINT != p->ast->seq->elements[0]->token_type)
@@ -376,7 +377,8 @@ bool validate_test_ab(HParseResult *p) {
static void test_attr_bool(gconstpointer backend) {
const HParser *ab_ = h_attr_bool(h_many1(h_choice(h_ch('a'), h_ch('b'), NULL)),
- validate_test_ab);
+ validate_test_ab,
+ NULL);
g_check_parse_match(ab_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", 2, "(u0x61 u0x61)");
g_check_parse_match(ab_, (HParserBackend)GPOINTER_TO_INT(backend), "bb", 2, "(u0x62 u0x62)");
@@ -410,11 +412,11 @@ static void test_leftrec(gconstpointer backend) {
HParser *a_ = h_ch('a');
HParser *lr_ = h_indirect();
- h_bind_indirect(lr_, h_choice(h_sequence(lr_, a_, NULL), a_, NULL));
+ h_bind_indirect(lr_, h_choice(h_sequence(lr_, a_, NULL), h_epsilon_p(), NULL));
- g_check_parse_match(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "a", 1, "u0x61");
- g_check_parse_match(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", 2, "(u0x61 u0x61)");
- g_check_parse_match(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", 3, "((u0x61 u0x61) u0x61)");
+ g_check_parse_match(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "a", 1, "(u0x61)");
+ g_check_parse_match(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aa", 2, "((u0x61) u0x61)");
+ g_check_parse_match(lr_, (HParserBackend)GPOINTER_TO_INT(backend), "aaa", 3, "(((u0x61) u0x61) u0x61)");
}
static void test_rightrec(gconstpointer backend) {
@@ -433,7 +435,7 @@ static void test_ambiguous(gconstpointer backend) {
HParser *p_ = h_ch('+');
HParser *E_ = h_indirect();
h_bind_indirect(E_, h_choice(h_sequence(E_, p_, E_, NULL), d_, NULL));
- HParser *expr_ = h_action(E_, h_act_flatten);
+ HParser *expr_ = h_action(E_, h_act_flatten, NULL);
g_check_parse_match(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d", 1, "(u0x64)");
g_check_parse_match(expr_, (HParserBackend)GPOINTER_TO_INT(backend), "d+d", 3, "(u0x64 u0x2b u0x64)");
@@ -483,7 +485,7 @@ void register_parser_tests(void) {
g_test_add_data_func("/core/parser/packrat/and", GINT_TO_POINTER(PB_PACKRAT), test_and);
g_test_add_data_func("/core/parser/packrat/not", GINT_TO_POINTER(PB_PACKRAT), test_not);
g_test_add_data_func("/core/parser/packrat/ignore", GINT_TO_POINTER(PB_PACKRAT), test_ignore);
- // g_test_add_data_func("/core/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec);
+ //g_test_add_data_func("/core/parser/packrat/leftrec", GINT_TO_POINTER(PB_PACKRAT), test_leftrec);
g_test_add_data_func("/core/parser/packrat/rightrec", GINT_TO_POINTER(PB_PACKRAT), test_rightrec);
g_test_add_data_func("/core/parser/llk/token", GINT_TO_POINTER(PB_LLk), test_token);
diff --git a/tools/scanreplace.py b/tools/scanreplace.py
new file mode 100644
index 0000000000000000000000000000000000000000..5321e4813d9ee0030964c4dd1bcf207710808037
--- /dev/null
+++ b/tools/scanreplace.py
@@ -0,0 +1,15 @@
+from string import Template
+
+def replace_action(target, source, env):
+ open(str(target[0]), 'w').write(Template(open(str(source[0]), 'r').read()).safe_substitute(env))
+ return 0
+
+def replace_string(target, source, env):
+ return "building '%s' from '%s'" % (str(target[0]), str(source[0]))
+
+def generate(env, **kw):
+ action = env.Action(replace_action, replace_string)
+ env['BUILDERS']['ScanReplace'] = env.Builder(action=action, src_suffix='.in', single_source=True)
+
+def exists(env):
+ return 1