src/backends/packrat.c

@@ -3,6 +3,17 @@
 #include "../internal.h"
 #include "../parsers/parser_internal.h"
 
+/* #define DETAILED_PACKRAT_STATISTICS */
+
+#ifdef DETAILED_PACKRAT_STATISTICS
+static size_t packrat_hash_count = 0;
+static size_t packrat_hash_bytes = 0;
+static size_t packrat_cmp_count = 0;
+static size_t packrat_cmp_bytes = 0;
+#endif
+
+static uint32_t cache_key_hash(const void* key);
+
 // short-hand for creating lowlevel parse cache values (parse result case)
 static
 HParserCacheValue * cached_result(HParseState *state, HParseResult *result) {
@@ -23,64 +34,73 @@ HParserCacheValue *cached_lr(HParseState *state, HLeftRec *lr) {
   return ret;
 }
 
-// Really library-internal tool to perform an uncached parse, and handle any common error-handling.
-static inline HParseResult* perform_lowlevel_parse(HParseState *state, const HParser *parser) {
-  // TODO(thequux): these nested conditions are ugly. Factor this appropriately, so that it is clear which codes is executed when.
-  HParseResult *tmp_res;
-  if (parser) {
-    HInputStream bak = state->input_stream;
-    tmp_res = parser->vtable->parse(parser->env, state);
-    if (tmp_res) {
-      tmp_res->arena = state->arena;
-      if (!state->input_stream.overrun) {
-	size_t bit_length = h_input_stream_pos(&state->input_stream) - h_input_stream_pos(&bak);
-	if (tmp_res->bit_length == 0) { // Don't modify if forwarding.
-	  tmp_res->bit_length = bit_length;
-	}
-	if (tmp_res->ast && tmp_res->ast->bit_length != 0) {
-	  ((HParsedToken*)(tmp_res->ast))->bit_length = bit_length;
-	}
-      } else
-	tmp_res->bit_length = 0;
-    }
-  } else
-    tmp_res = NULL;
-  if (state->input_stream.overrun)
-    return NULL; // overrun is always failure.
-#ifdef CONSISTENCY_CHECK
-  if (!tmp_res) {
-    state->input_stream = INVALID;
-    state->input_stream.input = key->input_pos.input;
+// internal helper to perform an uncached parse and common error-handling
+static inline
+HParseResult *perform_lowlevel_parse(HParseState *state, const HParser *parser)
+{
+  HParseResult *res;
+  HInputStream bak;
+  size_t len;
+
+  if (!parser)
+    return NULL;
+
+  bak = state->input_stream;
+  res = parser->vtable->parse(parser->env, state);
+
+  if (!res)
+    return NULL;	// NB: input position is considered invalid on failure
+
+  // combinators' parse functions by design do not have to check for overrun.
+  // turn such bogus successes into parse failure.
+  if (state->input_stream.overrun) {
+    res->bit_length = 0;
+    return NULL;
   }
-#endif
-  return tmp_res;
+
+  // update result length
+  res->arena = state->arena;
+  len = h_input_stream_pos(&state->input_stream) - h_input_stream_pos(&bak);
+  if (res->bit_length == 0)	// Don't modify if forwarding.
+    res->bit_length = len;
+  if (res->ast && res->ast->bit_length != 0)
+    ((HParsedToken *)(res->ast))->bit_length = len;
+
+  return res;
 }
 
-HParserCacheValue* recall(HParserCacheKey *k, HParseState *state) {
-  HParserCacheValue *cached = h_hashtable_get(state->cache, k);
+HParserCacheValue* recall(HParserCacheKey *k, HParseState *state, HHashValue keyhash) {
+  HParserCacheValue *cached = h_hashtable_get_precomp(state->cache, k, keyhash);
   HRecursionHead *head = h_hashtable_get(state->recursion_heads, &k->input_pos);
-  if (!head) { // No heads found
+
+  if (!head) {
+    /* No heads found */
     return cached;
-  } else { // Some heads found
+  } else {
+    /* Some heads found */
     if (!cached && head->head_parser != k->parser && !h_slist_find(head->involved_set, k->parser)) {
-      // Nothing in the cache, and the key parser is not involved
+      /* Nothing in the cache, and the key parser is not involved */
       cached = cached_result(state, NULL);
       cached->input_stream = k->input_pos;
     }
     if (h_slist_find(head->eval_set, k->parser)) {
-      // Something is in the cache, and the key parser is in the eval set. Remove the key parser from the eval set of the head. 
+      /*
+       * Something is in the cache, and the key parser is in the eval set.
+       * Remove the key parser from the eval set of the head.
+       */
       head->eval_set = h_slist_remove_all(head->eval_set, k->parser);
       HParseResult *tmp_res = perform_lowlevel_parse(state, k->parser);
-      // update the cache
+      /* update the cache */
       if (!cached) {
-	cached = cached_result(state, tmp_res);
-	h_hashtable_put(state->cache, k, cached);
+        cached = cached_result(state, tmp_res);
+        h_hashtable_put_precomp(state->cache, k, cached, keyhash);
       } else {
-	cached->value_type = PC_RIGHT;
-	cached->right = tmp_res;
-	cached->input_stream = state->input_stream;
+        cached->value_type = PC_RIGHT;
+        cached->right = tmp_res;
+        cached->input_stream = state->input_stream;
       }
     }
+
     return cached;
   }
 }
@@ -180,36 +200,52 @@ HParseResult* lr_answer(HParserCacheKey *k, HParseState *state, HLeftRec *growab
 /* Warth's recursion. Hi Alessandro! */
 HParseResult* h_do_parse(const HParser* parser, HParseState *state) {
   HParserCacheKey *key = a_new(HParserCacheKey, 1);
+  HHashValue keyhash;
+  HLeftRec *base = NULL;
+  HParserCacheValue *m = NULL, *cached = NULL;
+
   key->input_pos = state->input_stream; key->parser = parser;
-  HParserCacheValue *m = NULL;
+  keyhash = cache_key_hash(key);
+
   if (parser->vtable->higher) {
-    m = recall(key, state);
+    m = recall(key, state, keyhash);
   }
-  // check to see if there is already a result for this object...
+
+  /* check to see if there is already a result for this object... */
   if (!m) {
-    // It doesn't exist, so create a dummy result to cache
-    HLeftRec *base = NULL;
-    // But only cache it now if there's some chance it could grow; primitive parsers can't
+    /*
+     * But only cache it now if there's some chance it could grow; primitive
+     * parsers can't
+     */
     if (parser->vtable->higher) {
       base = a_new(HLeftRec, 1);
       base->seed = NULL; base->rule = parser; base->head = NULL;
       h_slist_push(state->lr_stack, base);
-      // cache it
-      h_hashtable_put(state->cache, key, cached_lr(state, base));
-      // parse the input
+      /* cache it */
+      h_hashtable_put_precomp(state->cache, key,
+                              cached_lr(state, base), keyhash);
     }
+
+    /* parse the input */
     HParseResult *tmp_res = perform_lowlevel_parse(state, parser);
     if (parser->vtable->higher) {
-      // the base variable has passed equality tests with the cache
+      /* the base variable has passed equality tests with the cache */
       h_slist_pop(state->lr_stack);
-      // update the cached value to our new position
-      HParserCacheValue *cached = h_hashtable_get(state->cache, key);
+      /* update the cached value to our new position */
+      cached = h_hashtable_get_precomp(state->cache, key, keyhash);
       assert(cached != NULL);
       cached->input_stream = state->input_stream;
     }
-    // setupLR, used below, mutates the LR to have a head if appropriate, so we check to see if we have one
+
+    /*
+     * setupLR, used below, mutates the LR to have a head if appropriate,
+     * so we check to see if we have one
+     */
     if (!base || NULL == base->head) {
-      h_hashtable_put(state->cache, key, cached_result(state, tmp_res));
+      if (parser->vtable->higher) {
+        h_hashtable_put_precomp(state->cache, key,
+                                cached_result(state, tmp_res), keyhash);
+      }
       return tmp_res;
     } else {
       base->seed = tmp_res;
@@ -217,7 +253,7 @@ HParseResult* h_do_parse(const HParser* parser, HParseState *state) {
       return res;
     }
   } else {
-    // it exists!
+    /* it exists! */
     state->input_stream = m->input_stream;
     if (PC_LEFT == m->value_type) {
       setupLR(parser, state, m->left);
@@ -229,27 +265,46 @@ HParseResult* h_do_parse(const HParser* parser, HParseState *state) {
 }
 
 int h_packrat_compile(HAllocator* mm__, HParser* parser, const void* params) {
+  parser->backend_vtable = &h__packrat_backend_vtable;
   parser->backend = PB_PACKRAT;
   return 0; // No compilation necessary, and everything should work
 	    // out of the box.
 }
 
 void h_packrat_free(HParser *parser) {
-  parser->backend = PB_PACKRAT; // revert to default, oh that's us
+  parser->backend_vtable = h_get_default_backend_vtable();
+  parser->backend = h_get_default_backend();
 }
 
 static uint32_t cache_key_hash(const void* key) {
+#ifdef DETAILED_PACKRAT_STATISTICS
+  ++(packrat_hash_count);
+  packrat_hash_bytes += sizeof(HParserCacheKey);
+#endif
   return h_djbhash(key, sizeof(HParserCacheKey));
 }
+
 static bool cache_key_equal(const void* key1, const void* key2) {
+#ifdef DETAILED_PACKRAT_STATISTICS
+  ++(packrat_cmp_count);
+  packrat_cmp_bytes += sizeof(HParserCacheKey);
+#endif
   return memcmp(key1, key2, sizeof(HParserCacheKey)) == 0;
 }
 
 static uint32_t pos_hash(const void* key) {
+#ifdef DETAILED_PACKRAT_STATISTICS
+  ++(packrat_hash_count);
+  packrat_hash_bytes += sizeof(HInputStream);
+#endif
   return h_djbhash(key, sizeof(HInputStream));
 }
 
 static bool pos_equal(const void* key1, const void* key2) {
+#ifdef DETAILED_PACKRAT_STATISTICS
+  ++(packrat_cmp_count);
+  packrat_cmp_bytes += sizeof(HInputStream);
+#endif
   return memcmp(key1, key2, sizeof(HInputStream)) == 0;
 }
 
@@ -271,7 +326,9 @@ HParseResult *h_packrat_parse(HAllocator* mm__, const HParser* parser, HInputStr
   parse_state->lr_stack = h_slist_new(arena);
   parse_state->recursion_heads = h_hashtable_new(arena, pos_equal, pos_hash);
   parse_state->arena = arena;
+  parse_state->symbol_table = NULL;
   HParseResult *res = h_do_parse(parser, parse_state);
+  *input_stream = parse_state->input_stream;
   h_slist_free(parse_state->lr_stack);
   h_hashtable_free(parse_state->recursion_heads);
   // tear down the parse state
@@ -282,8 +339,121 @@ HParseResult *h_packrat_parse(HAllocator* mm__, const HParser* parser, HInputStr
   return res;
 }
 
+// The following naive implementation of the iterative (chunked) parsing API
+// concatenates chunks and blindly re-runs the full parse on every call to
+// h_packrat_parse_chunk.
+//
+// NB: A full implementation will still have to concatenate the chunks to
+// support arbitrary backtracking, but should be able save much, if not all, of
+// the HParseState between calls.
+// Cutting unneeded past input should also be possible but is complicated by
+// the fact that only higher-order combinators are saved to the packrat cache,
+// so former input to bare primitive combinators must remain available.
+//
+// Note: The iterative API expects us to always consume an entire input chunk
+// when we suspend, even if packrat later backtracks into it. We will produce
+// the correct parse result and accurately consume from a final chunk, but all
+// earlier chunks will be reported as fully consumed and as being part of the
+// HParseResult in terms of its bit_length field.
+
+void h_packrat_parse_start(HSuspendedParser *s)
+{
+  // nothing to do here, we allocate lazily below
+}
+
+bool h_packrat_parse_chunk(HSuspendedParser *s, HInputStream *input)
+{
+  HAllocator *mm__ = s->mm__;
+  HParseResult *res;
+  HInputStream *cat;
+  size_t newlen;
+
+  if (s->backend_state == NULL) {	// this is the first chunk
+    // attempt to finish the parse on just the given input.
+    res = h_packrat_parse(mm__, s->parser, input);
+    if (input->last_chunk || !input->overrun) {
+      s->backend_state = res;		// pass on the result
+      return true;			// and signal we're done
+    }
+
+    // we ran out of input and are expecting more
+    // allocate and initialize an input stream to concatenate the chunks
+    cat = h_new(HInputStream, 1);
+    *cat = *input;
+    cat->input = h_alloc(mm__, input->length);
+    memcpy((void *)cat->input, input->input, input->length);
+    s->backend_state = cat;
+
+    goto suspend;
+  }
+
+  // we have received additional input - append it to the saved stream
+  cat = s->backend_state;
+  assert(input->pos == cat->length);
+  if (input->length > SIZE_MAX - cat->length)
+    h_platform_errx(1, "input length would overflow");
+  newlen = cat->length + input->length;
+  cat->input = h_realloc(mm__, (void *)cat->input, newlen);
+  memcpy((void *)cat->input + cat->length, input->input, input->length);
+  cat->length = newlen;
+  cat->last_chunk = input->last_chunk;
+
+  // reset our input stream and call the parser on it (again)
+  cat->index      = 0;
+  cat->bit_offset = 0;
+  cat->margin     = 0;
+  cat->endianness = DEFAULT_ENDIANNESS;
+  cat->overrun    = false;
+  res = h_packrat_parse(mm__, s->parser, cat);
+  assert(cat->index <= cat->length);
+  input->overrun    = cat->overrun;
+
+  // suspend if the parser still needs more input
+  if (input->overrun && !input->last_chunk)
+    goto suspend;
+  // otherwise the parse is finished...
+
+  // report final input position
+  if (cat->index < input->pos) {	// parser just needed some lookahead
+    input->index      = 0;		// don't consume this last chunk
+    input->bit_offset = 0;
+    input->margin     = 0;
+  } else {
+    input->index      = cat->index - input->pos;
+    input->bit_offset = cat->bit_offset;
+    input->margin     = cat->margin;
+    input->endianness = cat->endianness;
+  }
+
+  // clean up and return the result
+  h_free((void *)cat->input);
+  h_free(cat);
+  s->backend_state = res;
+
+  return true;				// don't call me again.
+
+suspend:
+  input->index = input->length;		// consume the entire chunk on suspend
+  input->margin = 0;
+  input->bit_offset = 0;
+  return false;				// come back with more input.
+}
+
+HParseResult *h_packrat_parse_finish(HSuspendedParser *s)
+{
+  return s->backend_state;
+}
+
 HParserBackendVTable h__packrat_backend_vtable = {
   .compile = h_packrat_compile,
   .parse = h_packrat_parse,
-  .free = h_packrat_free
+  .free = h_packrat_free,
+  .parse_start = h_packrat_parse_start,
+  .parse_chunk = h_packrat_parse_chunk,
+  .parse_finish = h_packrat_parse_finish,
+  /* Name/param resolution functions */
+  .backend_short_name = "packrat",
+  .backend_description = "Packrat parser with Warth's recursion",
+  .get_description_with_params = h_get_description_with_no_params,
+  .get_short_name_with_params = h_get_short_name_with_no_params
 };

src/backends/params.c

+#include "params.h"
+
+
+
+size_t h_get_param_k(void *param) {
+  uintptr_t params_int;
+
+  params_int = (uintptr_t)param;
+  return (size_t)params_int;
+
+}
+
+
+
+char * h_format_description_with_param_k(HAllocator *mm__, const char *backend_name, size_t k){
+  const char *format_str = "%s(%zu) parser backend";
+
+  const char *generic_descr_format_str =
+    "%s(k) parser backend (default k is %zu)";
+  size_t len;
+  char *descr = NULL;
+
+
+  if (k > 0) {
+    /* A specific k was given */
+    /* Measure how big a buffer we need */
+    len = snprintf(NULL, 0, format_str, backend_name, k);
+    /* Allocate it and do the real snprintf */
+    descr = h_new(char, len + 1);
+    if (descr) {
+      snprintf(descr, len + 1, format_str, backend_name, k);
+    }
+  } else {
+    /*
+     * No specific k, would use DEFAULT_KMAX.  We say what DEFAULT_KMAX
+     * was compiled in in the description.
+     */
+    len = snprintf(NULL, 0, generic_descr_format_str, backend_name, DEFAULT_KMAX);
+    /* Allocate and do the real snprintf */
+    descr = h_new(char, len + 1);
+    if (descr) {
+      snprintf(descr, len + 1, generic_descr_format_str, backend_name, DEFAULT_KMAX);
+    }
+  }
+
+  return descr;
+}
+
+char * h_format_name_with_param_k(HAllocator *mm__, const char *backend_name, size_t k){
+  const char  *format_str = "%s(%zu)", *generic_name = "%s(k)";
+  size_t len;
+  char *name = NULL;
+
+  if (k > 0) {
+    /* A specific k was given */
+    /* Measure how big a buffer we need */
+    len = snprintf(NULL, 0, format_str, backend_name, k);
+    /* Allocate it and do the real snprintf */
+    name = h_new(char, len + 1);
+    if (name) {
+      snprintf(name, len + 1, format_str, backend_name, k);
+    }
+  } else {
+    /* No specific k */
+    len = snprintf(NULL, 0, generic_name, backend_name, k);
+    name = h_new(char, len + 1);
+    if (name) {
+      snprintf(name, len + 1, generic_name, backend_name);
+    }
+  }
+
+  return name;
+}
+
+/*TODO better error handling*/
+int h_extract_param_k(HParserBackendWithParams * be_with_params, backend_with_params_t * be_with_params_t) {
+
+  be_with_params->params = NULL;
+
+  int param_0 = -1;
+  int success = 0;
+  uintptr_t param;
+
+  size_t expected_params_len = 1;
+  backend_params_t params_t = be_with_params_t->params;
+  size_t actual_params_len = params_t.len;
+
+  if(actual_params_len >= expected_params_len) {
+    backend_param_with_name_t param_t = params_t.params[0];
+    success = sscanf((char*)param_t.param.param, "%d", &param_0);
+  }
+
+  if(success) {
+    param = (uintptr_t) param_0;
+    be_with_params->params = (void *)param;
+  }
+
+  return success;
+
+}

src/backends/params.h

+#ifndef HAMMER_BACKENDS_PARAMS__H
+#define HAMMER_BACKENDS_PARAMS__H
+
+#include "../hammer.h"
+#include "../internal.h"
+
+static const size_t DEFAULT_KMAX = 1;
+
+
+
+size_t h_get_param_k(void *param);
+
+char * h_format_description_with_param_k(HAllocator *mm__, const char *backend_name, size_t k);
+
+char * h_format_name_with_param_k(HAllocator *mm__, const char *backend_name, size_t k);
+
+int h_extract_param_k(HParserBackendWithParams * be_with_params, backend_with_params_t * be_with_params_t);
+
+#endif /* !defined(HAMMER_BACKENDS_PARAMS__H) */

src/backends/regex.c

@@ -7,6 +7,8 @@
 
 #undef a_new
 #define a_new(typ, count) a_new_(arena, typ, count)
+#undef a_new0
+#define a_new0(typ, count) a_new0_(arena, typ, count)
 // Stack VM
 typedef enum HSVMOp_ {
   SVM_PUSH, // Push a mark. There is no VM insn to push an object.
@@ -67,13 +69,13 @@ void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_
     goto end;
 
   HSArray *heads_n = heads_a, *heads_p = heads_b;
-  uint8_t *insn_seen = a_new(uint8_t, prog->length); // 0 -> not seen, 1->processed, 2->queued
-  HRVMThread *ip_queue = a_new(HRVMThread, prog->length);
+  uint8_t *insn_seen = a_new0(uint8_t, prog->length); // 0 -> not seen, 1->processed, 2->queued
+  HRVMThread *ip_queue = a_new0(HRVMThread, prog->length);
   size_t ipq_top;
 
 #define THREAD ip_queue[ipq_top-1]
 #define PUSH_SVM(op_, arg_) do { \
-	  HRVMTrace *nt = a_new(HRVMTrace, 1); \
+	  HRVMTrace *nt = a_new0(HRVMTrace, 1); \
 	  nt->arg = (arg_);		       \
 	  nt->opcode = (op_);		       \
 	  nt->next = THREAD.trace;	       \
@@ -81,7 +83,7 @@ void* h_rvm_run__m(HAllocator *mm__, HRVMProg *prog, const uint8_t* input, size_
 	  THREAD.trace = nt;		       \
   } while(0)
 
-  ((HRVMTrace*)h_sarray_set(heads_n, 0, a_new(HRVMTrace, 1)))->opcode = SVM_NOP; // Initial thread
+  ((HRVMTrace*)h_sarray_set(heads_n, 0, a_new0(HRVMTrace, 1)))->opcode = SVM_NOP; // Initial thread
   
   size_t off = 0;
   int live_threads = 1; // May be redundant
@@ -223,7 +225,7 @@ bool svm_stack_ensure_cap(HAllocator *mm__, HSVMContext *ctx, size_t addl) {
  * the second return; here, the only variables that could matter for
  * are arena and ctx (because they're referenced in "goto fail").
  */
-#ifdef __GNUC__
+#if defined(__GNUC__) && !defined(__clang__)
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunknown-pragmas"
 #pragma GCC diagnostic ignored "-Wclobbered"
@@ -257,7 +259,7 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace,
       if (!svm_stack_ensure_cap(mm__, ctx, 1)) {
 	goto fail;
       }
-      tmp_res = a_new(HParsedToken, 1);
+      tmp_res = a_new0(HParsedToken, 1);
       tmp_res->token_type = TT_MARK;
       tmp_res->index = cur->input_pos;
       tmp_res->bit_offset = 0;
@@ -288,7 +290,7 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace,
       break;
     case SVM_ACCEPT:
       assert(ctx->stack_count <= 1);
-      HParseResult *res = a_new(HParseResult, 1);
+      HParseResult *res = a_new0(HParseResult, 1);
       if (ctx->stack_count == 1) {
 	res->ast = ctx->stack[0];
       } else {
@@ -311,7 +313,7 @@ HParseResult *run_trace(HAllocator *mm__, HRVMProg *orig_prog, HRVMTrace *trace,
   return NULL;
 }
 // Reenable -Wclobber
-#ifdef __GNUC__
+#if defined(__GNUC__) && !defined(__clang__)
 #pragma GCC diagnostic pop
 #endif
 
@@ -415,7 +417,8 @@ static void h_regex_free(HParser *parser) {
   h_free(prog->actions);
   h_free(prog);
   parser->backend_data = NULL;
-  parser->backend = PB_PACKRAT;
+  parser->backend_vtable = h_get_default_backend_vtable();
+  parser->backend = h_get_default_backend();
 }
 
 static int h_regex_compile(HAllocator *mm__, HParser* parser, const void* params) {
@@ -428,9 +431,10 @@ static int h_regex_compile(HAllocator *mm__, HParser* parser, const void* params
   prog->actions = NULL;
   prog->allocator = mm__;
   if (setjmp(prog->except)) {
-    return false;
+    return 3;
   }
   if (!h_compile_regex(prog, parser)) {
+    // this shouldn't normally fail when isValidRegular() returned true
     h_free(prog->insns);
     h_free(prog->actions);
     h_free(prog);
@@ -449,7 +453,12 @@ static HParseResult *h_regex_parse(HAllocator* mm__, const HParser* parser, HInp
 HParserBackendVTable h__regex_backend_vtable = {
   .compile = h_regex_compile,
   .parse = h_regex_parse,
-  .free = h_regex_free
+  .free = h_regex_free,
+  /* Name/param resolution functions */
+  .backend_short_name = "regex",
+  .backend_description = "Regular expression matcher (broken)",
+  .get_description_with_params = h_get_description_with_no_params,
+  .get_short_name_with_params = h_get_short_name_with_no_params
 };
 
 #ifndef NDEBUG

src/benchmark.c

@@ -46,7 +46,7 @@ HBenchmarkResults *h_benchmark__m(HAllocator* mm__, HParser* parser, HParserTest
   for (backend = PB_MIN; backend <= PB_MAX; backend++) {
     ret->results[backend].backend = backend;
     // Step 1: Compile grammar for given parser...
-    if (h_compile(parser, backend, NULL) == -1) {
+    if (h_compile(parser, backend, NULL)) {
       // backend inappropriate for grammar...
       fprintf(stderr, "Compiling for %s failed\n", HParserBackendNames[backend]);
       ret->results[backend].compile_success = false;

src/bindings/jni/ConfigureJNI.py

+#!python 
+
+from __future__ import absolute_import, division, print_function
+
+import os
+import sys
+
+def walkDirs(path):
+    """helper function to get a list of all subdirectories"""
+    def addDirs(pathlist, dirname, names):
+        """internal function to pass to os.walk"""
+        print("in addDirs")
+        for n in names:
+            f = os.path.join(dirname, n)
+            if os.path.isdir(f):
+                pathlist.append(f)
+    pathlist = [path]
+    os.walk(path, addDirs, pathlist)
+    print(pathlist)
+    return pathlist
+
+def ConfigureJNI(env):
+    """Configure the given environment for compiling Java Native Interface
+       c or c++ language files."""
+    
+    print( "Configuring JNI includes")
+
+    if not env.get('JAVAC'):
+        print( "The Java compiler must be installed and in the current path.")
+        return 0
+
+    # first look for a shell variable called JAVA_HOME
+    java_base = os.environ.get('JAVA_HOME')
+    if not java_base:
+        if sys.platform == 'darwin':
+            # Apple's OS X has its own special java base directory
+            java_base = '/System/Library/Frameworks/JavaVM.framework'
+        else:
+            # Search for the java compiler
+            print ("JAVA_HOME environment variable is not set. Searching for java... ")
+            jcdir = os.path.dirname(env.WhereIs('javac'))
+            if not jcdir:
+                print( "not found.")
+                return 0
+            # assuming the compiler found is in some directory like
+            # /usr/jdkX.X/bin/javac, java's home directory is /usr/jdkX.X
+            java_base = os.path.join(jcdir, "..")
+            print( "found.")
+
+    if sys.platform == 'cygwin':
+        # Cygwin and Sun Java have different ideas of how path names
+        # are defined. Use cygpath to convert the windows path to
+        # a cygwin path. i.e. C:\jdkX.X to /cygdrive/c/jdkX.X
+        java_base = os.popen("cygpath -up '"+java_base+"'").read().replace( \
+                 '\n', '')
+
+    if sys.platform == 'darwin':
+        # Apple does not use Sun's naming convention
+        java_headers = [os.path.join(java_base, 'Headers')]
+        java_libs = [os.path.join(java_base, 'Libraries')]
+    else:
+        # windows and linux
+        java_headers = [os.path.join(java_base, 'include')]
+        java_libs = [os.path.join(java_base, 'lib')]
+        # Sun's windows and linux JDKs keep system-specific header
+        # files in a sub-directory of include
+        if java_base == '/usr' or java_base == '/usr/local':
+            # too many possible subdirectories. Just use defaults
+            java_headers.append(os.path.join(java_headers[0], 'win32'))
+            java_headers.append(os.path.join(java_headers[0], 'linux'))
+            java_headers.append(os.path.join(java_headers[0], 'solaris'))
+        else:
+            # add all subdirs of 'include'. The system specific headers
+            # should be in there somewhere
+            java_headers = walkDirs(java_headers[0])
+
+    if not any(os.path.exists(os.path.join(path, 'jni.h'))
+               for path in java_headers):
+        print("Can't find jni.h in %s" % java_headers)
+        return 0
+
+    # add Java's include and lib directory to the environment
+    java_headers.append(os.path.join(java_headers[0], 'linux'))
+    env.Append(CPPPATH = java_headers)
+    env.Append(LIBPATH = java_libs)
+
+    # add any special platform-specific compilation or linking flags
+    if sys.platform == 'darwin':
+        env.Append(SHLINKFLAGS = '-dynamiclib -framework JavaVM')
+        env['SHLIBSUFFIX'] = '.jnilib'
+    elif sys.platform == 'cygwin':
+        env.Append(CCFLAGS = '-mno-cygwin')
+        env.Append(SHLINKFLAGS = '-mno-cygwin -Wl,--kill-at')
+
+    # Add extra potentially useful environment variables
+    env['JAVA_HOME'] = java_base
+    env['JNI_CPPPATH'] = java_headers
+    env['JNI_LIBPATH'] = java_libs
+    return 1
\ No newline at end of file

jni/Example.java → src/bindings/jni/Example.java

@@ -84,6 +84,20 @@ public static void main(String args[])
     Parser i3parsers[] = {Hammer.ch((byte)'i'), Hammer.uInt8(), Hammer.int8()};
     handle(Hammer.parse(Hammer.sequence(Hammer.ch((byte)'i'), Hammer.uInt8(), Hammer.int8()), i3, i3.length));
 
+    out("permutation");
+    byte ch3[] = {(byte) 'a', (byte) 'b', (byte) 'c'};
+    handle(Hammer.parse(Hammer.permutation(Hammer.ch((byte)'a'), Hammer.ch((byte)'b'), Hammer.ch((byte)'c')), ch3, ch3.length));
+    handle(Hammer.parse(Hammer.permutation(Hammer.ch((byte)'b'), Hammer.ch((byte)'a'), Hammer.ch((byte)'c')), ch3, ch3.length));
+    
+    out("skip");
+    byte ch6[] = {(byte) 'a', (byte) 'b', (byte) 'c', (byte) 'd', (byte) 'e', (byte) 'f'};
+    handle(Hammer.parse(Hammer.sequence(Hammer.ch((byte)'a'), Hammer.skip((int)32), Hammer.ch((byte)'f')), ch6, ch6.length));
+    
+    out("seek");
+    final int SEEK_SET = 0;	/* Seek from beginning of file.  */
+    //final int SEEK_CUR = 1;	/* Seek from current position.  */
+    //final int SEEK_END = 2;	/* Seek from end of file.  */
+    handle(Hammer.parse(Hammer.sequence(Hammer.ch((byte)'a'), Hammer.seek((int)40, (int)SEEK_SET), Hammer.ch((byte)'f')), ch6, ch6.length));
     
 }
 

jni/Makefile → src/bindings/jni/Makefile

@@ -5,17 +5,18 @@ CSOURCES := com_upstandinghackers_hammer_Hammer.c com_upstandinghackers_hammer_P
 # ls *.h *.o *.so com/upstandinghackers/hammer/*.class | grep -v jhammer.h | tr '\n' ' '; replace single $ with $$
 OUTPUTS := com/upstandinghackers/hammer/Action.class com/upstandinghackers/hammer/Hammer.class com_upstandinghackers_hammer_Hammer.h com_upstandinghackers_hammer_Hammer.o com/upstandinghackers/hammer/Hammer\$TokenType.class com_upstandinghackers_hammer_Hammer_TokenType.h com/upstandinghackers/hammer/ParsedToken.class com_upstandinghackers_hammer_ParsedToken.h com_upstandinghackers_hammer_ParsedToken.o com/upstandinghackers/hammer/Parser.class com/upstandinghackers/hammer/ParseResult.class com_upstandinghackers_hammer_ParseResult.h com_upstandinghackers_hammer_ParseResult.o com_upstandinghackers_hammer_Parser.h com_upstandinghackers_hammer_Parser.o com/upstandinghackers/hammer/Predicate.class libjhammer.so
 
-TOPLEVEL := ../
+TOPLEVEL := ../../../
 
 JC=javac
 JH=javah
 CP=com/upstandinghackers/hammer
 PACKAGE=com.upstandinghackers.hammer
 
-include ../common.mk
+include ../../../common.mk
 
-JNI_INCLUDE := /usr/lib/jvm/java-6-openjdk/include/
-CFLAGS += -fPIC -I. -I $(TOPLEVEL)/src/ -I jni -I $(JNI_INCLUDE) 
+JNI_INCLUDE := /usr/lib/jvm/java-8-oracle/include/
+JNI_INCLUDE_LINUX := /usr/lib/jvm/java-8-oracle/include/linux
+CFLAGS += -fPIC -I. -I $(TOPLEVEL)/src/ -I jni -I $(JNI_INCLUDE) -I $(JNI_INCLUDE_LINUX)
 
 %.java: $(call ifsilent,| $(HUSH))
 	$(call hush, "Compiling Java source $@") $(JC) $(CP)/$@
@@ -23,7 +24,7 @@ CFLAGS += -fPIC -I. -I $(TOPLEVEL)/src/ -I jni -I $(JNI_INCLUDE)
 all: javacc prepare compile link
 
 link: compile
-	$(call hush, "Generating libjhammer.so") $(CC) -shared $(CFLAGS) -o libjhammer.so *.o ../src/*.o ../src/backends/*.o ../src/parsers/*.o
+	$(call hush, "Generating libjhammer.so") $(CC) -shared $(CFLAGS) -o libjhammer.so *.o ../../../src/*.o ../../../src/backends/*.o ../../../src/parsers/*.o
 
 $(CSOURCES): prepare
 	$(call hush, "Compiling $@") $(CC) -c $(CFLAGS) $@

src/bindings/jni/SConscript

+# -*- python -*-
+
+from __future__ import absolute_import, division, print_function
+
+import os, os.path
+import sys
+Import('env libhammer_shared testruns targets')
+from src.bindings.jni.ConfigureJNI import ConfigureJNI
+
+javaenv = env.Clone()
+
+if not ConfigureJNI(javaenv):
+    print("Java Native Interface is required... Exiting")
+    Exit(0)
+
+
+javaenv.Append(CPPPATH=[".", "../.."],
+              LIBS=['hammer'],
+              LIBPATH=["../.."])
+
+# compile java classes
+jni_classes = javaenv.Java(".", "#src/bindings/jni/com")
+
+print(jni_classes)
+jni_headers = javaenv.JavaH(".", jni_classes)
+print(jni_headers)
+Default(jni_classes)
+Default(jni_headers)
+
+#print(javaenv.Dump())
+
+shlib_env = env.Clone(CPPPATH=javaenv['JNI_CPPPATH'] + ['../..'],
+              LIBS=['hammer'],
+              LIBPATH=["../.."])
+csources = ['com_upstandinghackers_hammer_Hammer.c',
+			'com_upstandinghackers_hammer_ParsedToken.c',
+			'com_upstandinghackers_hammer_Parser.c',
+			'com_upstandinghackers_hammer_ParseResult.c']
+
+libjhammer_shared = shlib_env.SharedLibrary('libjhammer', csources)
+Default(libjhammer_shared)
+

jni/com/upstandinghackers/hammer/Hammer.java → src/bindings/jni/com/upstandinghackers/hammer/Hammer.java

@@ -53,10 +53,12 @@ public class Hammer
     public static native Parser middle(Parser p, Parser x, Parser q);
 //    public static native Parser action(Parser p, Action a);
     public static native Parser in(byte[] charset, int length);
+    public static native Parser notIn(byte[] charset, int length);
     public static native Parser endP();
     public static native Parser nothingP();
     public static native Parser sequence(Parser... parsers);
     public static native Parser choice(Parser... parsers);
+    public static native Parser permutation(Parser... parsers);
     public static native Parser butNot(Parser p1, Parser p2);
     public static native Parser difference(Parser p1, Parser p2);
     public static native Parser xor(Parser p1, Parser p2);
@@ -73,4 +75,11 @@ public class Hammer
     public static native Parser and(Parser p);
     public static native Parser not(Parser p);
     public static native Parser indirect();
+//    public static native Parser bindIndirect(Parser indirect, Parser inner);
+    public static native Parser withEndianess(byte endianess, Parser p);
+//    public static native Parser bind(Parser p, HContinuation k, void *env);
+    public static native Parser skip(int n);
+    public static native Parser seek(int n, int whence);
+    public static native Parser tell();
+    
 }

jni/com_upstandinghackers_hammer_Hammer.c → src/bindings/jni/com_upstandinghackers_hammer_Hammer.c

@@ -157,6 +157,13 @@ JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_in
 }
 
 
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_notIn
+  (JNIEnv *env, jclass class, jbyteArray charset, jint length)
+{
+    RETURNWRAP(env, h_not_in((uint8_t*) ((*env)->GetByteArrayElements(env, charset, NULL)), (size_t)length));
+}
+
+
 JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_endP
   (JNIEnv *env, jclass class)
 {
@@ -227,6 +234,34 @@ JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_choice
 }
 
 
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_permutation
+  (JNIEnv *env, jclass class, jobjectArray permutation)
+{
+    jsize length;
+    void **parsers;
+    int i;
+    jobject current;
+    const HParser *result;
+
+    length = (*env)->GetArrayLength(env, permutation);
+    parsers = malloc(sizeof(HParser *)*(length+1));
+    if(NULL==parsers)
+    {
+        return NULL;
+    }
+
+    for(i=0; i<length; i++)
+    {
+        current = (*env)->GetObjectArrayElement(env, permutation, (jsize)i);
+        parsers[i] = UNWRAP(env, current);
+    }
+    parsers[length] = NULL;
+
+    result = h_permutation__a(parsers);
+    RETURNWRAP(env, result);
+}
+
+
 JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_butNot
   (JNIEnv *env, jclass class, jobject p, jobject q)
 {
@@ -332,4 +367,29 @@ JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_indirect
 }
 
 
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_with_endianness
+  (JNIEnv *env, jclass class, jbyte endianess, jobject p)
+{
+    RETURNWRAP(env, h_with_endianness((char) endianess, UNWRAP(env, p)));
+}
 
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_skip
+  (JNIEnv *env, jclass class, jint n)
+{
+    RETURNWRAP(env, h_skip((size_t) n));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_seek
+  (JNIEnv *env, jclass class, jint offset, jint whence)
+{
+    RETURNWRAP(env, h_seek((ssize_t) offset, (int) whence));
+}
+
+
+JNIEXPORT jobject JNICALL Java_com_upstandinghackers_hammer_Hammer_tell
+  (JNIEnv *env, jclass class)
+{
+    RETURNWRAP(env, h_tell());
+}