src/parsers/optional.c

@@ -6,9 +6,14 @@ static HParseResult* parse_optional(void* env, HParseState* state) {
   HParseResult *res0 = h_do_parse((HParser*)env, state);
   if (res0)
     return res0;
+  if (want_suspend(state))
+    return NULL;		// bail out early, leaving overrun flag
   state->input_stream = bak;
   HParsedToken *ast = a_new(HParsedToken, 1);
   ast->token_type = TT_NONE;
+  ast->index = 0;
+  ast->bit_length = 0;
+  ast->bit_offset = 0;
   return make_result(state->arena, ast);
 }
 

src/parsers/parser_internal.h

 /*
  * NOTE: This is an internal header and installed for use by extensions. The
  * API is not guaranteed stable.
-*/
+ */
 
-#ifndef HAMMER_PARSE_INTERNAL__H
-#define HAMMER_PARSE_INTERNAL__H
+#ifndef HAMMER_PARSER_INTERNAL__H
+#define HAMMER_PARSER_INTERNAL__H
 #include "../hammer.h"
 #include "../internal.h"
 #include "../backends/regex.h"
 #include "../backends/contextfree.h"
 
-#define a_new_(arena, typ, count) ((typ*)h_arena_malloc((arena), sizeof(typ)*(count)))
+#define a_new_(arena, typ, count) ((typ*)h_arena_malloc_noinit((arena), sizeof(typ)*(count)))
 #define a_new(typ, count) a_new_(state->arena, typ, count)
-// we can create a_new0 if necessary. It would allocate some memory and immediately zero it out.
+#define a_new0_(arena, typ, count) ((typ*)h_arena_malloc((arena), sizeof(typ)*(count)))
+#define a_new0(typ, count) a_new0_(state->arena, typ, count)
 
 static inline HParseResult* make_result(HArena *arena, HParsedToken *tok) {
-  HParseResult *ret = h_arena_malloc(arena, sizeof(HParseResult));
+  HParseResult *ret = h_arena_malloc_noinit(arena, sizeof(HParseResult));
   ret->ast = tok;
   ret->arena = arena;
   ret->bit_length = 0; // This way it gets overridden in h_do_parse
@@ -31,6 +32,11 @@ static inline size_t token_length(HParseResult *pr) {
   }
 }
 
+// tell if we should suspend a parser, i.e. overrun occured before final input
+static inline bool want_suspend(HParseState *state) {
+  return state->input_stream.overrun && !state->input_stream.last_chunk;
+}
+
 /* Epsilon rules happen during desugaring. This handles them. */
 static inline void desugar_epsilon(HAllocator *mm__, HCFStack *stk__, void *env) {
   HCFS_BEGIN_CHOICE() {
@@ -40,4 +46,4 @@ static inline void desugar_epsilon(HAllocator *mm__, HCFStack *stk__, void *env)
   } HCFS_END_CHOICE();
 }
 
-#endif // HAMMER_PARSE_INTERNAL__H
+#endif // HAMMER_PARSER_INTERNAL__H

src/parsers/permutation.c

@@ -29,6 +29,9 @@ static int parse_permutation_tail(const HSequence *s,
     if(set[i]) {
       match = h_do_parse(ps[i], state);
 
+      if (want_suspend(state))
+        return 0;		// bail out early, leaving overrun flag
+
       // save result
       if(match)
 	seq->elements[i] = (void *)match->ast;
@@ -89,6 +92,9 @@ static HParseResult *parse_permutation(void *env, HParseState *state)
     HParsedToken *tok = a_new(HParsedToken, 1);
     tok->token_type  = TT_SEQUENCE;
     tok->seq = seq;
+    tok->index = 0;
+    tok->bit_length = 0;
+    tok->bit_offset = 0;
     return make_result(state->arena, tok);
   } else {
     // no parse
@@ -175,7 +181,8 @@ HParser* h_permutation__ma(HAllocator* mm__, void *args[]) {
   HParser *ret = h_new(HParser, 1);
   ret->vtable = &permutation_vt; 
   ret->env = (void*)s;
-  ret->backend = PB_MIN;
+  ret->backend = h_get_default_backend();
+  ret->backend_vtable = h_get_default_backend_vtable();
   ret->desugared = NULL;
   return ret;
 }

src/parsers/seek.c

+#include "parser_internal.h"
+
+typedef struct {
+  ssize_t offset;
+  int whence;
+} HSeek;
+
+static HParseResult *parse_skip(void *env, HParseState *state)
+{
+  size_t n = (uintptr_t)env;
+
+  h_skip_bits(&state->input_stream, n);
+  return make_result(state->arena, NULL);
+}
+
+static HParseResult *parse_seek(void *env, HParseState *state)
+{
+  HSeek *s = (HSeek *)env;
+  HInputStream *stream = &state->input_stream;
+  size_t pos;
+
+  /* determine base position */
+  switch (s->whence) {
+  case SEEK_SET:
+    pos = 0;
+    break;
+  case SEEK_END:
+    if (!stream->last_chunk) {	/* the end is not yet known! */
+      stream->overrun = true;	/* we need more input */
+      return NULL;
+    }
+    pos = h_input_stream_length(stream);
+    break;
+  case SEEK_CUR:
+    pos = h_input_stream_pos(stream);
+    break;
+  default:
+    return NULL;	/* invalid argument */
+  }
+
+  /* calculate target position and do basic overflow checks */
+  if (s->offset < 0 && (size_t)(- s->offset) > pos)
+    return NULL;	/* underflow */
+  if (s->offset > 0 && SIZE_MAX - s->offset < pos)
+    return NULL;	/* overflow */
+  pos += s->offset;
+
+  /* perform the seek and check for overrun */
+  h_seek_bits(stream, pos);
+  if (stream->overrun)
+    return NULL;
+
+  HParsedToken *tok = a_new(HParsedToken, 1);
+  tok->token_type = TT_UINT;
+  tok->uint = pos;
+  tok->index = 0;
+  tok->bit_length = 0;
+  tok->bit_offset = 0;
+  return make_result(state->arena, tok);
+}
+
+static HParseResult *parse_tell(void *env, HParseState *state)
+{
+  HParsedToken *tok = a_new(HParsedToken, 1);
+  tok->token_type = TT_UINT;
+  tok->uint = h_input_stream_pos(&state->input_stream);
+  tok->index = 0;
+  tok->bit_length = 0;
+  tok->bit_offset = 0;
+  return make_result(state->arena, tok);
+}
+
+static const HParserVtable skip_vt = {
+  .parse = parse_skip,
+  .isValidRegular = h_false,
+  .isValidCF = h_false,
+  .compile_to_rvm = h_not_regular,
+  .higher = false,
+};
+
+static const HParserVtable seek_vt = {
+  .parse = parse_seek,
+  .isValidRegular = h_false,
+  .isValidCF = h_false,
+  .compile_to_rvm = h_not_regular,
+  .higher = false,
+};
+
+static const HParserVtable tell_vt = {
+  .parse = parse_tell,
+  .isValidRegular = h_false,
+  .isValidCF = h_false,
+  .compile_to_rvm = h_not_regular,
+  .higher = false,
+};
+
+HParser* h_skip(size_t n)
+{
+  return h_skip__m(&system_allocator, n);
+}
+
+HParser *h_skip__m(HAllocator* mm__, size_t n)
+{
+  return h_new_parser(mm__, &skip_vt, (void *)n);
+}
+
+HParser* h_seek(ssize_t offset, int whence)
+{
+  return h_seek__m(&system_allocator, offset, whence);
+}
+
+HParser *h_seek__m(HAllocator* mm__, ssize_t offset, int whence)
+{
+  HSeek *env = h_new(HSeek, 1);
+  env->offset = offset;
+  env->whence = whence;
+  return h_new_parser(mm__, &seek_vt, env);
+}
+
+HParser *h_tell()
+{
+  return h_tell__m(&system_allocator);
+}
+
+HParser *h_tell__m(HAllocator* mm__)
+{
+  return h_new_parser(mm__, &tell_vt, NULL);
+}

src/parsers/sequence.c

@@ -22,6 +22,9 @@ static HParseResult* parse_sequence(void *env, HParseState *state) {
   }
   HParsedToken *tok = a_new(HParsedToken, 1);
   tok->token_type = TT_SEQUENCE; tok->seq = seq;
+  tok->index = 0;
+  tok->bit_offset = 0;
+  tok->bit_length = 0;
   return make_result(state->arena, tok);
 }
 
@@ -60,6 +63,7 @@ static HParsedToken *reshape_sequence(const HParseResult *p, void* user_data) {
   res->seq = seq;
   res->index = p->ast->index;
   res->bit_offset = p->ast->bit_offset;
+  res->bit_length = p->bit_length;
 
   return res;
 }
@@ -170,7 +174,90 @@ HParser* h_sequence__ma(HAllocator* mm__, void *args[]) {
   HParser *ret = h_new(HParser, 1);
   ret->vtable = &sequence_vt; 
   ret->env = (void*)s; 
-  ret->backend = PB_MIN;
+  ret->backend = h_get_default_backend();
+  ret->backend_vtable = h_get_default_backend_vtable();
   ret->desugared = NULL;
   return ret;
 }
+
+HParser* h_drop_from_(HParser* p, ...) {
+  assert_message(p->vtable == &sequence_vt, "drop_from requires a sequence parser");
+  va_list ap;
+  va_start(ap, p);
+  HParser* ret = h_drop_from___mv(&system_allocator, p, ap);
+  va_end(ap);
+  return ret;
+}
+
+HParser* h_drop_from___m(HAllocator* mm__, HParser* p, ...) {
+  assert_message(p->vtable == &sequence_vt, "drop_from requires a sequence parser");
+  va_list ap;
+  va_start(ap, p);
+  HParser* ret = h_drop_from___mv(mm__, p, ap);
+  va_end(ap);
+  return ret;
+}
+
+HParser* h_drop_from___v(HParser* p, va_list ap) {
+  assert_message(p->vtable == &sequence_vt, "drop_from requires a sequence parser");
+  return h_drop_from___mv(&system_allocator, p, ap);
+}
+
+HParser* h_drop_from___mv(HAllocator* mm__, HParser *p, va_list ap) {
+  /* Ok, here's where things get funny.
+   *
+   * Saying `h_drop_from(h_sequence(a, b, c, d, e, NULL), 0, 4, -1)` is functionally
+   * equivalent to `h_sequence(h_ignore(a), b, c, d, h_ignore(e), NULL)`. Thus, this
+   * term rewrites itself, becoming an h_sequence where some parsers are ignored.
+   */
+  HSequence *s = (HSequence*)(p->env);
+  size_t indices[s->len];
+  size_t count = 0;
+  int arg = 0;
+  
+  for (arg = va_arg(ap, int); arg >= 0; arg = va_arg(ap, int)) {
+    indices[count] = arg;
+    count++;
+  }
+  va_end(ap);
+
+  HSequence *rewrite = h_new(HSequence, 1);
+  rewrite->p_array = h_new(HParser *, s->len);
+  rewrite->len = s->len;
+  for (size_t i=0, j=0; i<s->len; ++i) {
+    if (indices[j]==i) {
+      rewrite->p_array[i] = h_ignore(s->p_array[i]);
+      ++j;
+    } else {
+      rewrite->p_array[i] = s->p_array[i];
+    }
+  }
+  
+  return h_new_parser(mm__, &sequence_vt, rewrite);
+}
+
+HParser* h_drop_from___a(void *args[]) {
+  return h_drop_from___ma(&system_allocator, args);
+}
+
+HParser* h_drop_from___ma(HAllocator* mm__, void *args[]) {
+  HParser *p = (HParser*)(args[0]);
+  assert_message(p->vtable == &sequence_vt, "drop_from requires a sequence parser");
+  HSequence *s = (HSequence*)(p->env);
+  HSequence *rewrite = h_new(HSequence, 1);
+  rewrite->p_array = h_new(HParser *, s->len);
+  rewrite->len = s->len;
+
+  int i=0, *argp = (int*)(args[1]);
+  while (*argp >= 0) {
+    if (i == *argp) {
+      rewrite->p_array[i] = h_ignore(s->p_array[i]);
+      ++argp;
+    } else {
+      rewrite->p_array[i] = s->p_array[i];
+    }
+    ++i;
+  }
+
+  return h_new_parser(mm__, &sequence_vt, rewrite);
+}