src/glue.c

@@ -60,15 +60,8 @@ static void act_flatten_(HCountedArray *seq, const HParsedToken *tok) {
 }
 
 HParsedToken *h_act_flatten(const HParseResult *p, void* user_data) {
-  HCountedArray *seq = h_carray_new(p->arena);
-
-  act_flatten_(seq, p->ast);
-
-  HParsedToken *res = a_new_(p->arena, HParsedToken, 1);
-  res->token_type = TT_SEQUENCE;
-  res->seq = seq;
-  res->index = p->ast->index;
-  res->bit_offset = p->ast->bit_offset;
+  HParsedToken *res = h_make_seq(p->arena);
+  act_flatten_(res->seq, p->ast);
   return res;
 }
 
@@ -106,7 +99,7 @@ HParsedToken *h_make_seqn(HArena *arena, size_t n)
   return ret;
 }
 
-HParsedToken *h_make_bytes(HArena *arena, uint8_t *array, size_t len)
+HParsedToken *h_make_bytes(HArena *arena, const uint8_t *array, size_t len)
 {
   HParsedToken *ret = h_make_(arena, TT_BYTES);
   ret->bytes.len = len;
@@ -128,6 +121,20 @@ HParsedToken *h_make_uint(HArena *arena, uint64_t val)
   return ret;
 }
 
+HParsedToken *h_make_double(HArena *arena, double val)
+{
+  HParsedToken *ret = h_make_(arena, TT_DOUBLE);
+  ret->dbl = val;
+  return ret;
+}
+
+HParsedToken *h_make_float(HArena *arena, float val)
+{
+  HParsedToken *ret = h_make_(arena, TT_FLOAT);
+  ret->flt = val;
+  return ret;
+}
+
 // XXX -> internal
 HParsedToken *h_carray_index(const HCountedArray *a, size_t i)
 {

src/glue.h

@@ -195,9 +195,11 @@ HParsedToken *h_act_ignore(const HParseResult *p, void* user_data);
 HParsedToken *h_make(HArena *arena, HTokenType type, void *value);
 HParsedToken *h_make_seq(HArena *arena);  // Makes empty sequence.
 HParsedToken *h_make_seqn(HArena *arena, size_t n);  // Makes empty sequence of expected size n.
-HParsedToken *h_make_bytes(HArena *arena, uint8_t *array, size_t len);
+HParsedToken *h_make_bytes(HArena *arena, const uint8_t *array, size_t len);
 HParsedToken *h_make_sint(HArena *arena, int64_t val);
 HParsedToken *h_make_uint(HArena *arena, uint64_t val);
+HParsedToken *h_make_double(HArena *arena, double val);
+HParsedToken *h_make_float(HArena *arena, float val);
 
 // Standard short-hands to make tokens in an action.
 #define H_MAKE(TYP, VAL)  h_make(p->arena, (HTokenType)TT_ ## TYP, VAL)
@@ -206,6 +208,8 @@ HParsedToken *h_make_uint(HArena *arena, uint64_t val);
 #define H_MAKE_BYTES(VAL, LEN) h_make_bytes(p->arena, VAL, LEN)
 #define H_MAKE_SINT(VAL)  h_make_sint(p->arena, VAL)
 #define H_MAKE_UINT(VAL)  h_make_uint(p->arena, VAL)
+#define H_MAKE_DOUBLE(VAL) h_make_double(p->arena, VAL)
+#define H_MAKE_FLOAT(VAL) h_make_float(p->arena, VAL)
 
 // Extract (cast) type-specific value back from HParsedTokens...
 
@@ -218,6 +222,8 @@ HParsedToken *h_make_uint(HArena *arena, uint64_t val);
 #define H_ASSERT_BYTES(TOK)  h_assert_type(TT_BYTES, TOK)
 #define H_ASSERT_SINT(TOK)   h_assert_type(TT_SINT, TOK)
 #define H_ASSERT_UINT(TOK)   h_assert_type(TT_UINT, TOK)
+#define H_ASSERT_DOUBLE(TOK) h_assert_type(TT_DOUBLE, TOK)
+#define H_ASSERT_FLOAT(TOK)  h_assert_type(TT_FLOAT, TOK)
 
 // Assert expected type and return contained value.
 #define H_CAST(TYP, TOK)   ((TYP *) H_ASSERT(TYP, TOK)->user)
@@ -225,6 +231,8 @@ HParsedToken *h_make_uint(HArena *arena, uint64_t val);
 #define H_CAST_BYTES(TOK)  (H_ASSERT_BYTES(TOK)->bytes)
 #define H_CAST_SINT(TOK)   (H_ASSERT_SINT(TOK)->sint)
 #define H_CAST_UINT(TOK)   (H_ASSERT_UINT(TOK)->uint)
+#define H_CAST_DOUBLE(TOK) (H_ASSERT_DOUBLE(TOK)->dbl)
+#define H_CAST_FLOAT(TOK)  (H_ASSERT_FLOAT(TOK)->flt)
 
 // Sequence access...
 
@@ -247,7 +255,9 @@ HParsedToken *h_seq_index_vpath(const HParsedToken *p, size_t i, va_list va);
 #define H_INDEX_BYTES(SEQ, ...)  H_CAST_BYTES(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
 #define H_INDEX_SINT(SEQ, ...)   H_CAST_SINT(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
 #define H_INDEX_UINT(SEQ, ...)   H_CAST_UINT(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
-#define H_INDEX_TOKEN(SEQ, ...)  h_seq_index_path(SEQ, __VA_ARGS__, -1)
+#define H_INDEX_DOUBLE(SEQ, ...) H_CAST_DOUBLE(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_FLOAT(SEQ, ...)  H_CAST_FLOAT(H_INDEX_TOKEN(SEQ, __VA_ARGS__))
+#define H_INDEX_TOKEN(SEQ, ...)  h_seq_index_path(H_ASSERT_SEQ(SEQ), __VA_ARGS__, -1)
 
 // Standard short-hand to access and cast elements on a sequence token.
 #define H_FIELD(TYP, ...)  H_INDEX(TYP, p->ast, __VA_ARGS__)
@@ -255,6 +265,9 @@ HParsedToken *h_seq_index_vpath(const HParsedToken *p, size_t i, va_list va);
 #define H_FIELD_BYTES(...) H_INDEX_BYTES(p->ast, __VA_ARGS__)
 #define H_FIELD_SINT(...)  H_INDEX_SINT(p->ast, __VA_ARGS__)
 #define H_FIELD_UINT(...)  H_INDEX_UINT(p->ast, __VA_ARGS__)
+#define H_FIELD_DOUBLE(...) H_INDEX_DOUBLE(p->ast, __VA_ARGS__)
+#define H_FIELD_FLOAT(...) H_INDEX_FLOAT(p->ast, __VA_ARGS__)
+#define H_FIELD_TOKEN(...) H_INDEX_TOKEN(p->ast, __VA_ARGS__)
 
 // Lower-level helper for h_seq_index.
 HParsedToken *h_carray_index(const HCountedArray *a, size_t i); // XXX -> internal

src/parsers/and.c

@@ -3,18 +3,18 @@
 static HParseResult *parse_and(void* env, HParseState* state) {
   HInputStream bak = state->input_stream;
   HParseResult *res = h_do_parse((HParser*)env, state);
+  if (!res)
+    return NULL;	// propagate failed input state, esp. overrun
   state->input_stream = bak;
-  if (res)
-    return make_result(state->arena, NULL);
-  return NULL;
+  return make_result(state->arena, NULL);
 }
 
 static const HParserVtable and_vt = {
   .parse = parse_and,
   .isValidRegular = h_false, /* TODO: strictly speaking this should be regular,
-				but it will be a huge amount of work and difficult
-				to get right, so we're leaving it for a future
-				revision. --mlp, 18/12/12 */
+				but it will be a huge amount of work and
+				difficult to get right, so we're leaving it for
+				a future revision. --mlp, 18/12/12 */
   .isValidCF = h_false,      /* despite TODO above, this remains false. */
   .compile_to_rvm = h_not_regular,
   .higher = true,

src/parsers/bind.c

@@ -21,8 +21,7 @@ static void *aa_alloc(HAllocator *allocator, size_t size)
 static void *aa_realloc(HAllocator *allocator, void *ptr, size_t size)
 {
     HArena *arena = ((ArenaAllocator *)allocator)->arena;
-    assert(((void)"XXX need realloc for arena allocator", 0));
-    return NULL;
+    return h_arena_realloc(arena, ptr, size);
 }
 
 static void aa_free(HAllocator *allocator, void *ptr)

src/parsers/bits.c

@@ -14,6 +14,9 @@ static HParseResult* parse_bits(void* env, HParseState *state) {
     result->sint = h_read_bits(&state->input_stream, env_->length, true);
   else
     result->uint = h_read_bits(&state->input_stream, env_->length, false);
+  result->index = 0;
+  result->bit_length = 0;
+  result->bit_offset = 0;
   return make_result(state->arena, result);
 }
 
@@ -29,7 +32,7 @@ static HParsedToken *reshape_bits(const HParseResult *p, void* signedp_p) {
   HParsedToken *ret = h_arena_malloc(p->arena, sizeof(HParsedToken));
   ret->token_type = TT_UINT;
 
-  if(signedp && (seq->elements[0]->uint & 128))
+  if(signedp && seq->used > 0 && (seq->elements[0]->uint & 128))
     ret->uint = -1; // all ones
 
   for(size_t i=0; i<seq->used; i++) {

src/parsers/butnot.c

@@ -5,7 +5,6 @@ typedef struct {
   const HParser *p2;
 } HTwoParsers;
 
-
 static HParseResult* parse_butnot(void *env, HParseState *state) {
   HTwoParsers *parsers = (HTwoParsers*)env;
   // cache the initial state of the input stream
@@ -19,15 +18,18 @@ static HParseResult* parse_butnot(void *env, HParseState *state) {
   HInputStream after_p1_state = state->input_stream;
   state->input_stream = start_state;
   HParseResult *r2 = h_do_parse(parsers->p2, state);
-  // TODO(mlp): I'm pretty sure the input stream state should be the post-p1 state in all cases
+  // don't touch the input state (overrun flag) if we must suspend
+  if (want_suspend(state)) {
+    return NULL;
+  }
+  // in all other cases, the input stream should be in the post-p1 state
   state->input_stream = after_p1_state;
-  // if p2 failed, restore post-p1 state and bail out early
   if (NULL == r2) {
     return r1;
   }
   size_t r1len = token_length(r1);
   size_t r2len = token_length(r2);
-  // if both match but p1's text is shorter than than p2's (or the same length), fail
+  // if both match but p1's text no longer than p2's, fail
   if (r1len <= r2len) {
     return NULL;
   } else {

src/parsers/bytes.c

+#include "parser_internal.h"
+
+struct bytes_env {
+  size_t length;
+};
+
+static HParseResult *parse_bytes(void *env_, HParseState *state)
+{
+  struct bytes_env *env = env_;
+  uint8_t *bs;
+  size_t i;
+
+  bs = a_new(uint8_t, env->length);
+  for (i=0; i < env->length && !state->input_stream.overrun; i++)
+    bs[i] = h_read_bits(&state->input_stream, 8, false);
+
+  HParsedToken *result = a_new(HParsedToken, 1);
+  result->token_type = TT_BYTES;
+  result->bytes.token = bs;
+  result->bytes.len = env->length;
+  result->index = 0;
+  result->bit_length = 0;
+  result->bit_offset = 0;
+  return make_result(state->arena, result);
+}
+
+static const HParserVtable bytes_vt = {
+  .parse = parse_bytes,
+  .isValidRegular = h_false,	// XXX need desugar_bytes, reshape_bytes
+  .isValidCF = h_false,		// XXX need bytes_ctrvm
+};
+
+HParser *h_bytes(size_t len)
+{
+  return h_bytes__m(&system_allocator, len);
+}
+
+HParser *h_bytes__m(HAllocator *mm__, size_t len)
+{
+  struct bytes_env *env = h_new(struct bytes_env, 1);
+  env->length = len;
+  return h_new_parser(mm__, &bytes_vt, env);
+}

src/parsers/ch.c

@@ -8,6 +8,9 @@ static HParseResult* parse_ch(void* env, HParseState *state) {
   if (c == r) {
     HParsedToken *tok = a_new(HParsedToken, 1);    
     tok->token_type = TT_UINT; tok->uint = r;
+    tok->index = 0;
+    tok->bit_length = 0;
+    tok->bit_offset = 0;
     return make_result(state->arena, tok);
   } else {
     return NULL;

src/parsers/charset.c

@@ -10,6 +10,9 @@ static HParseResult* parse_charset(void *env, HParseState *state) {
   if (charset_isset(cs, in)) {
     HParsedToken *tok = a_new(HParsedToken, 1);
     tok->token_type = TT_UINT; tok->uint = in;
+    tok->index = 0;
+    tok->bit_length = 0;
+    tok->bit_offset = 0;
     return make_result(state->arena, tok);    
   } else
     return NULL;

src/parsers/ignore.c

@@ -8,6 +8,7 @@ static HParseResult* parse_ignore(void* env, HParseState* state) {
   HParseResult *res = a_new(HParseResult, 1);
   res->ast = NULL;
   res->arena = state->arena;
+  res->bit_length = 0;
   return res;
 }