From a4179fca026f312a32a4b4d22101ab83d615c077 Mon Sep 17 00:00:00 2001
From: "Sven M. Hallberg" <pesco@khjk.org>
Date: Thu, 17 Sep 2015 00:33:14 +0200
Subject: [PATCH] note why token position is not set correctly by LL(k) right
now
---
src/backends/llk.c | 18 ++++++++++++++----
src/t_parser.c | 2 +-
2 files changed, 15 insertions(+), 5 deletions(-)
diff --git a/src/backends/llk.c b/src/backends/llk.c
index 865c30e6..0ab4610a 100644
--- a/src/backends/llk.c
+++ b/src/backends/llk.c
@@ -438,13 +438,12 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser,
// the top of stack is such that there will be a result...
tok = h_arena_malloc(arena, sizeof(HParsedToken));
- tok->index = stream->pos + stream->index;
- tok->bit_offset = stream->bit_offset;
if(x == MARK) {
// hit stack frame boundary...
// wrap the accumulated parse result, this sequence is finished
tok->token_type = TT_SEQUENCE;
tok->seq = seq;
+ // XXX would have to set token pos but we've forgotten pos of seq
// recover original nonterminal and result sequence
x = h_slist_pop(stack);
@@ -454,6 +453,9 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser,
else {
// x is a terminal or simple charset; match against input
+ tok->index = stream->pos + stream->index;
+ tok->bit_offset = stream->bit_offset;
+
// consume the input token
uint8_t input = h_read_bits(stream, 8, false);
@@ -500,8 +502,16 @@ static HCountedArray *llk_parse_chunk_(HLLkState *s, const HParser* parser,
// 'tok' has been parsed; process it
// perform token reshape if indicated
- if(x->reshape)
- tok = (HParsedToken *)x->reshape(make_result(arena, tok), x->user_data);
+ if(x->reshape) {
+ HParsedToken *t = x->reshape(make_result(arena, tok), x->user_data);
+ if(t) {
+ t->index = tok->index;
+ t->bit_offset = tok->bit_offset;
+ } else {
+ h_arena_free(arena, tok);
+ }
+ tok = t;
+ }
// call validation and semantic action, if present
if(x->pred && !x->pred(make_result(tarena, tok), x->user_data))
diff --git a/src/t_parser.c b/src/t_parser.c
index 4268ee79..c42eca91 100644
--- a/src/t_parser.c
+++ b/src/t_parser.c
@@ -832,7 +832,7 @@ void register_parser_tests(void) {
//g_test_add_data_func("/core/parser/llk/leftrec", GINT_TO_POINTER(PB_LLk), test_leftrec);
g_test_add_data_func("/core/parser/llk/rightrec", GINT_TO_POINTER(PB_LLk), test_rightrec);
g_test_add_data_func("/core/parser/llk/result_length", GINT_TO_POINTER(PB_LLk), test_result_length);
- //XXX g_test_add_data_func("/core/parser/llk/token_position", GINT_TO_POINTER(PB_LLk), test_token_position);
+ //g_test_add_data_func("/core/parser/llk/token_position", GINT_TO_POINTER(PB_LLk), test_token_position);
g_test_add_data_func("/core/parser/llk/iterative", GINT_TO_POINTER(PB_LLk), test_iterative);
g_test_add_data_func("/core/parser/llk/iterative/lookahead", GINT_TO_POINTER(PB_LLk), test_iterative_lookahead);
g_test_add_data_func("/core/parser/llk/iterative/result_length", GINT_TO_POINTER(PB_LLk), test_iterative_result_length);
--
GitLab