From 3d791412f0eff46e6a052c79a30e5213153361a1 Mon Sep 17 00:00:00 2001
From: Dan Hirsch <thequux@upstandinghackers.com>
Date: Sat, 11 Jan 2014 03:24:39 +0100
Subject: [PATCH] Fixed character parsing
---
lib/tsgenruby.pl | 4 +-
src/bindings/ruby/lib/hammer.rb | 2 +-
src/bindings/ruby/lib/hammer/parser.rb | 20 ++--
src/bindings/ruby/test/autogen_test.rb | 144 ++++++++++++-------------
src/bindings/ruby/test/parser_test.rb | 20 ++++
5 files changed, 108 insertions(+), 82 deletions(-)
diff --git a/lib/tsgenruby.pl b/lib/tsgenruby.pl
index ad83cf16..d866eeed 100644
--- a/lib/tsgenruby.pl
+++ b/lib/tsgenruby.pl
@@ -96,8 +96,8 @@ pp_parser(num(Num)) --> !,
"-0x", {RNum is -Num}; "0x", {RNum = Num} ),
pp_hexnum_guts(RNum).
pp_parser(char(C)) --> !,
- pp_parser(num(C)).
- %"'", pp_char_guts(C), "'", !.
+ pp_parser(num(C)), ".chr". % Ruby is encoding-aware; this is a
+ % more reasonable implementation
pp_parser(ref(Name)) -->
{atom_codes(Name,CName)},
diff --git a/src/bindings/ruby/lib/hammer.rb b/src/bindings/ruby/lib/hammer.rb
index 79fb52dc..916a0a50 100644
--- a/src/bindings/ruby/lib/hammer.rb
+++ b/src/bindings/ruby/lib/hammer.rb
@@ -48,5 +48,5 @@ $r = parser.parse 'abcdefgh'
h = Hammer::Parser
-parser = h.many(h.attr_bool(h.uint8) { |r| r.ast.data <= 100 })
+parser = h.many(h.attr_bool(h.uint8) { |r| r.data <= 100 })
#p parser.parse('abcdefgh').ast.data.map(&:data)
diff --git a/src/bindings/ruby/lib/hammer/parser.rb b/src/bindings/ruby/lib/hammer/parser.rb
index cdd2c349..3a11d2ec 100644
--- a/src/bindings/ruby/lib/hammer/parser.rb
+++ b/src/bindings/ruby/lib/hammer/parser.rb
@@ -81,8 +81,10 @@ module Hammer
predicate = block if predicate.nil?
raise ArgumentError, 'no predicate' if predicate.nil?
- h_parser = Hammer::Internal.h_attr_bool(parser.h_parser, predicate)
- return Hammer::Parser.new(:attr_bool, h_parser, [parser, predicate])
+ real_pred = Proc.new {|hpr| predicate.call hpr.ast}
+
+ h_parser = Hammer::Internal.h_attr_bool(parser.h_parser, real_pred)
+ return Hammer::Parser.new(:attr_bool, h_parser, [parser, predicate, real_pred])
end
def self.token(string)
@@ -118,19 +120,23 @@ module Hammer
return num
end
private_class_method :marshal_ch_arg
-
+
+ def self.ch_parser_wrapper(parser)
+ return Hammer::Parser.action(parser) {|x| x.data.chr}
+ end
+
def self.ch(ch)
num = marshal_ch_arg(ch)
h_parser = Hammer::Internal.h_ch(num)
- return Hammer::Parser.new(:ch, h_parser, nil)
+ return ch_parser_wrapper(Hammer::Parser.new(:ch, h_parser, nil))
end
def self.ch_range(ch1, ch2)
ch1 = marshal_ch_arg(ch1)
ch2 = marshal_ch_arg(ch2)
h_parser = Hammer::Internal.h_ch_range(ch1, ch2)
- return Hammer::Parser.new(:ch_range, h_parser, nil)
+ return ch_parser_wrapper(Hammer::Parser.new(:ch_range, h_parser, nil))
end
def self.int_range(parser, i1, i2)
@@ -142,7 +148,7 @@ module Hammer
raise ArgumentError, "Expected a String" unless charset.is_a?(String)
ibuf = FFI::MemoryPointer.from_string(charset)
h_parser = Hammer::Internal.h_in(ibuf, charset.bytesize)
- return Hammer::Parser.new(:in, h_parser, nil)
+ return ch_parser_wrapper(Hammer::Parser.new(:in, h_parser, nil))
end
def self.repeat_n(parser, count)
@@ -154,7 +160,7 @@ module Hammer
raise ArgumentError, "Expected a String" unless charset.is_a?(String)
ibuf = FFI::MemoryPointer.from_string(charset)
h_parser = Hammer::Internal.h_not_in(ibuf, charset.bytesize)
- return Hammer::Parser.new(:not_in, h_parser, nil)
+ return ch_parser_wrapper(Hammer::Parser.new(:not_in, h_parser, nil))
end
# Defines a parser constructor with the given name.
diff --git a/src/bindings/ruby/test/autogen_test.rb b/src/bindings/ruby/test/autogen_test.rb
index 93a0c7a5..0600c0f6 100644
--- a/src/bindings/ruby/test/autogen_test.rb
+++ b/src/bindings/ruby/test/autogen_test.rb
@@ -25,7 +25,7 @@ class TestCh < Minitest::Test
end
def test_1
- assert_parse_ok @parser_1, "\xa2", 0xa2
+ assert_parse_ok @parser_1, "\xa2", 0xa2.chr
end
def test_2
@@ -41,7 +41,7 @@ class TestChRange < Minitest::Test
end
def test_1
- assert_parse_ok @parser_1, "b", 0x62
+ assert_parse_ok @parser_1, "b", 0x62.chr
end
def test_2
@@ -218,19 +218,19 @@ class TestWhitespace < Minitest::Test
end
def test_1
- assert_parse_ok @parser_1, "a", 0x61
+ assert_parse_ok @parser_1, "a", 0x61.chr
end
def test_2
- assert_parse_ok @parser_1, " a", 0x61
+ assert_parse_ok @parser_1, " a", 0x61.chr
end
def test_3
- assert_parse_ok @parser_1, " a", 0x61
+ assert_parse_ok @parser_1, " a", 0x61.chr
end
def test_4
- assert_parse_ok @parser_1, "\x09a", 0x61
+ assert_parse_ok @parser_1, "\x09a", 0x61.chr
end
def test_5
@@ -258,7 +258,7 @@ class TestLeft < Minitest::Test
end
def test_1
- assert_parse_ok @parser_1, "a ", 0x61
+ assert_parse_ok @parser_1, "a ", 0x61.chr
end
def test_2
@@ -278,11 +278,11 @@ class TestMiddle < Minitest::Test
def setup
super
h = Hammer::Parser
- @parser_1 = h.middle(h.ch(0x20), h.ch(0x61), h.ch(0x20))
+ @parser_1 = h.middle(h.ch(0x20.chr), h.ch(0x61.chr), h.ch(0x20.chr))
end
def test_1
- assert_parse_ok @parser_1, " a ", 0x61
+ assert_parse_ok @parser_1, " a ", 0x61.chr
end
def test_2
@@ -318,7 +318,7 @@ class TestIn < Minitest::Test
end
def test_1
- assert_parse_ok @parser_1, "b", 0x62
+ assert_parse_ok @parser_1, "b", 0x62.chr
end
def test_2
@@ -334,7 +334,7 @@ class TestNotIn < Minitest::Test
end
def test_1
- assert_parse_ok @parser_1, "d", 0x64
+ assert_parse_ok @parser_1, "d", 0x64.chr
end
def test_2
@@ -346,11 +346,11 @@ class TestEndP < Minitest::Test
def setup
super
h = Hammer::Parser
- @parser_1 = h.sequence(h.ch(0x61), h.end_p)
+ @parser_1 = h.sequence(h.ch(0x61.chr), h.end_p)
end
def test_1
- assert_parse_ok @parser_1, "a", [0x61]
+ assert_parse_ok @parser_1, "a", [0x61.chr]
end
def test_2
@@ -374,12 +374,12 @@ class TestSequence < Minitest::Test
def setup
super
h = Hammer::Parser
- @parser_1 = h.sequence(h.ch(0x61), h.ch(0x62))
- @parser_2 = h.sequence(h.ch(0x61), h.whitespace(h.ch(0x62)))
+ @parser_1 = h.sequence(h.ch(0x61.chr), h.ch(0x62.chr))
+ @parser_2 = h.sequence(h.ch(0x61.chr), h.whitespace(h.ch(0x62.chr)))
end
def test_1
- assert_parse_ok @parser_1, "ab", [0x61, 0x62]
+ assert_parse_ok @parser_1, "ab", [0x61.chr, 0x62.chr]
end
def test_2
@@ -391,15 +391,15 @@ class TestSequence < Minitest::Test
end
def test_4
- assert_parse_ok @parser_2, "ab", [0x61, 0x62]
+ assert_parse_ok @parser_2, "ab", [0x61.chr, 0x62.chr]
end
def test_5
- assert_parse_ok @parser_2, "a b", [0x61, 0x62]
+ assert_parse_ok @parser_2, "a b", [0x61.chr, 0x62.chr]
end
def test_6
- assert_parse_ok @parser_2, "a b", [0x61, 0x62]
+ assert_parse_ok @parser_2, "a b", [0x61.chr, 0x62.chr]
end
end
@@ -407,19 +407,19 @@ class TestChoice < Minitest::Test
def setup
super
h = Hammer::Parser
- @parser_1 = h.choice(h.ch(0x61), h.ch(0x62))
+ @parser_1 = h.choice(h.ch(0x61.chr), h.ch(0x62.chr))
end
def test_1
- assert_parse_ok @parser_1, "a", 0x61
+ assert_parse_ok @parser_1, "a", 0x61.chr
end
def test_2
- assert_parse_ok @parser_1, "b", 0x62
+ assert_parse_ok @parser_1, "b", 0x62.chr
end
def test_3
- assert_parse_ok @parser_1, "ab", 0x61
+ assert_parse_ok @parser_1, "ab", 0x61.chr
end
def test_4
@@ -431,12 +431,12 @@ class TestButnot < Minitest::Test
def setup
super
h = Hammer::Parser
- @parser_1 = h.butnot(h.ch(0x61), h.token("ab"))
- @parser_2 = h.butnot(h.ch_range(0x30, 0x39), h.ch(0x36))
+ @parser_1 = h.butnot(h.ch(0x61.chr), h.token("ab"))
+ @parser_2 = h.butnot(h.ch_range(0x30.chr, 0x39.chr), h.ch(0x36.chr))
end
def test_1
- assert_parse_ok @parser_1, "a", 0x61
+ assert_parse_ok @parser_1, "a", 0x61.chr
end
def test_2
@@ -444,11 +444,11 @@ class TestButnot < Minitest::Test
end
def test_3
- assert_parse_ok @parser_1, "aa", 0x61
+ assert_parse_ok @parser_1, "aa", 0x61.chr
end
def test_4
- assert_parse_ok @parser_2, "5", 0x35
+ assert_parse_ok @parser_2, "5", 0x35.chr
end
def test_5
@@ -460,7 +460,7 @@ class TestDifference < Minitest::Test
def setup
super
h = Hammer::Parser
- @parser_1 = h.difference(h.token("ab"), h.ch(0x61))
+ @parser_1 = h.difference(h.token("ab"), h.ch(0x61.chr))
end
def test_1
@@ -476,15 +476,15 @@ class TestXor < Minitest::Test
def setup
super
h = Hammer::Parser
- @parser_1 = h.xor(h.ch_range(0x30, 0x36), h.ch_range(0x35, 0x39))
+ @parser_1 = h.xor(h.ch_range(0x30.chr, 0x36.chr), h.ch_range(0x35.chr, 0x39.chr))
end
def test_1
- assert_parse_ok @parser_1, "0", 0x30
+ assert_parse_ok @parser_1, "0", 0x30.chr
end
def test_2
- assert_parse_ok @parser_1, "9", 0x39
+ assert_parse_ok @parser_1, "9", 0x39.chr
end
def test_3
@@ -500,7 +500,7 @@ class TestMany < Minitest::Test
def setup
super
h = Hammer::Parser
- @parser_1 = h.many(h.choice(h.ch(0x61), h.ch(0x62)))
+ @parser_1 = h.many(h.choice(h.ch(0x61.chr), h.ch(0x62.chr)))
end
def test_1
@@ -508,15 +508,15 @@ class TestMany < Minitest::Test
end
def test_2
- assert_parse_ok @parser_1, "a", [0x61]
+ assert_parse_ok @parser_1, "a", [0x61.chr]
end
def test_3
- assert_parse_ok @parser_1, "b", [0x62]
+ assert_parse_ok @parser_1, "b", [0x62.chr]
end
def test_4
- assert_parse_ok @parser_1, "aabbaba", [0x61, 0x61, 0x62, 0x62, 0x61, 0x62, 0x61]
+ assert_parse_ok @parser_1, "aabbaba", [0x61.chr, 0x61.chr, 0x62.chr, 0x62.chr, 0x61.chr, 0x62.chr, 0x61.chr]
end
end
@@ -524,7 +524,7 @@ class TestMany1 < Minitest::Test
def setup
super
h = Hammer::Parser
- @parser_1 = h.many1(h.choice(h.ch(0x61), h.ch(0x62)))
+ @parser_1 = h.many1(h.choice(h.ch(0x61.chr), h.ch(0x62.chr)))
end
def test_1
@@ -532,15 +532,15 @@ class TestMany1 < Minitest::Test
end
def test_2
- assert_parse_ok @parser_1, "a", [0x61]
+ assert_parse_ok @parser_1, "a", [0x61.chr]
end
def test_3
- assert_parse_ok @parser_1, "b", [0x62]
+ assert_parse_ok @parser_1, "b", [0x62.chr]
end
def test_4
- assert_parse_ok @parser_1, "aabbaba", [0x61, 0x61, 0x62, 0x62, 0x61, 0x62, 0x61]
+ assert_parse_ok @parser_1, "aabbaba", [0x61.chr, 0x61.chr, 0x62.chr, 0x62.chr, 0x61.chr, 0x62.chr, 0x61.chr]
end
def test_5
@@ -552,7 +552,7 @@ class TestRepeatN < Minitest::Test
def setup
super
h = Hammer::Parser
- @parser_1 = h.repeat_n(h.choice(h.ch(0x61), h.ch(0x62)), 0x2)
+ @parser_1 = h.repeat_n(h.choice(h.ch(0x61.chr), h.ch(0x62.chr)), 0x2)
end
def test_1
@@ -560,7 +560,7 @@ class TestRepeatN < Minitest::Test
end
def test_2
- assert_parse_ok @parser_1, "abdef", [0x61, 0x62]
+ assert_parse_ok @parser_1, "abdef", [0x61.chr, 0x62.chr]
end
def test_3
@@ -572,19 +572,19 @@ class TestOptional < Minitest::Test
def setup
super
h = Hammer::Parser
- @parser_1 = h.sequence(h.ch(0x61), h.optional(h.choice(h.ch(0x62), h.ch(0x63))), h.ch(0x64))
+ @parser_1 = h.sequence(h.ch(0x61.chr), h.optional(h.choice(h.ch(0x62.chr), h.ch(0x63.chr))), h.ch(0x64.chr))
end
def test_1
- assert_parse_ok @parser_1, "abd", [0x61, 0x62, 0x64]
+ assert_parse_ok @parser_1, "abd", [0x61.chr, 0x62.chr, 0x64.chr]
end
def test_2
- assert_parse_ok @parser_1, "acd", [0x61, 0x63, 0x64]
+ assert_parse_ok @parser_1, "acd", [0x61.chr, 0x63.chr, 0x64.chr]
end
def test_3
- assert_parse_ok @parser_1, "ad", [0x61, nil, 0x64]
+ assert_parse_ok @parser_1, "ad", [0x61.chr, nil, 0x64.chr]
end
def test_4
@@ -604,11 +604,11 @@ class TestIgnore < Minitest::Test
def setup
super
h = Hammer::Parser
- @parser_1 = h.sequence(h.ch(0x61), h.ignore(h.ch(0x62)), h.ch(0x63))
+ @parser_1 = h.sequence(h.ch(0x61.chr), h.ignore(h.ch(0x62.chr)), h.ch(0x63.chr))
end
def test_1
- assert_parse_ok @parser_1, "abc", [0x61, 0x63]
+ assert_parse_ok @parser_1, "abc", [0x61.chr, 0x63.chr]
end
def test_2
@@ -620,23 +620,23 @@ class TestSepBy < Minitest::Test
def setup
super
h = Hammer::Parser
- @parser_1 = h.sepBy(h.choice(h.ch(0x31), h.ch(0x32), h.ch(0x33)), h.ch(0x2c))
+ @parser_1 = h.sepBy(h.choice(h.ch(0x31.chr), h.ch(0x32.chr), h.ch(0x33.chr)), h.ch(0x2c.chr))
end
def test_1
- assert_parse_ok @parser_1, "1,2,3", [0x31, 0x32, 0x33]
+ assert_parse_ok @parser_1, "1,2,3", [0x31.chr, 0x32.chr, 0x33.chr]
end
def test_2
- assert_parse_ok @parser_1, "1,3,2", [0x31, 0x33, 0x32]
+ assert_parse_ok @parser_1, "1,3,2", [0x31.chr, 0x33.chr, 0x32.chr]
end
def test_3
- assert_parse_ok @parser_1, "1,3", [0x31, 0x33]
+ assert_parse_ok @parser_1, "1,3", [0x31.chr, 0x33.chr]
end
def test_4
- assert_parse_ok @parser_1, "3", [0x33]
+ assert_parse_ok @parser_1, "3", [0x33.chr]
end
def test_5
@@ -648,23 +648,23 @@ class TestSepBy1 < Minitest::Test
def setup
super
h = Hammer::Parser
- @parser_1 = h.sepBy1(h.choice(h.ch(0x31), h.ch(0x32), h.ch(0x33)), h.ch(0x2c))
+ @parser_1 = h.sepBy1(h.choice(h.ch(0x31.chr), h.ch(0x32.chr), h.ch(0x33.chr)), h.ch(0x2c.chr))
end
def test_1
- assert_parse_ok @parser_1, "1,2,3", [0x31, 0x32, 0x33]
+ assert_parse_ok @parser_1, "1,2,3", [0x31.chr, 0x32.chr, 0x33.chr]
end
def test_2
- assert_parse_ok @parser_1, "1,3,2", [0x31, 0x33, 0x32]
+ assert_parse_ok @parser_1, "1,3,2", [0x31.chr, 0x33.chr, 0x32.chr]
end
def test_3
- assert_parse_ok @parser_1, "1,3", [0x31, 0x33]
+ assert_parse_ok @parser_1, "1,3", [0x31.chr, 0x33.chr]
end
def test_4
- assert_parse_ok @parser_1, "3", [0x33]
+ assert_parse_ok @parser_1, "3", [0x33.chr]
end
def test_5
@@ -676,13 +676,13 @@ class TestAnd < Minitest::Test
def setup
super
h = Hammer::Parser
- @parser_1 = h.sequence(h.and(h.ch(0x30)), h.ch(0x30))
- @parser_2 = h.sequence(h.and(h.ch(0x30)), h.ch(0x31))
- @parser_3 = h.sequence(h.ch(0x31), h.and(h.ch(0x32)))
+ @parser_1 = h.sequence(h.and(h.ch(0x30.chr)), h.ch(0x30.chr))
+ @parser_2 = h.sequence(h.and(h.ch(0x30.chr)), h.ch(0x31.chr))
+ @parser_3 = h.sequence(h.ch(0x31.chr), h.and(h.ch(0x32.chr)))
end
def test_1
- assert_parse_ok @parser_1, "0", [0x30]
+ assert_parse_ok @parser_1, "0", [0x30.chr]
end
def test_2
@@ -698,7 +698,7 @@ class TestAnd < Minitest::Test
end
def test_5
- assert_parse_ok @parser_3, "12", [0x31]
+ assert_parse_ok @parser_3, "12", [0x31.chr]
end
def test_6
@@ -710,12 +710,12 @@ class TestNot < Minitest::Test
def setup
super
h = Hammer::Parser
- @parser_1 = h.sequence(h.ch(0x61), h.choice(h.token("+"), h.token("++")), h.ch(0x62))
- @parser_2 = h.sequence(h.ch(0x61), h.choice(h.sequence(h.token("+"), h.not(h.ch(0x2b))), h.token("++")), h.ch(0x62))
+ @parser_1 = h.sequence(h.ch(0x61.chr), h.choice(h.token("+"), h.token("++")), h.ch(0x62.chr))
+ @parser_2 = h.sequence(h.ch(0x61.chr), h.choice(h.sequence(h.token("+"), h.not(h.ch(0x2b.chr))), h.token("++")), h.ch(0x62.chr))
end
def test_1
- assert_parse_ok @parser_1, "a+b", [0x61, "+", 0x62]
+ assert_parse_ok @parser_1, "a+b", [0x61.chr, "+", 0x62.chr]
end
def test_2
@@ -723,11 +723,11 @@ class TestNot < Minitest::Test
end
def test_3
- assert_parse_ok @parser_2, "a+b", [0x61, ["+"], 0x62]
+ assert_parse_ok @parser_2, "a+b", [0x61.chr, ["+"], 0x62.chr]
end
def test_4
- assert_parse_ok @parser_2, "a++b", [0x61, "++", 0x62]
+ assert_parse_ok @parser_2, "a++b", [0x61.chr, "++", 0x62.chr]
end
end
@@ -736,20 +736,20 @@ class TestRightrec < Minitest::Test
super
h = Hammer::Parser
@sp_rr = h.indirect
- @sp_rr.bind h.choice(h.sequence(h.ch(0x61), @sp_rr), h.epsilon_p)
+ @sp_rr.bind h.choice(h.sequence(h.ch(0x61.chr), @sp_rr), h.epsilon_p)
@parser_1 = @sp_rr
end
def test_1
- assert_parse_ok @parser_1, "a", [0x61]
+ assert_parse_ok @parser_1, "a", [0x61.chr]
end
def test_2
- assert_parse_ok @parser_1, "aa", [0x61, [0x61]]
+ assert_parse_ok @parser_1, "aa", [0x61.chr, [0x61.chr]]
end
def test_3
- assert_parse_ok @parser_1, "aaa", [0x61, [0x61, [0x61]]]
+ assert_parse_ok @parser_1, "aaa", [0x61.chr, [0x61.chr, [0x61.chr]]]
end
end
diff --git a/src/bindings/ruby/test/parser_test.rb b/src/bindings/ruby/test/parser_test.rb
index b9fb37f2..a8b9c7b8 100644
--- a/src/bindings/ruby/test/parser_test.rb
+++ b/src/bindings/ruby/test/parser_test.rb
@@ -92,3 +92,23 @@ class ParserTest < Minitest::Test
test_token_encoding('EUC-JP')
end
end
+
+class AttrBoolTest < Minitest::Test
+ def setup
+ h = Hammer::Parser
+ @parser = h.attr_bool(h.many1(h.choice(h.ch('a'), h.ch('b')))) {|x|
+ data = x.unmarshal
+ data.length > 1 && data[0] == data[1]
+ }
+ end
+
+ def test_1
+ assert_parse_ok @parser, "aa", ['a','a']
+ end
+ def test_2
+ assert_parse_ok @parser, "bb", ['b','b']
+ end
+ def test_3
+ refute_parse_ok @parser, "ab"
+ end
+end
--
GitLab