diff --git a/src/bindings/ruby/lib/hammer/parser.rb b/src/bindings/ruby/lib/hammer/parser.rb index 8722cc6639d94420b7053e043d58ad0d06ec2fc3..1d7a76ed7408ec14fd5d4e2ada80ae1eb9c3a818 100644 --- a/src/bindings/ruby/lib/hammer/parser.rb +++ b/src/bindings/ruby/lib/hammer/parser.rb @@ -23,7 +23,7 @@ module Hammer raise RuntimeError, '@h_parser is nil' if @h_parser.nil? raise ArgumentError, 'expecting a String' unless data.is_a? String # TODO: Not needed, FFI checks that. - result = Hammer::Internal.h_parse(@h_parser, data, data.length) + result = Hammer::Internal.h_parse(@h_parser, data, data.bytesize) return result unless result.null? end @@ -37,9 +37,8 @@ module Hammer # Need to copy string to a memory buffer (not just string.dup) # * Original string might be modified, this must not affect existing tokens # * We need a constant memory address (Ruby string might be moved around by the Ruby VM) - # * Use string.length instead of h_string.size to handle multibyte characters correctly. buffer = FFI::MemoryPointer.from_string(string) - h_parser = Hammer::Internal.h_token(buffer, string.length) + h_parser = Hammer::Internal.h_token(buffer, buffer.size-1) # buffer.size includes the null byte at the end return Hammer::Parser.new(:token, h_parser, buffer) end diff --git a/src/bindings/ruby/test/parser_test.rb b/src/bindings/ruby/test/parser_test.rb index d3c4bbb1c0ead85eb94676bb76604d1e71a0336c..f5f12f2dcdb84962ce9434847b8dab1950617e16 100644 --- a/src/bindings/ruby/test/parser_test.rb +++ b/src/bindings/ruby/test/parser_test.rb @@ -74,9 +74,10 @@ class ParserTest < Minitest::Test def test_multibyte_token parser = Hammer::Parser.build { token '今日' + token 'a' end_p } - refute_nil parser.parse('今日') + refute_nil parser.parse('今日a') end end