Commit cdb112f8 authored by Dominik Röttsches's avatar Dominik Röttsches Committed by Commit Bot

Define a character class iterator as adapter to Ragel scanner code

Blink text segmentation works on UChar* buffers. Define an adapter class
that can iterate over the Blink UTF-16 buffers returning character
classes that are compatible with the Ragel grammar that we will be using
for scanning text runs for text presentation or emoji presentation.

The Ragel grammar and generated code can be found
here. https://github.com/googlei18n/emoji-segmenter Integration to a
Chromium mirror is tracked in https://crbug.com/925792.

Bug: 892589
Change-Id: I76c9a4e4aa63eaf8bcd4b3d4e8a8111bb5244a4a
Reviewed-on: https://chromium-review.googlesource.com/c/1439298
Commit-Queue: Dominik Röttsches <drott@chromium.org>
Reviewed-by: default avatarEmil A Eklund <eae@chromium.org>
Auto-Submit: Dominik Röttsches <drott@chromium.org>
Cr-Commit-Position: refs/heads/master@{#626954}
parent 7dcf7883
...@@ -739,6 +739,8 @@ jumbo_component("platform") { ...@@ -739,6 +739,8 @@ jumbo_component("platform") {
"fonts/typesetting_features.h", "fonts/typesetting_features.h",
"fonts/unicode_range_set.cc", "fonts/unicode_range_set.cc",
"fonts/unicode_range_set.h", "fonts/unicode_range_set.h",
"fonts/utf16_ragel_iterator.cc",
"fonts/utf16_ragel_iterator.h",
"fonts/utf16_text_iterator.cc", "fonts/utf16_text_iterator.cc",
"fonts/utf16_text_iterator.h", "fonts/utf16_text_iterator.h",
"fonts/vdmx_parser.cc", "fonts/vdmx_parser.cc",
...@@ -1704,6 +1706,7 @@ jumbo_source_set("blink_platform_unittests_sources") { ...@@ -1704,6 +1706,7 @@ jumbo_source_set("blink_platform_unittests_sources") {
"fonts/symbols_iterator_test.cc", "fonts/symbols_iterator_test.cc",
"fonts/typesetting_features_test.cc", "fonts/typesetting_features_test.cc",
"fonts/unicode_range_set_test.cc", "fonts/unicode_range_set_test.cc",
"fonts/utf16_ragel_iterator_test.cc",
"geometry/double_point_test.cc", "geometry/double_point_test.cc",
"geometry/double_rect_test.cc", "geometry/double_rect_test.cc",
"geometry/float_box_test.cc", "geometry/float_box_test.cc",
......
...@@ -135,7 +135,7 @@ class PLATFORM_EXPORT CachingWordShapeIterator final { ...@@ -135,7 +135,7 @@ class PLATFORM_EXPORT CachingWordShapeIterator final {
// ZWJ and modifier check in order not to split those Emoji sequences. // ZWJ and modifier check in order not to split those Emoji sequences.
if (U_GET_GC_MASK(ch) & (U_GC_M_MASK | U_GC_LM_MASK | U_GC_SK_MASK) || if (U_GET_GC_MASK(ch) & (U_GC_M_MASK | U_GC_LM_MASK | U_GC_SK_MASK) ||
ch == kZeroWidthJoinerCharacter || Character::IsModifier(ch) || ch == kZeroWidthJoinerCharacter || Character::IsModifier(ch) ||
Character::IsEmojiFlagSequenceTag(ch)) Character::IsEmojiTagSequence(ch) || ch == kCancelTag)
continue; continue;
// Avoid delimiting COMMON/INHERITED alone, which makes harder to // Avoid delimiting COMMON/INHERITED alone, which makes harder to
// identify the script. // identify the script.
......
...@@ -72,7 +72,8 @@ bool SymbolsIterator::Consume(unsigned* symbols_limit, ...@@ -72,7 +72,8 @@ bool SymbolsIterator::Consume(unsigned* symbols_limit,
next_char_ == kStaffOfAesculapiusCharacter) && next_char_ == kStaffOfAesculapiusCharacter) &&
previous_font_fallback_priority_ == previous_font_fallback_priority_ ==
FontFallbackPriority::kEmojiEmoji) && FontFallbackPriority::kEmojiEmoji) &&
!Character::IsEmojiFlagSequenceTag(next_char_)) || !Character::IsEmojiTagSequence(next_char_) &&
kCancelTag != next_char_) ||
current_font_fallback_priority_ == FontFallbackPriority::kInvalid) { current_font_fallback_priority_ == FontFallbackPriority::kInvalid) {
current_font_fallback_priority_ = current_font_fallback_priority_ =
FontFallbackPriorityForCharacter(next_char_); FontFallbackPriorityForCharacter(next_char_);
......
// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "third_party/blink/renderer/platform/fonts/utf16_ragel_iterator.h"
#include "third_party/blink/renderer/platform/text/character.h"
#include "third_party/blink/renderer/platform/wtf/text/character_names.h"
namespace blink {
namespace {
char EmojiSegmentationCategory(UChar32 codepoint) {
// For the grammar to work, we need to check for more specific character
// classes first, then expand towards more generic ones. So we match single
// characters and small ranges first, then return EMOJI and
// EMOJI_TEXT_PRESENTATION for the remaining ones.
if (codepoint == kCombiningEnclosingKeycapCharacter)
return UTF16RagelIterator::COMBINING_ENCLOSING_KEYCAP;
if (codepoint == kCombiningEnclosingCircleBackslashCharacter)
return UTF16RagelIterator::COMBINING_ENCLOSING_CIRCLE_BACKSLASH;
if (codepoint == kZeroWidthJoinerCharacter)
return UTF16RagelIterator::ZWJ;
if (codepoint == kVariationSelector15Character)
return UTF16RagelIterator::VS15;
if (codepoint == kVariationSelector16Character)
return UTF16RagelIterator::VS16;
if (codepoint == 0x1F3F4)
return UTF16RagelIterator::TAG_BASE;
if (Character::IsEmojiTagSequence(codepoint))
return UTF16RagelIterator::TAG_SEQUENCE;
if (codepoint == kCancelTag) {
// http://www.unicode.org/reports/tr51/#def_emoji_tag_sequence
// defines a TAG_TERM grammar rule for U+E007F CANCEL TAG.
return UTF16RagelIterator::TAG_TERM;
}
if (Character::IsEmojiModifierBase(codepoint))
return UTF16RagelIterator::EMOJI_MODIFIER_BASE;
if (Character::IsModifier(codepoint))
return UTF16RagelIterator::EMOJI_MODIFIER;
if (Character::IsRegionalIndicator(codepoint))
return UTF16RagelIterator::REGIONAL_INDICATOR;
if (Character::IsEmojiKeycapBase(codepoint))
return UTF16RagelIterator::KEYCAP_BASE;
if (Character::IsEmojiEmojiDefault(codepoint))
return UTF16RagelIterator::EMOJI_EMOJI_PRESENTATION;
if (Character::IsEmojiTextDefault(codepoint))
return UTF16RagelIterator::EMOJI_TEXT_PRESENTATION;
if (Character::IsEmoji(codepoint))
return UTF16RagelIterator::EMOJI;
// Ragel state machine will interpret unknown category as "any".
return UTF16RagelIterator::kMaxEmojiScannerCategory;
}
} // namespace
void UTF16RagelIterator::UpdateCachedCategory() {
cached_category_ = EmojiSegmentationCategory(Codepoint());
}
} // namespace blink
// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef THIRD_PARTY_BLINK_RENDERER_PLATFORM_FONTS_UTF16_RAGEL_ITERATOR_H_
#define THIRD_PARTY_BLINK_RENDERER_PLATFORM_FONTS_UTF16_RAGEL_ITERATOR_H_
#include <unicode/uchar.h>
#include "base/logging.h"
#include "third_party/blink/renderer/platform/platform_export.h"
namespace blink {
// UTF16RagelIterator is set up on top of a UTF-16 UChar* buffer iterating over
// a Blink internal text string and as such is used as an adapter between Blink
// strings and the Ragel-based emoji scanner. It supports forwarding and
// reversing using arithmetic operators. Dereferencing the iterator means
// retrieving a character class as defined in the Ragel grammar of
// third-party/emoji-segmenter. The dereferenced character category is cached
// since Ragel dereferences multiple times without moving the iterator's cursor.
class PLATFORM_EXPORT UTF16RagelIterator {
public:
UTF16RagelIterator() : buffer_(nullptr), buffer_size_(0), cursor_(0){};
UTF16RagelIterator(const UChar* buffer,
unsigned buffer_size,
unsigned cursor = 0)
: buffer_(buffer),
buffer_size_(buffer_size),
cursor_(cursor),
cached_category_(kMaxEmojiScannerCategory) {
UpdateCachedCategory();
};
UTF16RagelIterator end() {
UTF16RagelIterator ret = *this;
ret.cursor_ = buffer_size_;
return ret;
}
unsigned cursor() { return cursor_; }
UTF16RagelIterator& operator+=(int v) {
if (v > 0) {
U16_FWD_N(buffer_, cursor_, buffer_size_, v);
} else if (v < 0) {
U16_BACK_N(buffer_, 0, cursor_, -v);
}
UpdateCachedCategory();
return *this;
}
UTF16RagelIterator& operator-=(int v) { return *this += -v; }
UTF16RagelIterator operator+(int v) {
UTF16RagelIterator ret = *this;
return ret += v;
}
UTF16RagelIterator operator-(int v) { return *this + -v; }
int operator-(const UTF16RagelIterator& other) {
DCHECK_EQ(buffer_, other.buffer_);
return cursor_ - other.cursor_;
}
UTF16RagelIterator& operator++() {
DCHECK_LT(cursor_, buffer_size_);
U16_FWD_1(buffer_, cursor_, buffer_size_);
UpdateCachedCategory();
return *this;
}
UTF16RagelIterator& operator--() {
DCHECK_GT(cursor_, 0u);
U16_BACK_1(buffer_, 0, cursor_);
UpdateCachedCategory();
return *this;
}
UTF16RagelIterator operator++(int) {
UTF16RagelIterator ret = *this;
++(*this);
return ret;
}
UTF16RagelIterator operator--(int) {
UTF16RagelIterator ret = *this;
--(*this);
return ret;
}
UTF16RagelIterator operator=(int v) {
// We need this integer assignment operator because Ragel has initialization
// code for assigning 0 to ts, te.
DCHECK_EQ(v, 0);
UTF16RagelIterator ret = *this;
ret.cursor_ = v;
return ret;
}
UChar32 operator*() {
CHECK(buffer_size_);
return cached_category_;
}
bool operator==(const UTF16RagelIterator& other) const {
return buffer_ == other.buffer_ && buffer_size_ == other.buffer_size_ &&
cursor_ == other.cursor_;
}
bool operator!=(const UTF16RagelIterator& other) const {
return !(*this == other);
}
// Must match the categories defined in third-party/emoji-segmenter/.
// TODO(drott): Add static asserts once emoji-segmenter is imported to
// third-party.
enum EmojiScannerCharacterClass {
EMOJI = 0,
EMOJI_TEXT_PRESENTATION = 1,
EMOJI_EMOJI_PRESENTATION = 2,
EMOJI_MODIFIER_BASE = 3,
EMOJI_MODIFIER = 4,
EMOJI_VS_BASE = 5,
REGIONAL_INDICATOR = 6,
KEYCAP_BASE = 7,
COMBINING_ENCLOSING_KEYCAP = 8,
COMBINING_ENCLOSING_CIRCLE_BACKSLASH = 9,
ZWJ = 10,
VS15 = 11,
VS16 = 12,
TAG_BASE = 13,
TAG_SEQUENCE = 14,
TAG_TERM = 15,
kMaxEmojiScannerCategory = 16
};
private:
UChar32 Codepoint() const {
DCHECK_GT(buffer_size_, 0u);
UChar32 output;
U16_GET(buffer_, 0, cursor_, buffer_size_, output);
return output;
}
void UpdateCachedCategory();
const UChar* buffer_;
unsigned buffer_size_;
unsigned cursor_;
unsigned char cached_category_;
};
} // namespace blink
#endif // THIRD_PARTY_BLINK_RENDERER_PLATFORM_FONTS_UTF16_RAGEL_ITERATOR_H_
// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "third_party/blink/renderer/platform/fonts/utf16_ragel_iterator.h"
#include <unicode/unistr.h>
#include "base/stl_util.h"
#include "testing/gtest/include/gtest/gtest.h"
#include "third_party/blink/renderer/platform/text/character.h"
#include "third_party/blink/renderer/platform/wtf/text/character_names.h"
namespace blink {
TEST(UTF16RagelIteratorTest, CharacterClasses) {
UChar32 class_examples_codepoints[] = {
kCombiningEnclosingKeycapCharacter,
kCombiningEnclosingCircleBackslashCharacter,
kZeroWidthJoinerCharacter,
kVariationSelector15Character,
kVariationSelector16Character,
0x1f3f4,
0xE0030,
kCancelTag,
0x261D,
0x1F3FB,
0x1F1E6,
0x0030,
0x231A,
0x00A9};
icu_63::UnicodeString class_examples_unicode_string =
icu_63::UnicodeString::fromUTF32(class_examples_codepoints,
base::size(class_examples_codepoints));
char categories[] = {UTF16RagelIterator::COMBINING_ENCLOSING_KEYCAP,
UTF16RagelIterator::COMBINING_ENCLOSING_CIRCLE_BACKSLASH,
UTF16RagelIterator::ZWJ,
UTF16RagelIterator::VS15,
UTF16RagelIterator::VS16,
UTF16RagelIterator::TAG_BASE,
UTF16RagelIterator::TAG_SEQUENCE,
UTF16RagelIterator::TAG_TERM,
UTF16RagelIterator::EMOJI_MODIFIER_BASE,
UTF16RagelIterator::EMOJI_MODIFIER,
UTF16RagelIterator::REGIONAL_INDICATOR,
UTF16RagelIterator::KEYCAP_BASE,
UTF16RagelIterator::EMOJI_EMOJI_PRESENTATION,
UTF16RagelIterator::EMOJI_TEXT_PRESENTATION};
UTF16RagelIterator ragel_iterator(
reinterpret_cast<const UChar*>(class_examples_unicode_string.getBuffer()),
class_examples_unicode_string.length());
for (char& category : categories) {
CHECK_EQ(category, *ragel_iterator);
ragel_iterator++;
}
UTF16RagelIterator reverse_ragel_iterator(
reinterpret_cast<const UChar*>(class_examples_unicode_string.getBuffer()),
class_examples_unicode_string.length(),
class_examples_unicode_string.length() - 1);
size_t i = base::size(categories) - 1;
while (reverse_ragel_iterator.cursor() > 0) {
CHECK_EQ(categories[i], *reverse_ragel_iterator);
i--;
reverse_ragel_iterator--;
};
}
TEST(UTF16RagelIteratorTest, ArithmeticOperators) {
UChar32 class_examples_codepoints[] = {
kVariationSelector15Character, kVariationSelector15Character,
kVariationSelector15Character, kVariationSelector16Character,
kVariationSelector16Character, kVariationSelector16Character,
};
icu_63::UnicodeString class_examples_unicode_string =
icu_63::UnicodeString::fromUTF32(class_examples_codepoints,
base::size(class_examples_codepoints));
UTF16RagelIterator ragel_iterator(
reinterpret_cast<const UChar*>(class_examples_unicode_string.getBuffer()),
class_examples_unicode_string.length());
CHECK_EQ(*ragel_iterator, UTF16RagelIterator::VS15);
CHECK_EQ(*(ragel_iterator + 2), UTF16RagelIterator::VS15);
CHECK_EQ(*(ragel_iterator + 3), UTF16RagelIterator::VS16);
CHECK_EQ(*(ragel_iterator + 5), UTF16RagelIterator::VS16);
CHECK_EQ(*(ragel_iterator += 3), UTF16RagelIterator::VS16);
CHECK_EQ(*(ragel_iterator += 2), UTF16RagelIterator::VS16);
CHECK_EQ(*(ragel_iterator -= 4), UTF16RagelIterator::VS15);
CHECK_EQ(*(ragel_iterator += 1), UTF16RagelIterator::VS15);
ragel_iterator += 3;
UTF16RagelIterator ragel_iterator_begin = ragel_iterator - 5;
CHECK(ragel_iterator != ragel_iterator_begin);
CHECK(ragel_iterator == ragel_iterator.end() - 1);
CHECK_EQ(*ragel_iterator, UTF16RagelIterator::VS16);
CHECK_EQ(*(ragel_iterator - 2), UTF16RagelIterator::VS16);
CHECK_EQ(*(ragel_iterator - 3), UTF16RagelIterator::VS15);
CHECK_EQ(*(ragel_iterator - 5), UTF16RagelIterator::VS15);
}
TEST(UTF16RagelIteratorTest, InvalidOperationOnEmpty) {
UTF16RagelIterator ragel_iterator;
CHECK_EQ(ragel_iterator.cursor(), 0u);
EXPECT_DEATH_IF_SUPPORTED(ragel_iterator++, "");
EXPECT_DEATH_IF_SUPPORTED(ragel_iterator--, "");
EXPECT_DEATH_IF_SUPPORTED(*ragel_iterator, "");
}
} // namespace blink
...@@ -258,12 +258,10 @@ bool Character::CanReceiveTextEmphasis(UChar32 c) { ...@@ -258,12 +258,10 @@ bool Character::CanReceiveTextEmphasis(UChar32 c) {
return true; return true;
} }
bool Character::IsEmojiFlagSequenceTag(UChar32 c) { bool Character::IsEmojiTagSequence(UChar32 c) {
// Only allow valid sequences from
// http://www.unicode.org/reports/tr51/proposed.html#valid-emoji-tag-sequences // http://www.unicode.org/reports/tr51/proposed.html#valid-emoji-tag-sequences
return (c >= kTagDigitZero && c <= kTagDigitNine) || return (c >= kTagDigitZero && c <= kTagDigitNine) ||
(c >= kTagLatinSmallLetterA && c <= kTagLatinSmallLetterZ) || (c >= kTagLatinSmallLetterA && c <= kTagLatinSmallLetterZ);
c == kCancelTag;
} }
template <typename CharacterType> template <typename CharacterType>
......
...@@ -153,7 +153,7 @@ class PLATFORM_EXPORT Character { ...@@ -153,7 +153,7 @@ class PLATFORM_EXPORT Character {
static bool IsRegionalIndicator(UChar32); static bool IsRegionalIndicator(UChar32);
static bool IsModifier(UChar32 c) { return c >= 0x1F3FB && c <= 0x1F3FF; } static bool IsModifier(UChar32 c) { return c >= 0x1F3FB && c <= 0x1F3FF; }
// http://www.unicode.org/reports/tr51/proposed.html#flag-emoji-tag-sequences // http://www.unicode.org/reports/tr51/proposed.html#flag-emoji-tag-sequences
static bool IsEmojiFlagSequenceTag(UChar32); static bool IsEmojiTagSequence(UChar32);
static inline UChar NormalizeSpaces(UChar character) { static inline UChar NormalizeSpaces(UChar character) {
if (TreatAsSpace(character)) if (TreatAsSpace(character))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment