[css-properties-values-api] Syntax string parser.

This CL implements the syntax string parser according to the new spec. The
parser operates on the "preprocessed input stream" described by css-syntax,
and while it reuses some algorithms used by the tokenizer, it does not
rely on actual tokens.

Overview of this CL:

 * Added CSSSyntaxStringParser, which can parse the syntax string into
   a CSSSyntaxDescriptor. Usage:
   auto descriptor = CSSSyntaxStringParser("<color>").Parse();
 * The previous syntax string parser functions were mixed together with
   functions for parsing a token stream against a registered syntax, which
   was a little confusing. Hence this CL splits the syntax string parsing
   into a separate class.
 * Moved a few parser algorithms from CSSTokenizer to css_parser_idioms,
   to make them usable without a tokenizer.
 * Unit tests: there is some overlap between the unit tests and WPT.
   However, the unit tests can (and do) check the resulting internal
   structure of the CSSSyntaxDescriptors, so they do add additional value
   over just WPT.

R=futhark@chromium.org, ikilpatrick@chromium.org

Change-Id: I2d671ca0e58cd123a7809a24a4e83aec3883b175
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1503755Reviewed-by: Ian Kilpatrick <ikilpatrick@chromium.org>
Reviewed-by: Rune Lillesveen <futhark@chromium.org>
Commit-Queue: Anders Hartvoll Ruud <andruud@chromium.org>
Cr-Commit-Position: refs/heads/master@{#638178}

third_party/blink/renderer/core/BUILD.gn

@@ -1753,6 +1753,7 @@ jumbo_source_set("unit_tests") {
     "css/css_selector_watch_test.cc",
     "css/css_style_declaration_test.cc",
     "css/css_style_sheet_test.cc",
+    "css/css_syntax_string_parser_test.cc",
     "css/css_test_helpers.cc",
     "css/css_test_helpers.h",
     "css/css_value_test_helper.h",

third_party/blink/renderer/core/css/BUILD.gn

@@ -170,6 +170,8 @@ blink_core_sources("css") {
     "css_syntax_component.h",
     "css_syntax_descriptor.cc",
     "css_syntax_descriptor.h",
+    "css_syntax_string_parser.cc",
+    "css_syntax_string_parser.h",
     "css_timing_function_value.cc",
     "css_timing_function_value.h",
     "css_to_length_conversion_data.cc",
@@ -369,6 +371,7 @@ blink_core_sources("css") {
     "parser/css_parser_context.h",
     "parser/css_parser_fast_paths.cc",
     "parser/css_parser_fast_paths.h",
+    "parser/css_parser_idioms.cc",
     "parser/css_parser_idioms.h",
     "parser/css_parser_impl.cc",
     "parser/css_parser_impl.h",

third_party/blink/renderer/core/css/css_syntax_descriptor.cc

@@ -4,6 +4,7 @@
 
 #include "third_party/blink/renderer/core/css/css_syntax_descriptor.h"
 
+#include <utility>
 #include "third_party/blink/renderer/core/css/css_custom_property_declaration.h"
 #include "third_party/blink/renderer/core/css/css_syntax_component.h"
 #include "third_party/blink/renderer/core/css/css_uri_value.h"
@@ -17,141 +18,6 @@
 
 namespace blink {
 
-void ConsumeWhitespace(const String& string, wtf_size_t& offset) {
-  while (IsHTMLSpace(string[offset]))
-    offset++;
-}
-
-bool ConsumeCharacterAndWhitespace(const String& string,
-                                   char character,
-                                   wtf_size_t& offset) {
-  if (string[offset] != character)
-    return false;
-  offset++;
-  ConsumeWhitespace(string, offset);
-  return true;
-}
-
-CSSSyntaxType ParseSyntaxType(String type) {
-  // TODO(timloh): Are these supposed to be case sensitive?
-  if (type == "length")
-    return CSSSyntaxType::kLength;
-  if (type == "number")
-    return CSSSyntaxType::kNumber;
-  if (type == "percentage")
-    return CSSSyntaxType::kPercentage;
-  if (type == "length-percentage")
-    return CSSSyntaxType::kLengthPercentage;
-  if (type == "color")
-    return CSSSyntaxType::kColor;
-  if (RuntimeEnabledFeatures::CSSVariables2ImageValuesEnabled()) {
-    if (type == "image")
-      return CSSSyntaxType::kImage;
-  }
-  if (type == "url")
-    return CSSSyntaxType::kUrl;
-  if (type == "integer")
-    return CSSSyntaxType::kInteger;
-  if (type == "angle")
-    return CSSSyntaxType::kAngle;
-  if (type == "time")
-    return CSSSyntaxType::kTime;
-  if (type == "resolution")
-    return CSSSyntaxType::kResolution;
-  if (RuntimeEnabledFeatures::CSSVariables2TransformValuesEnabled()) {
-    if (type == "transform-function")
-      return CSSSyntaxType::kTransformFunction;
-    if (type == "transform-list")
-      return CSSSyntaxType::kTransformList;
-  }
-  if (type == "custom-ident")
-    return CSSSyntaxType::kCustomIdent;
-  // Not an Ident, just used to indicate failure
-  return CSSSyntaxType::kIdent;
-}
-
-bool ConsumeSyntaxType(const String& input,
-                       wtf_size_t& offset,
-                       CSSSyntaxType& type) {
-  DCHECK_EQ(input[offset], '<');
-  offset++;
-  wtf_size_t type_start = offset;
-  while (offset < input.length() && input[offset] != '>')
-    offset++;
-  if (offset == input.length())
-    return false;
-  type = ParseSyntaxType(input.Substring(type_start, offset - type_start));
-  if (type == CSSSyntaxType::kIdent)
-    return false;
-  offset++;
-  return true;
-}
-
-bool ConsumeSyntaxIdent(const String& input,
-                        wtf_size_t& offset,
-                        String& ident) {
-  wtf_size_t ident_start = offset;
-  while (IsNameCodePoint(input[offset]))
-    offset++;
-  if (offset == ident_start)
-    return false;
-  ident = input.Substring(ident_start, offset - ident_start);
-  return !css_property_parser_helpers::IsCSSWideKeyword(ident);
-}
-
-CSSSyntaxDescriptor::CSSSyntaxDescriptor(const String& input) {
-  wtf_size_t offset = 0;
-  ConsumeWhitespace(input, offset);
-
-  if (ConsumeCharacterAndWhitespace(input, '*', offset)) {
-    if (offset != input.length())
-      return;
-    syntax_components_.push_back(CSSSyntaxComponent(
-        CSSSyntaxType::kTokenStream, g_empty_string, CSSSyntaxRepeat::kNone));
-    return;
-  }
-
-  do {
-    CSSSyntaxType type;
-    String ident;
-    bool success;
-
-    if (input[offset] == '<') {
-      success = ConsumeSyntaxType(input, offset, type);
-    } else {
-      type = CSSSyntaxType::kIdent;
-      success = ConsumeSyntaxIdent(input, offset, ident);
-    }
-
-    if (!success) {
-      syntax_components_.clear();
-      return;
-    }
-
-    CSSSyntaxRepeat repeat = CSSSyntaxRepeat::kNone;
-
-    if (ConsumeCharacterAndWhitespace(input, '+', offset))
-      repeat = CSSSyntaxRepeat::kSpaceSeparated;
-    else if (ConsumeCharacterAndWhitespace(input, '#', offset))
-      repeat = CSSSyntaxRepeat::kCommaSeparated;
-
-    // <transform-list> is already a space separated list,
-    // <transform-list>+ is invalid.
-    // TODO(andruud): Is <transform-list># invalid?
-    if (type == CSSSyntaxType::kTransformList &&
-        repeat != CSSSyntaxRepeat::kNone) {
-      syntax_components_.clear();
-      return;
-    }
-    ConsumeWhitespace(input, offset);
-    syntax_components_.push_back(CSSSyntaxComponent(type, ident, repeat));
-
-  } while (ConsumeCharacterAndWhitespace(input, '|', offset));
-
-  if (offset != input.length())
-    syntax_components_.clear();
-}
-
 const CSSValue* ConsumeSingleType(const CSSSyntaxComponent& syntax,
                                   CSSParserTokenRange& range,
                                   const CSSParserContext* context) {
@@ -262,4 +128,16 @@ const CSSValue* CSSSyntaxDescriptor::Parse(CSSParserTokenRange range,
                                                          is_animation_tainted);
 }
 
+CSSSyntaxDescriptor::CSSSyntaxDescriptor(Vector<CSSSyntaxComponent> components)
+    : syntax_components_(std::move(components)) {
+  DCHECK(syntax_components_.size());
+}
+
+CSSSyntaxDescriptor CSSSyntaxDescriptor::CreateUniversal() {
+  Vector<CSSSyntaxComponent> components;
+  components.push_back(CSSSyntaxComponent(
+      CSSSyntaxType::kTokenStream, g_empty_string, CSSSyntaxRepeat::kNone));
+  return CSSSyntaxDescriptor(std::move(components));
+}
+
 }  // namespace blink

third_party/blink/renderer/core/css/css_syntax_descriptor.h

@@ -15,14 +15,11 @@ class CSSValue;
 
 class CORE_EXPORT CSSSyntaxDescriptor {
  public:
-  explicit CSSSyntaxDescriptor(const String& syntax);
-
   const CSSValue* Parse(CSSParserTokenRange,
                         const CSSParserContext*,
                         bool is_animation_tainted) const;
   const CSSSyntaxComponent* Match(const CSSStyleValue&) const;
   bool CanTake(const CSSStyleValue&) const;
-  bool IsValid() const { return !syntax_components_.IsEmpty(); }
   bool IsTokenStream() const {
     return syntax_components_.size() == 1 &&
            syntax_components_[0].GetType() == CSSSyntaxType::kTokenStream;
@@ -45,6 +42,14 @@ class CORE_EXPORT CSSSyntaxDescriptor {
   }
 
  private:
+  friend class CSSSyntaxStringParser;
+  friend class CSSSyntaxStringParserTest;
+
+  explicit CSSSyntaxDescriptor(Vector<CSSSyntaxComponent>);
+
+  // https://drafts.css-houdini.org/css-properties-values-api-1/#universal-syntax-descriptor
+  static CSSSyntaxDescriptor CreateUniversal();
+
   Vector<CSSSyntaxComponent> syntax_components_;
 };
 

third_party/blink/renderer/core/css/css_syntax_string_parser.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/blink/renderer/core/css/css_syntax_string_parser.h"
+
+#include <utility>
+#include "third_party/blink/renderer/core/css/css_syntax_component.h"
+#include "third_party/blink/renderer/core/css/parser/css_parser_idioms.h"
+#include "third_party/blink/renderer/core/css/parser/css_property_parser_helpers.h"
+#include "third_party/blink/renderer/core/html/parser/html_parser_idioms.h"
+#include "third_party/blink/renderer/platform/runtime_enabled_features.h"
+
+namespace blink {
+
+namespace {
+
+// https://drafts.css-houdini.org/css-properties-values-api-1/#supported-names
+base::Optional<CSSSyntaxType> ParseSyntaxType(StringView type) {
+  if (type == "length")
+    return CSSSyntaxType::kLength;
+  if (type == "number")
+    return CSSSyntaxType::kNumber;
+  if (type == "percentage")
+    return CSSSyntaxType::kPercentage;
+  if (type == "length-percentage")
+    return CSSSyntaxType::kLengthPercentage;
+  if (type == "color")
+    return CSSSyntaxType::kColor;
+  if (RuntimeEnabledFeatures::CSSVariables2ImageValuesEnabled()) {
+    if (type == "image")
+      return CSSSyntaxType::kImage;
+  }
+  if (type == "url")
+    return CSSSyntaxType::kUrl;
+  if (type == "integer")
+    return CSSSyntaxType::kInteger;
+  if (type == "angle")
+    return CSSSyntaxType::kAngle;
+  if (type == "time")
+    return CSSSyntaxType::kTime;
+  if (type == "resolution")
+    return CSSSyntaxType::kResolution;
+  if (RuntimeEnabledFeatures::CSSVariables2TransformValuesEnabled()) {
+    if (type == "transform-function")
+      return CSSSyntaxType::kTransformFunction;
+    if (type == "transform-list")
+      return CSSSyntaxType::kTransformList;
+  }
+  if (type == "custom-ident")
+    return CSSSyntaxType::kCustomIdent;
+  return base::nullopt;
+}
+
+bool IsPreMultiplied(CSSSyntaxType type) {
+  return type == CSSSyntaxType::kTransformList;
+}
+
+}  // namespace
+
+CSSSyntaxStringParser::CSSSyntaxStringParser(const String& string)
+    : string_(string.StripWhiteSpace()), input_(string_) {}
+
+base::Optional<CSSSyntaxDescriptor> CSSSyntaxStringParser::Parse() {
+  if (string_.IsEmpty())
+    return base::nullopt;
+  if (string_.length() == 1 && string_[0] == '*')
+    return CSSSyntaxDescriptor::CreateUniversal();
+
+  Vector<CSSSyntaxComponent> components;
+
+  while (true) {
+    UChar cc = input_.NextInputChar();
+    input_.Advance();
+    if (IsHTMLSpace(cc))
+      continue;
+    if (cc == '\0')
+      break;
+    if (cc == '|') {
+      if (!components.size())
+        return base::nullopt;
+    } else {
+      input_.PushBack(cc);
+    }
+    if (!ConsumeSyntaxComponent(components))
+      return base::nullopt;
+  }
+
+  return CSSSyntaxDescriptor(std::move(components));
+}
+
+bool CSSSyntaxStringParser::ConsumeSyntaxComponent(
+    Vector<CSSSyntaxComponent>& components) {
+  input_.AdvanceUntilNonWhitespace();
+
+  CSSSyntaxType type = CSSSyntaxType::kTokenStream;
+  String ident;
+
+  UChar cc = input_.NextInputChar();
+  input_.Advance();
+
+  if (cc == '<') {
+    if (!ConsumeDataTypeName(type))
+      return false;
+  } else if (IsNameStartCodePoint(cc) || cc == '\\') {
+    if (NextCharsAreIdentifier(cc, input_)) {
+      input_.PushBack(cc);
+      type = CSSSyntaxType::kIdent;
+      if (!ConsumeIdent(ident))
+        return false;
+    }
+  } else {
+    return false;
+  }
+
+  DCHECK_NE(type, CSSSyntaxType::kTokenStream);
+
+  CSSSyntaxRepeat repeat =
+      IsPreMultiplied(type) ? CSSSyntaxRepeat::kNone : ConsumeRepeatIfPresent();
+  components.emplace_back(type, ident, repeat);
+  return true;
+}
+
+CSSSyntaxRepeat CSSSyntaxStringParser::ConsumeRepeatIfPresent() {
+  UChar cc = input_.NextInputChar();
+  if (cc == '+') {
+    input_.Advance();
+    return CSSSyntaxRepeat::kSpaceSeparated;
+  }
+  if (cc == '#') {
+    input_.Advance();
+    return CSSSyntaxRepeat::kCommaSeparated;
+  }
+  return CSSSyntaxRepeat::kNone;
+}
+
+bool CSSSyntaxStringParser::ConsumeDataTypeName(CSSSyntaxType& type) {
+  for (unsigned size = 0;; ++size) {
+    UChar cc = input_.PeekWithoutReplacement(size);
+    if (IsNameCodePoint(cc))
+      continue;
+    if (cc == '>') {
+      unsigned start = input_.Offset();
+      input_.Advance(size + 1);
+      if (auto syntax_type = ParseSyntaxType(input_.RangeAt(start, size))) {
+        type = *syntax_type;
+        return true;
+      }
+      return false;
+    }
+    return false;
+  }
+}
+
+bool CSSSyntaxStringParser::ConsumeIdent(String& ident) {
+  ident = ConsumeName(input_);
+  return !css_property_parser_helpers::IsCSSWideKeyword(ident);
+}
+
+}  // namespace blink

third_party/blink/renderer/core/css/css_syntax_string_parser.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BLINK_RENDERER_CORE_CSS_CSS_SYNTAX_STRING_PARSER_H_
+#define THIRD_PARTY_BLINK_RENDERER_CORE_CSS_CSS_SYNTAX_STRING_PARSER_H_
+
+#include "base/optional.h"
+#include "third_party/blink/renderer/core/css/css_syntax_descriptor.h"
+#include "third_party/blink/renderer/core/css/parser/css_tokenizer_input_stream.h"
+
+namespace blink {
+
+class CSSTokenizerInputStream;
+
+// Produces a CSSSyntaxDescriptor from a CSSTokenizerInputStream.
+//
+// https://drafts.css-houdini.org/css-properties-values-api-1/#parsing-syntax
+class CORE_EXPORT CSSSyntaxStringParser {
+ public:
+  explicit CSSSyntaxStringParser(const String&);
+
+  // https://drafts.css-houdini.org/css-properties-values-api-1/#consume-syntax-descriptor
+  base::Optional<CSSSyntaxDescriptor> Parse();
+
+ private:
+  // https://drafts.css-houdini.org/css-properties-values-api-1/#consume-syntax-component
+  //
+  // Appends a CSSSyntaxComponent to the Vector on success.
+  bool ConsumeSyntaxComponent(Vector<CSSSyntaxComponent>&);
+
+  // https://drafts.css-houdini.org/css-properties-values-api-1/#consume-data-type-name
+  //
+  // Returns true if the input stream contained a supported data type name, i.e.
+  // a string with a corresponding CSSSyntaxType.
+  //
+  // https://drafts.css-houdini.org/css-properties-values-api-1/#supported-names
+  bool ConsumeDataTypeName(CSSSyntaxType&);
+
+  // Consumes a name from the input stream, and stores the result in 'ident'.
+  // Returns true if the value returned via 'ident' is not a css-wide keyword.
+  bool ConsumeIdent(String& ident);
+
+  // Consumes a '+' or '#' from the input stream (if present), and returns
+  // the appropriate CSSSyntaxRepeat. CSSSyntaxRepeat::kNone is returned if
+  // the next input code point is not '+' or '#'.
+  CSSSyntaxRepeat ConsumeRepeatIfPresent();
+
+  String string_;
+  CSSTokenizerInputStream input_;
+};
+
+}  // namespace blink
+
+#endif  // THIRD_PARTY_BLINK_RENDERER_CORE_CSS_CSS_SYNTAX_STRING_PARSER_H_

third_party/blink/renderer/core/css/parser/css_parser_idioms.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/blink/renderer/core/css/parser/css_parser_idioms.h"
+#include "third_party/blink/renderer/core/css/parser/css_tokenizer_input_stream.h"
+#include "third_party/blink/renderer/core/html/parser/html_parser_idioms.h"
+#include "third_party/blink/renderer/core/html/parser/input_stream_preprocessor.h"
+#include "third_party/blink/renderer/platform/wtf/text/character_names.h"
+#include "third_party/blink/renderer/platform/wtf/text/string_builder.h"
+
+namespace blink {
+
+void ConsumeSingleWhitespaceIfNext(CSSTokenizerInputStream& input) {
+  // We check for \r\n and HTML spaces since we don't do preprocessing
+  UChar next = input.PeekWithoutReplacement(0);
+  if (next == '\r' && input.PeekWithoutReplacement(1) == '\n')
+    input.Advance(2);
+  else if (IsHTMLSpace(next))
+    input.Advance();
+}
+
+// https://drafts.csswg.org/css-syntax/#consume-an-escaped-code-point
+UChar32 ConsumeEscape(CSSTokenizerInputStream& input) {
+  UChar cc = input.NextInputChar();
+  input.Advance();
+  DCHECK(!IsCSSNewLine(cc));
+  if (IsASCIIHexDigit(cc)) {
+    unsigned consumed_hex_digits = 1;
+    StringBuilder hex_chars;
+    hex_chars.Append(cc);
+    while (consumed_hex_digits < 6 &&
+           IsASCIIHexDigit(input.PeekWithoutReplacement(0))) {
+      cc = input.NextInputChar();
+      input.Advance();
+      hex_chars.Append(cc);
+      consumed_hex_digits++;
+    };
+    ConsumeSingleWhitespaceIfNext(input);
+    bool ok = false;
+    UChar32 code_point = hex_chars.ToString().HexToUIntStrict(&ok);
+    DCHECK(ok);
+    if (code_point == 0 || (0xD800 <= code_point && code_point <= 0xDFFF) ||
+        code_point > 0x10FFFF)
+      return kReplacementCharacter;
+    return code_point;
+  }
+
+  if (cc == kEndOfFileMarker)
+    return kReplacementCharacter;
+  return cc;
+}
+
+// http://www.w3.org/TR/css3-syntax/#consume-a-name
+String ConsumeName(CSSTokenizerInputStream& input) {
+  StringBuilder result;
+  while (true) {
+    UChar cc = input.NextInputChar();
+    input.Advance();
+    if (IsNameCodePoint(cc)) {
+      result.Append(cc);
+      continue;
+    }
+    if (TwoCharsAreValidEscape(cc, input.PeekWithoutReplacement(0))) {
+      result.Append(ConsumeEscape(input));
+      continue;
+    }
+    input.PushBack(cc);
+    return result.ToString();
+  }
+}
+
+// https://drafts.csswg.org/css-syntax/#would-start-an-identifier
+bool NextCharsAreIdentifier(UChar first, const CSSTokenizerInputStream& input) {
+  UChar second = input.PeekWithoutReplacement(0);
+  if (IsNameStartCodePoint(first) || TwoCharsAreValidEscape(first, second))
+    return true;
+
+  if (first == '-') {
+    return IsNameStartCodePoint(second) || second == '-' ||
+           TwoCharsAreValidEscape(second, input.PeekWithoutReplacement(1));
+  }
+
+  return false;
+}
+
+}  // namespace blink

third_party/blink/renderer/core/css/parser/css_parser_idioms.h

@@ -32,15 +32,23 @@
 #define THIRD_PARTY_BLINK_RENDERER_CORE_CSS_PARSER_CSS_PARSER_IDIOMS_H_
 
 #include "third_party/blink/renderer/platform/wtf/text/unicode.h"
+#include "third_party/blink/renderer/platform/wtf/text/wtf_string.h"
 
 namespace blink {
 
+class CSSTokenizerInputStream;
+
 // Space characters as defined by the CSS specification.
 // http://www.w3.org/TR/css3-syntax/#whitespace
 inline bool IsCSSSpace(UChar c) {
   return c == ' ' || c == '\t' || c == '\n';
 }
 
+inline bool IsCSSNewLine(UChar cc) {
+  // We check \r and \f here, since we have no preprocessing stage
+  return (cc == '\r' || cc == '\n' || cc == '\f');
+}
+
 // https://drafts.csswg.org/css-syntax/#name-start-code-point
 template <typename CharacterType>
 bool IsNameStartCodePoint(CharacterType c) {
@@ -52,6 +60,28 @@ template <typename CharacterType>
 bool IsNameCodePoint(CharacterType c) {
   return IsNameStartCodePoint(c) || IsASCIIDigit(c) || c == '-';
 }
+
+// https://drafts.csswg.org/css-syntax/#check-if-two-code-points-are-a-valid-escape
+inline bool TwoCharsAreValidEscape(UChar first, UChar second) {
+  return first == '\\' && !IsCSSNewLine(second);
 }
 
-#endif
+// Consumes a single whitespace, if the stream is currently looking at a
+// whitespace. Note that \r\n counts as a single whitespace, as we don't do
+// input preprocessing as a separate step.
+//
+// See https://drafts.csswg.org/css-syntax-3/#input-preprocessing
+void ConsumeSingleWhitespaceIfNext(CSSTokenizerInputStream&);
+
+// https://drafts.csswg.org/css-syntax/#consume-an-escaped-code-point
+UChar32 ConsumeEscape(CSSTokenizerInputStream&);
+
+// http://www.w3.org/TR/css3-syntax/#consume-a-name
+String ConsumeName(CSSTokenizerInputStream&);
+
+// https://drafts.csswg.org/css-syntax/#would-start-an-identifier
+bool NextCharsAreIdentifier(UChar, const CSSTokenizerInputStream&);
+
+}  // namespace blink
+
+#endif  // THIRD_PARTY_BLINK_RENDERER_CORE_CSS_PARSER_CSS_PARSER_IDIOMS_H_

third_party/blink/renderer/core/css/parser/css_tokenizer.cc

@@ -67,16 +67,6 @@ wtf_size_t CSSTokenizer::TokenCount() {
   return token_count_;
 }
 
-static bool IsNewLine(UChar cc) {
-  // We check \r and \f here, since we have no preprocessing stage
-  return (cc == '\r' || cc == '\n' || cc == '\f');
-}
-
-// https://drafts.csswg.org/css-syntax/#check-if-two-code-points-are-a-valid-escape
-static bool TwoCharsAreValidEscape(UChar first, UChar second) {
-  return first == '\\' && !IsNewLine(second);
-}
-
 void CSSTokenizer::Reconsume(UChar c) {
   input_.PushBack(c);
 }
@@ -397,7 +387,7 @@ CSSParserToken CSSTokenizer::ConsumeStringTokenUntil(UChar ending_code_point) {
       input_.Advance(size + 1);
       return CSSParserToken(kStringToken, input_.RangeAt(start_offset, size));
     }
-    if (IsNewLine(cc)) {
+    if (IsCSSNewLine(cc)) {
       input_.Advance(size);
       return CSSParserToken(kBadStringToken);
     }
@@ -410,14 +400,14 @@ CSSParserToken CSSTokenizer::ConsumeStringTokenUntil(UChar ending_code_point) {
     UChar cc = Consume();
     if (cc == ending_code_point || cc == kEndOfFileMarker)
       return CSSParserToken(kStringToken, RegisterString(output.ToString()));
-    if (IsNewLine(cc)) {
+    if (IsCSSNewLine(cc)) {
       Reconsume(cc);
       return CSSParserToken(kBadStringToken);
     }
     if (cc == '\\') {
       if (input_.NextInputChar() == kEndOfFileMarker)
         continue;
-      if (IsNewLine(input_.PeekWithoutReplacement(0)))
+      if (IsCSSNewLine(input_.PeekWithoutReplacement(0)))
         ConsumeSingleWhitespaceIfNext();  // This handles \r\n for us
       else
         output.Append(ConsumeEscape());
@@ -527,12 +517,7 @@ void CSSTokenizer::ConsumeBadUrlRemnants() {
 }
 
 void CSSTokenizer::ConsumeSingleWhitespaceIfNext() {
-  // We check for \r\n and HTML spaces since we don't do preprocessing
-  UChar next = input_.PeekWithoutReplacement(0);
-  if (next == '\r' && input_.PeekWithoutReplacement(1) == '\n')
-    input_.Advance(2);
-  else if (IsHTMLSpace(next))
-    input_.Advance();
+  blink::ConsumeSingleWhitespaceIfNext(input_);
 }
 
 void CSSTokenizer::ConsumeUntilCommentEndFound() {
@@ -581,49 +566,12 @@ StringView CSSTokenizer::ConsumeName() {
     return input_.RangeAt(start_offset, size);
   }
 
-  StringBuilder result;
-  while (true) {
-    UChar cc = Consume();
-    if (IsNameCodePoint(cc)) {
-      result.Append(cc);
-      continue;
-    }
-    if (TwoCharsAreValidEscape(cc, input_.PeekWithoutReplacement(0))) {
-      result.Append(ConsumeEscape());
-      continue;
-    }
-    Reconsume(cc);
-    return RegisterString(result.ToString());
-  }
+  return RegisterString(blink::ConsumeName(input_));
 }
 
 // https://drafts.csswg.org/css-syntax/#consume-an-escaped-code-point
 UChar32 CSSTokenizer::ConsumeEscape() {
-  UChar cc = Consume();
-  DCHECK(!IsNewLine(cc));
-  if (IsASCIIHexDigit(cc)) {
-    unsigned consumed_hex_digits = 1;
-    StringBuilder hex_chars;
-    hex_chars.Append(cc);
-    while (consumed_hex_digits < 6 &&
-           IsASCIIHexDigit(input_.PeekWithoutReplacement(0))) {
-      cc = Consume();
-      hex_chars.Append(cc);
-      consumed_hex_digits++;
-    };
-    ConsumeSingleWhitespaceIfNext();
-    bool ok = false;
-    UChar32 code_point = hex_chars.ToString().HexToUIntStrict(&ok);
-    DCHECK(ok);
-    if (code_point == 0 || (0xD800 <= code_point && code_point <= 0xDFFF) ||
-        code_point > 0x10FFFF)
-      return kReplacementCharacter;
-    return code_point;
-  }
-
-  if (cc == kEndOfFileMarker)
-    return kReplacementCharacter;
-  return cc;
+  return blink::ConsumeEscape(input_);
 }
 
 bool CSSTokenizer::NextTwoCharsAreValidEscape() {
@@ -653,15 +601,7 @@ bool CSSTokenizer::NextCharsAreNumber() {
 
 // https://drafts.csswg.org/css-syntax/#would-start-an-identifier
 bool CSSTokenizer::NextCharsAreIdentifier(UChar first) {
-  UChar second = input_.PeekWithoutReplacement(0);
-  if (IsNameStartCodePoint(first) || TwoCharsAreValidEscape(first, second))
-    return true;
-
-  if (first == '-')
-    return IsNameStartCodePoint(second) || second == '-' ||
-           NextTwoCharsAreValidEscape();
-
-  return false;
+  return blink::NextCharsAreIdentifier(first, input_);
 }
 
 bool CSSTokenizer::NextCharsAreIdentifier() {

third_party/blink/renderer/core/css/property_registration.cc

@@ -7,6 +7,7 @@
 #include "third_party/blink/renderer/core/animation/css_interpolation_types_map.h"
 #include "third_party/blink/renderer/core/css/css_style_sheet.h"
 #include "third_party/blink/renderer/core/css/css_syntax_descriptor.h"
+#include "third_party/blink/renderer/core/css/css_syntax_string_parser.h"
 #include "third_party/blink/renderer/core/css/css_value_list.h"
 #include "third_party/blink/renderer/core/css/css_variable_reference_value.h"
 #include "third_party/blink/renderer/core/css/parser/css_parser_context.h"
@@ -115,8 +116,9 @@ void PropertyRegistration::registerProperty(
     return;
   }
 
-  CSSSyntaxDescriptor syntax_descriptor(descriptor->syntax());
-  if (!syntax_descriptor.IsValid()) {
+  base::Optional<CSSSyntaxDescriptor> syntax_descriptor =
+      CSSSyntaxStringParser(descriptor->syntax()).Parse();
+  if (!syntax_descriptor) {
     exception_state.ThrowDOMException(
         DOMExceptionCode::kSyntaxError,
         "The syntax provided is not a valid custom property syntax.");
@@ -132,8 +134,8 @@ void PropertyRegistration::registerProperty(
     CSSTokenizer tokenizer(descriptor->initialValue());
     const auto tokens = tokenizer.TokenizeToEOF();
     bool is_animation_tainted = false;
-    initial = syntax_descriptor.Parse(CSSParserTokenRange(tokens),
-                                      parser_context, is_animation_tainted);
+    initial = syntax_descriptor->Parse(CSSParserTokenRange(tokens),
+                                       parser_context, is_animation_tainted);
     if (!initial) {
       exception_state.ThrowDOMException(
           DOMExceptionCode::kSyntaxError,
@@ -152,7 +154,7 @@ void PropertyRegistration::registerProperty(
         CSSParserTokenRange(tokens), is_animation_tainted, false,
         parser_context->BaseURL(), parser_context->Charset());
   } else {
-    if (!syntax_descriptor.IsTokenStream()) {
+    if (!syntax_descriptor->IsTokenStream()) {
       exception_state.ThrowDOMException(
           DOMExceptionCode::kSyntaxError,
           "An initial value must be provided if the syntax is not '*'");
@@ -161,7 +163,7 @@ void PropertyRegistration::registerProperty(
   }
   registry.RegisterProperty(
       atomic_name, *MakeGarbageCollected<PropertyRegistration>(
-                       atomic_name, syntax_descriptor, descriptor->inherits(),
+                       atomic_name, *syntax_descriptor, descriptor->inherits(),
                        initial, std::move(initial_variable_data)));
 
   document->GetStyleEngine().CustomPropertyRegistered();

third_party/blink/renderer/core/css/resolver/css_variable_resolver_test.cc

@@ -4,6 +4,7 @@
 
 #include "third_party/blink/renderer/core/css/resolver/css_variable_resolver.h"
 #include "third_party/blink/renderer/core/css/css_custom_property_declaration.h"
+#include "third_party/blink/renderer/core/css/css_syntax_string_parser.h"
 #include "third_party/blink/renderer/core/css/css_variable_reference_value.h"
 #include "third_party/blink/renderer/core/css/document_style_environment_variables.h"
 #include "third_party/blink/renderer/core/css/parser/css_parser_context.h"
@@ -253,17 +254,19 @@ TEST_F(CSSVariableResolverTest, NeedsResolutionClearedByResolver) {
   const auto* prop3 = CreateCustomProperty("--prop3", "var(--prop2)");
 
   // Register prop3 to make it non-inherited.
-  CSSSyntaxDescriptor token_syntax("*");
+  base::Optional<CSSSyntaxDescriptor> token_syntax =
+      CSSSyntaxStringParser("*").Parse();
+  ASSERT_TRUE(token_syntax);
   String initial_value_str("foo");
   const auto tokens = CSSTokenizer(initial_value_str).TokenizeToEOF();
   const CSSParserContext* context = CSSParserContext::Create(GetDocument());
   const CSSValue* initial_value =
-      token_syntax.Parse(CSSParserTokenRange(tokens), context, false);
+      token_syntax->Parse(CSSParserTokenRange(tokens), context, false);
   ASSERT_TRUE(initial_value);
   ASSERT_TRUE(initial_value->IsVariableReferenceValue());
   PropertyRegistration* registration =
       MakeGarbageCollected<PropertyRegistration>(
-          "--prop3", token_syntax, false, initial_value,
+          "--prop3", *token_syntax, false, initial_value,
           ToCSSVariableReferenceValue(*initial_value).VariableDataValue());
   ASSERT_TRUE(GetDocument().GetPropertyRegistry());
   GetDocument().GetPropertyRegistry()->RegisterProperty("--prop3",

third_party/blink/renderer/modules/csspaint/paint_worklet_global_scope.cc

@@ -13,6 +13,7 @@
 #include "third_party/blink/renderer/bindings/modules/v8/v8_paint_rendering_context_2d_settings.h"
 #include "third_party/blink/renderer/core/css/css_property_names.h"
 #include "third_party/blink/renderer/core/css/css_syntax_descriptor.h"
+#include "third_party/blink/renderer/core/css/css_syntax_string_parser.h"
 #include "third_party/blink/renderer/core/dom/document.h"
 #include "third_party/blink/renderer/core/frame/local_dom_window.h"
 #include "third_party/blink/renderer/core/frame/local_frame.h"
@@ -56,12 +57,13 @@ bool ParseInputArguments(v8::Local<v8::Context> context,
         return false;
 
       for (const auto& type : argument_types) {
-        CSSSyntaxDescriptor syntax_descriptor(type);
-        if (!syntax_descriptor.IsValid()) {
+        base::Optional<CSSSyntaxDescriptor> syntax_descriptor =
+            CSSSyntaxStringParser(type).Parse();
+        if (!syntax_descriptor) {
           exception_state->ThrowTypeError("Invalid argument types.");
           return false;
         }
-        input_argument_types->push_back(std::move(syntax_descriptor));
+        input_argument_types->push_back(std::move(*syntax_descriptor));
       }
     }
   }

third_party/blink/web_tests/external/wpt/css/css-properties-values-api/register-property-syntax-parsing-expected.txt

 This is a testharness.js-based test.
-Found 133 tests; 132 PASS, 1 FAIL, 0 TIMEOUT, 0 NOTRUN.
+Found 149 tests; 148 PASS, 1 FAIL, 0 TIMEOUT, 0 NOTRUN.
 PASS syntax:'*', initialValue:'a' is valid
 PASS syntax:' * ', initialValue:'b' is valid
 PASS syntax:'<length>', initialValue:'2px' is valid
@@ -7,9 +7,12 @@ PASS syntax:' <number>', initialValue:'5' is valid
 PASS syntax:'<percentage> ', initialValue:'10%' is valid
 PASS syntax:'<color>+', initialValue:'red' is valid
 PASS syntax:' <length>+ | <percentage>', initialValue:'2px 8px' is valid
+PASS syntax:' <length>+ | <color>#', initialValue:'red, blue' is valid
 PASS syntax:'<length>|<percentage>|<length-percentage>', initialValue:'2px' is valid
 PASS syntax:'<color> | <image> | <url> | <integer> | <angle>', initialValue:'red' is valid
 PASS syntax:'<time> | <resolution> | <transform-list> | <custom-ident>', initialValue:'red' is valid
+PASS syntax:'	<color>
+|   foo', initialValue:'foo' is valid
 PASS syntax:'*', initialValue:':> hello' is valid
 PASS syntax:'*', initialValue:'([ brackets ]) { yay (??)}' is valid
 PASS syntax:'*', initialValue:'yep 'this is valid too'' is valid
@@ -58,6 +61,7 @@ PASS syntax:'banana', initialValue:'banana' is valid
 PASS syntax:'bAnAnA', initialValue:'bAnAnA' is valid
 PASS syntax:'ba-na-nya', initialValue:'ba-na-nya' is valid
 PASS syntax:'banana', initialValue:'banan\61' is valid
+PASS syntax:'banan\61', initialValue:'banana' is valid
 PASS syntax:'<custom-ident>', initialValue:'banan\61' is valid
 PASS syntax:'big | bigger | BIGGER', initialValue:'bigger' is valid
 PASS syntax:'foo+|bar', initialValue:'foo foo foo' is valid
@@ -71,8 +75,12 @@ PASS syntax:'ba
 PASS syntax:'null', initialValue:'null' is valid
 PASS syntax:'undefined', initialValue:'undefined' is valid
 PASS syntax:'array', initialValue:'array' is valid
+PASS syntax:'\1F914', initialValue:'🤔' is valid
+PASS syntax:'hmm\1F914', initialValue:'hmm🤔' is valid
+PASS syntax:'\1F914hmm', initialValue:'🤔hmm' is valid
+PASS syntax:'\1F914 hmm', initialValue:'🤔hmm' is valid
+PASS syntax:'\1F914\1F914', initialValue:'🤔🤔' is valid
 PASS syntax:'banana,nya', initialValue:'banana' is invalid
-PASS syntax:'banan\61', initialValue:'banana' is invalid
 PASS syntax:'<\6c ength>', initialValue:'10px' is invalid
 PASS syntax:'<banana>', initialValue:'banana' is invalid
 PASS syntax:'<Number>', initialValue:'10' is invalid
@@ -81,10 +89,19 @@ PASS syntax:'<LENGTH>', initialValue:'10px' is invalid
 PASS syntax:'< length>', initialValue:'10px' is invalid
 PASS syntax:'<length >', initialValue:'10px' is invalid
 PASS syntax:'<length> +', initialValue:'10px' is invalid
+PASS syntax:'<transform-list>+', initialValue:'scale(2)' is invalid
+PASS syntax:'<transform-list>#', initialValue:'scale(2)' is invalid
 PASS syntax:'<length>++', initialValue:'10px' is invalid
+PASS syntax:'<length>##', initialValue:'10px' is invalid
+PASS syntax:'<length>+#', initialValue:'10px' is invalid
+PASS syntax:'<length>#+', initialValue:'10px' is invalid
 PASS syntax:'<length> | *', initialValue:'10px' is invalid
 PASS syntax:'*|banana', initialValue:'banana' is invalid
+PASS syntax:'|banana', initialValue:'banana' is invalid
 PASS syntax:'*+', initialValue:'banana' is invalid
+PASS syntax:'|', initialValue:'banana' is invalid
+PASS syntax:' |', initialValue:'banana' is invalid
+PASS syntax:'||', initialValue:'banana' is invalid
 PASS syntax:'initial', initialValue:'initial' is invalid
 PASS syntax:'inherit', initialValue:'inherit' is invalid
 PASS syntax:'unset', initialValue:'unset' is invalid

third_party/blink/web_tests/external/wpt/css/css-properties-values-api/register-property-syntax-parsing.html

 <!DOCTYPE HTML>
+<meta charset="utf-8">
 <link rel="help" href="https://drafts.css-houdini.org/css-properties-values-api/#dom-css-registerproperty" />
 <link rel="help" href="https://drafts.css-houdini.org/css-properties-values-api/#supported-syntax-strings" />
 <script src="/resources/testharness.js"></script>
@@ -29,9 +30,11 @@ assert_valid(" <number>", "5");
 assert_valid("<percentage> ", "10%");
 assert_valid("<color>+", "red");
 assert_valid(" <length>+ | <percentage>", "2px 8px");
+assert_valid(" <length>+ | <color>#", "red, blue");
 assert_valid("<length>|<percentage>|<length-percentage>", "2px"); // Valid but silly
 assert_valid("<color> | <image> | <url> | <integer> | <angle>", "red");
 assert_valid("<time> | <resolution> | <transform-list> | <custom-ident>", "red");
+assert_valid("\t<color>\n|   foo", "foo");
 
 assert_valid("*", ":> hello");
 assert_valid("*", "([ brackets ]) { yay (??)}");
@@ -86,6 +89,7 @@ assert_valid("banana", "banana");
 assert_valid("bAnAnA", "bAnAnA");
 assert_valid("ba-na-nya", "ba-na-nya");
 assert_valid("banana", "banan\\61");
+assert_valid("banan\\61", "banana");
 assert_valid("<custom-ident>", "banan\\61");
 assert_valid("big | bigger | BIGGER", "bigger");
 assert_valid("foo+|bar", "foo foo foo");
@@ -99,9 +103,14 @@ assert_valid(null, "null");
 assert_valid(undefined, "undefined");
 assert_valid(["array"], "array");
 
+assert_valid("\\1F914", "🤔");
+assert_valid("hmm\\1F914", "hmm🤔");
+assert_valid("\\1F914hmm", "🤔hmm");
+assert_valid("\\1F914 hmm", "🤔hmm");
+assert_valid("\\1F914\\1F914", "🤔🤔");
+
 // Invalid syntax
 assert_invalid("banana,nya", "banana");
-assert_invalid("banan\\61", "banana");
 assert_invalid("<\\6c ength>", "10px");
 assert_invalid("<banana>", "banana");
 assert_invalid("<Number>", "10");
@@ -110,11 +119,20 @@ assert_invalid("<LENGTH>", "10px");
 assert_invalid("< length>", "10px");
 assert_invalid("<length >", "10px");
 assert_invalid("<length> +", "10px");
+assert_invalid("<transform-list>+", "scale(2)");
+assert_invalid("<transform-list>#", "scale(2)");
 
 assert_invalid("<length>++", "10px");
+assert_invalid("<length>##", "10px");
+assert_invalid("<length>+#", "10px");
+assert_invalid("<length>#+", "10px");
 assert_invalid("<length> | *", "10px");
 assert_invalid("*|banana", "banana");
+assert_invalid("|banana", "banana");
 assert_invalid("*+", "banana");
+assert_invalid("|", "banana");
+assert_invalid(" |", "banana");
+assert_invalid("||", "banana");
 
 assert_invalid("initial", "initial");
 assert_invalid("inherit", "inherit");