[CSSParser] Use streaming tokenizer for @supports rules and CSS.supports

This gets us one step closer to using the streaming tokenizer for
declarations, which will allow us to preserve raw text for CSS variable
values.

The existing logic made heavy use of sub-ranges to track block
boundaries and restart parsing to find the correct branch of a
production. To support the new logic, the streaming tokenizer's
BlockGuard gained the ability to tell its caller whether the end of
the block it was created for has been reached. Since the streaming
parser cannot be rewound to a previous position, the supports parser
had to be restructured to peek ahead more than one token for the
supports-in-parens grammar, and to decide based on lead token whether
to parse supports-feature or general-enclosed.

Finally, to maintain support for the CSSConditionRule.conditionText
property, the streaming tokenizer gained the ability to return raw
text for an underlying range.

Bug: 661854
Change-Id: I7fcf244e5f360e3715c8543ac7283e351d6b2baf
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2238512Reviewed-by: Anders Hartvoll Ruud <andruud@chromium.org>
Commit-Queue: Kevin Babbitt <kbabbitt@microsoft.com>
Cr-Commit-Position: refs/heads/master@{#789195}

third_party/blink/renderer/core/css/parser/css_parser.cc

@@ -10,6 +10,7 @@
 #include "third_party/blink/renderer/core/css/css_keyframe_rule.h"
 #include "third_party/blink/renderer/core/css/parser/css_parser_fast_paths.h"
 #include "third_party/blink/renderer/core/css/parser/css_parser_impl.h"
+#include "third_party/blink/renderer/core/css/parser/css_parser_token_stream.h"
 #include "third_party/blink/renderer/core/css/parser/css_property_parser.h"
 #include "third_party/blink/renderer/core/css/parser/css_selector_parser.h"
 #include "third_party/blink/renderer/core/css/parser/css_supports_parser.h"
@@ -212,13 +213,17 @@ StyleRuleKeyframe* CSSParser::ParseKeyframeRule(const CSSParserContext* context,
 
 bool CSSParser::ParseSupportsCondition(const String& condition,
                                        SecureContextMode secure_context_mode) {
-  CSSTokenizer tokenizer(condition);
-  const auto tokens = tokenizer.TokenizeToEOF();
+  // window.CSS.supports requires to parse as-if it was wrapped in parenthesis.
+  String wrapped_condition = "(" + condition + ")";
+  CSSTokenizer tokenizer(wrapped_condition);
+  CSSParserTokenStream stream(tokenizer);
   CSSParserImpl parser(StrictCSSParserContext(secure_context_mode));
-  return CSSSupportsParser::SupportsCondition(
-             CSSParserTokenRange(tokens), parser,
-             CSSSupportsParser::Mode::kForWindowCSS) ==
-         CSSSupportsParser::Result::kSupported;
+  CSSSupportsParser::Result result =
+      CSSSupportsParser::ConsumeSupportsCondition(stream, parser);
+  if (!stream.AtEnd())
+    result = CSSSupportsParser::Result::kParseFailure;
+
+  return result == CSSSupportsParser::Result::kSupported;
 }
 
 bool CSSParser::ParseColor(Color& color, const String& string, bool strict) {

third_party/blink/renderer/core/css/parser/css_parser_impl.cc

@@ -730,14 +730,19 @@ StyleRuleMedia* CSSParserImpl::ConsumeMediaRule(CSSParserTokenStream& stream) {
 StyleRuleSupports* CSSParserImpl::ConsumeSupportsRule(
     CSSParserTokenStream& stream) {
   wtf_size_t prelude_offset_start = stream.LookAheadOffset();
-  CSSParserTokenRange prelude = ConsumeAtRulePrelude(stream);
+  CSSSupportsParser::Result supported =
+      CSSSupportsParser::ConsumeSupportsCondition(stream, *this);
+  // Check whether the entire prelude was consumed. If it wasn't, ensure we
+  // consume any leftovers plus the block before returning a parse error.
+  stream.ConsumeWhitespace();
+  CSSParserTokenRange prelude_remainder = ConsumeAtRulePrelude(stream);
+  if (!prelude_remainder.AtEnd())
+    supported = CSSSupportsParser::Result::kParseFailure;
   wtf_size_t prelude_offset_end = stream.LookAheadOffset();
   if (!ConsumeEndOfPreludeForAtRuleWithBlock(stream))
     return nullptr;
   CSSParserTokenStream::BlockGuard guard(stream);
 
-  CSSSupportsParser::Result supported = CSSSupportsParser::SupportsCondition(
-      prelude, *this, CSSSupportsParser::Mode::kForAtRule);
   if (supported == CSSSupportsParser::Result::kParseFailure)
     return nullptr;  // Parse error, invalid @supports condition
 
@@ -747,7 +752,12 @@ StyleRuleSupports* CSSParserImpl::ConsumeSupportsRule(
     observer_->StartRuleBody(stream.Offset());
   }
 
-  const auto prelude_serialized = prelude.Serialize().StripWhiteSpace();
+  const auto prelude_serialized =
+      stream
+          .StringRangeAt(prelude_offset_start,
+                         prelude_offset_end - prelude_offset_start)
+          .ToString()
+          .SimplifyWhiteSpace();
 
   HeapVector<Member<StyleRuleBase>> rules;
   ConsumeRuleList(stream, kRegularRuleList,

third_party/blink/renderer/core/css/parser/css_parser_token_stream.cc

@@ -6,6 +6,15 @@
 
 namespace blink {
 
+StringView CSSParserTokenStream::StringRangeAt(wtf_size_t start,
+                                               wtf_size_t length) const {
+  return tokenizer_.StringRangeAt(start, length);
+}
+
+wtf_size_t CSSParserTokenStream::BlockStackDepth() const {
+  return tokenizer_.BlockStackDepth();
+}
+
 void CSSParserTokenStream::ConsumeWhitespace() {
   while (Peek().GetType() == kWhitespaceToken)
     UncheckedConsume();

third_party/blink/renderer/core/css/parser/css_parser_token_stream.h

@@ -47,15 +47,24 @@ class CORE_EXPORT CSSParserTokenStream {
     explicit BlockGuard(CSSParserTokenStream& stream) : stream_(stream) {
       const CSSParserToken next = stream.ConsumeInternal();
       DCHECK_EQ(next.GetBlockType(), CSSParserToken::kBlockStart);
+      initial_stack_depth_ = stream_.BlockStackDepth();
+      DCHECK_GT(initial_stack_depth_, 0u);
     }
 
     ~BlockGuard() {
       stream_.EnsureLookAhead();
       stream_.UncheckedSkipToEndOfBlock();
+      DCHECK(AtEndOfBlock());
+    }
+
+    bool AtEndOfBlock() const {
+      return (stream_.BlockStackDepth() == initial_stack_depth_ - 1) ||
+             stream_.AtEnd();
     }
 
    private:
     CSSParserTokenStream& stream_;
+    wtf_size_t initial_stack_depth_;
   };
 
   // We found that this value works well empirically by printing out the
@@ -132,6 +141,12 @@ class CORE_EXPORT CSSParserTokenStream {
     return tokenizer_.PreviousOffset();
   }
 
+  // Returns a view on a range of characters in the original string.
+  StringView StringRangeAt(wtf_size_t start, wtf_size_t length) const;
+
+  // Returns the block stack depth for the underlying tokenizer.
+  wtf_size_t BlockStackDepth() const;
+
   void ConsumeWhitespace();
   CSSParserToken ConsumeIncludingWhitespace();
   void UncheckedConsumeComponentValue();

third_party/blink/renderer/core/css/parser/css_supports_parser.h

@@ -11,14 +11,13 @@
 namespace blink {
 
 class CSSParserImpl;
-class CSSParserTokenRange;
+class CSSParserToken;
+class CSSParserTokenStream;
 
 class CORE_EXPORT CSSSupportsParser {
   STACK_ALLOCATED();
 
  public:
-  enum class Mode { kForAtRule, kForWindowCSS };
-
   enum class Result {
     // kUnsupported/kSupported means that we parsed the @supports
     // successfully, and conclusively determined that we either support or
@@ -47,20 +46,20 @@ class CORE_EXPORT CSSSupportsParser {
     kParseFailure
   };
 
-  static Result SupportsCondition(CSSParserTokenRange, CSSParserImpl&, Mode);
+  static Result ConsumeSupportsCondition(CSSParserTokenStream&, CSSParserImpl&);
 
  private:
   friend class CSSSupportsParserTest;
 
   CSSSupportsParser(CSSParserImpl& parser) : parser_(parser) {}
 
-  // True if the current token is a kIdentToken with the specified value
+  // True if the given token is a kIdentToken with the specified value
   // (case-insensitive).
-  bool AtIdent(CSSParserTokenRange&, const char*);
+  static bool AtIdent(const CSSParserToken&, const char*);
 
   // If the current token is a kIdentToken with the specified value (case
   // insensitive), consumes the token and returns true.
-  bool ConsumeIfIdent(CSSParserTokenRange&, const char*);
+  static bool ConsumeIfIdent(CSSParserTokenStream&, const char*);
 
   // Parsing functions follow, as defined by:
   // https://drafts.csswg.org/css-conditional-3/#typedef-supports-condition
@@ -68,24 +67,35 @@ class CORE_EXPORT CSSSupportsParser {
   // <supports-condition> = not <supports-in-parens>
   //                   | <supports-in-parens> [ and <supports-in-parens> ]*
   //                   | <supports-in-parens> [ or <supports-in-parens> ]*
-  Result ConsumeSupportsCondition(CSSParserTokenRange&);
+  Result ConsumeSupportsCondition(CSSParserTokenStream&);
 
   // <supports-in-parens> = ( <supports-condition> )
   //                    | <supports-feature>
   //                    | <general-enclosed>
-  Result ConsumeSupportsInParens(CSSParserTokenRange&);
+  Result ConsumeSupportsInParens(CSSParserTokenStream&);
 
   // <supports-feature> = <supports-selector-fn> | <supports-decl>
-  Result ConsumeSupportsFeature(CSSParserTokenRange&);
+  Result ConsumeSupportsFeature(const CSSParserToken&, CSSParserTokenStream&);
 
   // <supports-selector-fn> = selector( <complex-selector> )
-  Result ConsumeSupportsSelectorFn(CSSParserTokenRange&);
+  Result ConsumeSupportsSelectorFn(const CSSParserToken&,
+                                   CSSParserTokenStream&);
 
   // <supports-decl> = ( <declaration> )
-  Result ConsumeSupportsDecl(CSSParserTokenRange&);
+  Result ConsumeSupportsDecl(const CSSParserToken&, CSSParserTokenStream&);
 
   // <general-enclosed>
-  Result ConsumeGeneralEnclosed(CSSParserTokenRange&);
+  Result ConsumeGeneralEnclosed(const CSSParserToken&, CSSParserTokenStream&);
+
+  // Parsing helpers.
+  static bool IsSupportsInParens(const CSSParserToken&);
+  static bool IsEnclosedSupportsCondition(const CSSParserToken&,
+                                          const CSSParserToken&);
+  static bool IsSupportsSelectorFn(const CSSParserToken&,
+                                   const CSSParserToken&);
+  static bool IsSupportsDecl(const CSSParserToken&, const CSSParserToken&);
+  static bool IsSupportsFeature(const CSSParserToken&, const CSSParserToken&);
+  static bool IsGeneralEnclosed(const CSSParserToken&);
 
   CSSParserImpl& parser_;
 };

third_party/blink/renderer/core/css/parser/css_supports_parser_test.cc

@@ -6,6 +6,7 @@
 #include "testing/gtest/include/gtest/gtest.h"
 #include "third_party/blink/renderer/core/css/parser/css_parser_context.h"
 #include "third_party/blink/renderer/core/css/parser/css_parser_impl.h"
+#include "third_party/blink/renderer/core/css/parser/css_parser_token_stream.h"
 #include "third_party/blink/renderer/core/css/parser/css_tokenizer.h"
 #include "third_party/blink/renderer/platform/testing/runtime_enabled_features_test_helpers.h"
 
@@ -24,66 +25,81 @@ class CSSSupportsParserTest : public testing::Test {
     return CSSTokenizer(string).TokenizeToEOF();
   }
 
-  Result SupportsCondition(String string, CSSSupportsParser::Mode mode) {
+  Result StaticConsumeSupportsCondition(String string) {
     CSSParserImpl impl(MakeContext());
-    auto tokens = Tokenize(string);
-    return CSSSupportsParser::SupportsCondition(tokens, impl, mode);
+    CSSTokenizer tokenizer(string);
+    CSSParserTokenStream stream(tokenizer);
+    Result result = CSSSupportsParser::ConsumeSupportsCondition(stream, impl);
+    return stream.AtEnd() ? result : Result::kParseFailure;
   }
 
   Result AtSupports(String string) {
-    return SupportsCondition(string, CSSSupportsParser::Mode::kForAtRule);
+    return StaticConsumeSupportsCondition(string);
   }
 
   Result WindowCSSSupports(String string) {
-    return SupportsCondition(string, CSSSupportsParser::Mode::kForWindowCSS);
+    String wrapped_condition = "(" + string + ")";
+    return StaticConsumeSupportsCondition(wrapped_condition);
   }
 
   Result ConsumeSupportsCondition(String string) {
     CSSParserImpl impl(MakeContext());
     CSSSupportsParser parser(impl);
-    auto tokens = Tokenize(string);
-    CSSParserTokenRange range = tokens;
-    return parser.ConsumeSupportsCondition(range);
+    CSSTokenizer tokenizer(string);
+    CSSParserTokenStream stream(tokenizer);
+    return parser.ConsumeSupportsCondition(stream);
   }
 
   Result ConsumeSupportsInParens(String string) {
     CSSParserImpl impl(MakeContext());
     CSSSupportsParser parser(impl);
-    auto tokens = Tokenize(string);
-    CSSParserTokenRange range = tokens;
-    return parser.ConsumeSupportsInParens(range);
+    CSSTokenizer tokenizer(string);
+    CSSParserTokenStream stream(tokenizer);
+    return parser.ConsumeSupportsInParens(stream);
   }
 
   Result ConsumeSupportsFeature(String string) {
     CSSParserImpl impl(MakeContext());
     CSSSupportsParser parser(impl);
-    auto tokens = Tokenize(string);
-    CSSParserTokenRange range = tokens;
-    return parser.ConsumeSupportsFeature(range);
+    CSSTokenizer tokenizer(string);
+    CSSParserTokenStream stream(tokenizer);
+    CSSParserToken first_token = stream.Peek();
+    CSSParserTokenStream::BlockGuard guard(stream);
+    stream.ConsumeWhitespace();
+    return parser.ConsumeSupportsFeature(first_token, stream);
   }
 
   Result ConsumeSupportsSelectorFn(String string) {
     CSSParserImpl impl(MakeContext());
     CSSSupportsParser parser(impl);
-    auto tokens = Tokenize(string);
-    CSSParserTokenRange range = tokens;
-    return parser.ConsumeSupportsSelectorFn(range);
+    CSSTokenizer tokenizer(string);
+    CSSParserTokenStream stream(tokenizer);
+    CSSParserToken first_token = stream.Peek();
+    CSSParserTokenStream::BlockGuard guard(stream);
+    stream.ConsumeWhitespace();
+    return parser.ConsumeSupportsSelectorFn(first_token, stream);
   }
 
   Result ConsumeSupportsDecl(String string) {
     CSSParserImpl impl(MakeContext());
     CSSSupportsParser parser(impl);
-    auto tokens = Tokenize(string);
-    CSSParserTokenRange range = tokens;
-    return parser.ConsumeSupportsDecl(range);
+    CSSTokenizer tokenizer(string);
+    CSSParserTokenStream stream(tokenizer);
+    CSSParserToken first_token = stream.Peek();
+    CSSParserTokenStream::BlockGuard guard(stream);
+    stream.ConsumeWhitespace();
+    return parser.ConsumeSupportsDecl(first_token, stream);
   }
 
   Result ConsumeGeneralEnclosed(String string) {
     CSSParserImpl impl(MakeContext());
     CSSSupportsParser parser(impl);
-    auto tokens = Tokenize(string);
-    CSSParserTokenRange range = tokens;
-    return parser.ConsumeGeneralEnclosed(range);
+    CSSTokenizer tokenizer(string);
+    CSSParserTokenStream stream(tokenizer);
+    CSSParserToken first_token = stream.Peek();
+    CSSParserTokenStream::BlockGuard guard(stream);
+    stream.ConsumeWhitespace();
+    return parser.ConsumeGeneralEnclosed(first_token, stream);
   }
 };
 
@@ -190,7 +206,27 @@ TEST_F(CSSSupportsParserTest, ConsumeSupportsInParens) {
               ConsumeSupportsInParens("selector(div)or (color:green)"));
     EXPECT_EQ(Result::kSupported,
               ConsumeSupportsInParens("selector(div)and (color:green)"));
+
+    // Invalid <supports-selector-fn> formerly handled by
+    // ConsumeSupportsSelectorFn()
+    EXPECT_EQ(Result::kParseFailure, ConsumeSupportsInParens("#test"));
+    EXPECT_EQ(Result::kParseFailure, ConsumeSupportsInParens("test"));
+
+    // Invalid <supports-selector-fn> but valid <general-enclosed>
+    EXPECT_EQ(Result::kUnsupported, ConsumeSupportsInParens("test(1)"));
   }
+
+  // Invalid <supports-decl> formerly handled by ConsumeSupportsDecl()
+  EXPECT_EQ(Result::kParseFailure, ConsumeSupportsInParens(""));
+  EXPECT_EQ(Result::kParseFailure, ConsumeSupportsInParens(")"));
+  EXPECT_EQ(Result::kParseFailure, ConsumeSupportsInParens("color:red)"));
+  EXPECT_EQ(Result::kParseFailure, ConsumeSupportsInParens("color:red"));
+
+  // Invalid <general-enclosed> formerly handled by ConsumeGeneralEnclosed()
+  EXPECT_EQ(Result::kParseFailure, ConsumeSupportsInParens(""));
+  EXPECT_EQ(Result::kParseFailure, ConsumeSupportsInParens(")"));
+  EXPECT_EQ(Result::kParseFailure, ConsumeSupportsInParens("color:red"));
+  EXPECT_EQ(Result::kParseFailure, ConsumeSupportsInParens("asdf"));
 }
 
 TEST_F(CSSSupportsParserTest, ConsumeSupportsSelectorFn) {
@@ -256,10 +292,6 @@ TEST_F(CSSSupportsParserTest, ConsumeSupportsSelectorFn) {
   EXPECT_EQ(Result::kUnsupported, ConsumeSupportsSelectorFn("selector(:asdf)"));
   EXPECT_EQ(Result::kUnsupported,
             ConsumeSupportsSelectorFn("selector(::asdf)"));
-
-  EXPECT_EQ(Result::kParseFailure, ConsumeSupportsSelectorFn("#test"));
-  EXPECT_EQ(Result::kParseFailure, ConsumeSupportsSelectorFn("test"));
-  EXPECT_EQ(Result::kParseFailure, ConsumeSupportsSelectorFn("test(1)"));
 }
 
 TEST_F(CSSSupportsParserTest, ConsumeSupportsSelectorFnWithFeatureDisabled) {
@@ -290,12 +322,8 @@ TEST_F(CSSSupportsParserTest, ConsumeSupportsDecl) {
   EXPECT_EQ(Result::kUnsupported, ConsumeSupportsDecl("(color)"));
   EXPECT_EQ(Result::kUnsupported, ConsumeSupportsDecl("(color:)"));
 
-  EXPECT_EQ(Result::kParseFailure, ConsumeSupportsDecl(""));
   EXPECT_EQ(Result::kParseFailure, ConsumeSupportsDecl("("));
-  EXPECT_EQ(Result::kParseFailure, ConsumeSupportsDecl(")"));
   EXPECT_EQ(Result::kParseFailure, ConsumeSupportsDecl("()"));
-  EXPECT_EQ(Result::kParseFailure, ConsumeSupportsDecl("color:red)"));
-  EXPECT_EQ(Result::kParseFailure, ConsumeSupportsDecl("color:red"));
 }
 
 TEST_F(CSSSupportsParserTest, ConsumeSupportsFeature) {
@@ -315,26 +343,8 @@ TEST_F(CSSSupportsParserTest, ConsumeGeneralEnclosed) {
   EXPECT_EQ(Result::kUnknown, ConsumeGeneralEnclosed("asdf(1, 2)"));
   EXPECT_EQ(Result::kUnknown, ConsumeGeneralEnclosed("asdf(1, 2)\t"));
 
-  EXPECT_EQ(Result::kParseFailure, ConsumeGeneralEnclosed(""));
-  EXPECT_EQ(Result::kParseFailure, ConsumeGeneralEnclosed("("));
-  EXPECT_EQ(Result::kParseFailure, ConsumeGeneralEnclosed(")"));
-  EXPECT_EQ(Result::kParseFailure, ConsumeGeneralEnclosed("()"));
-  EXPECT_EQ(Result::kParseFailure, ConsumeGeneralEnclosed("color:red"));
-  EXPECT_EQ(Result::kParseFailure, ConsumeGeneralEnclosed("asdf"));
-
-  EXPECT_EQ(Result::kUnknown, ConsumeGeneralEnclosed("(asdf)"));
-  EXPECT_EQ(Result::kUnknown, ConsumeGeneralEnclosed("( asdf )"));
-  EXPECT_EQ(Result::kUnknown, ConsumeGeneralEnclosed("(3)"));
-  EXPECT_EQ(Result::kUnknown, ConsumeGeneralEnclosed("max(1, 2)"));
-  EXPECT_EQ(Result::kUnknown, ConsumeGeneralEnclosed("asdf(1, 2)"));
-  EXPECT_EQ(Result::kUnknown, ConsumeGeneralEnclosed("asdf(1, 2)\t"));
-
-  EXPECT_EQ(Result::kParseFailure, ConsumeGeneralEnclosed(""));
   EXPECT_EQ(Result::kParseFailure, ConsumeGeneralEnclosed("("));
-  EXPECT_EQ(Result::kParseFailure, ConsumeGeneralEnclosed(")"));
   EXPECT_EQ(Result::kParseFailure, ConsumeGeneralEnclosed("()"));
-  EXPECT_EQ(Result::kParseFailure, ConsumeGeneralEnclosed("color:red"));
-  EXPECT_EQ(Result::kParseFailure, ConsumeGeneralEnclosed("asdf"));
 
   // Invalid <any-value>:
   EXPECT_EQ(Result::kParseFailure, ConsumeGeneralEnclosed("(asdf})"));

third_party/blink/renderer/core/css/parser/css_tokenizer.cc

@@ -48,6 +48,15 @@ Vector<CSSParserToken, 32> CSSTokenizer::TokenizeToEOF() {
   }
 }
 
+StringView CSSTokenizer::StringRangeAt(wtf_size_t start,
+                                       wtf_size_t length) const {
+  return input_.RangeAt(start, length);
+}
+
+wtf_size_t CSSTokenizer::BlockStackDepth() const {
+  return block_stack_.size();
+}
+
 CSSParserToken CSSTokenizer::TokenizeSingle() {
   while (true) {
     prev_offset_ = input_.Offset();

third_party/blink/renderer/core/css/parser/css_tokenizer.h

@@ -31,6 +31,8 @@ class CORE_EXPORT CSSTokenizer {
 
   wtf_size_t Offset() const { return input_.Offset(); }
   wtf_size_t PreviousOffset() const { return prev_offset_; }
+  StringView StringRangeAt(wtf_size_t start, wtf_size_t length) const;
+  wtf_size_t BlockStackDepth() const;
 
  private:
   CSSParserToken TokenizeSingle();