PDF a11y: Don't break paragraphs on font changes

The current heuristics always creates a new paragraph whenever the
font-size changes, which is undesirable.

Simply removing the size check causes many of the existing tests to fail
because the size-change is often the only reason that we have multiple
paragraphs at all.  The problem is our PDF file has too few lines on
each page to compute reasonable line & paragraph size thresholds.

So this change required changing the heuristics.  The new heuristic is
as follows:

1. We keep track of the top & bottom of the current line, as weighted
averages of the (recent) text boxes on the line.
2. When we encounter a new text box, if it significantly overlaps the
top-to-bottom range, it's considered part of the same line.
3. If we are starting a new line, we also check the paragraph threshold
to see if we should also start a new paragraph.  If the paragraph
threshold couldn't be computed (because there weren't enough lines on
the page), we compare against the line size.

We also introduce the `PDFExtensionAccessibilityTextExtractionTest`
test suite.  These tests are like the tree-dump tests, but they dump
raw text content, split into lines and paragraphs.  (Compared to
tree-dump tests, this approach allows us to test the kNextOnLine and
kPreviousOnLine attributes are correct.)

Bug: 985604
Change-Id: Idfce6edfef42580e7fac4d8a7753c82495c15bd1
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1743032
Commit-Queue: Ian Prest <iapres@microsoft.com>
Reviewed-by: Lei Zhang <thestig@chromium.org>
Reviewed-by: Kevin Babbitt <kbabbitt@microsoft.com>
Cr-Commit-Position: refs/heads/master@{#690153}

chrome/test/data/pdf/accessibility/directional-text-runs-expected-auralinux.txt

@@ -4,6 +4,4 @@
 ++++++[paragraph]
 ++++++++[text] name='Hello,<newline> world!<newline><newline>Goodbye,<newline> world!<newline>'
 ++++++[paragraph]
-++++++++[text] name='This is some text '
-++++++[paragraph]
-++++++++[text] name='abc<newline>def'
+++++++++[text] name='This is some text abc<newline>def'
\ No newline at end of file

chrome/test/data/pdf/accessibility/directional-text-runs-expected-blink.txt

@@ -8,9 +8,7 @@ embeddedObject
 ++++++++++inlineTextBox name='Goodbye,<newline> ' restriction=readOnly
 ++++++++++inlineTextBox name='world!<newline>' restriction=readOnly
 ++++++paragraph restriction=readOnly
-++++++++staticText name='This is some text ' restriction=readOnly
+++++++++staticText name='This is some text abc<newline>def' restriction=readOnly
 ++++++++++inlineTextBox name='This is some text ' restriction=readOnly
-++++++paragraph restriction=readOnly
-++++++++staticText name='abc<newline>def' restriction=readOnly
 ++++++++++inlineTextBox name='abc<newline>' restriction=readOnly
-++++++++++inlineTextBox name='def' restriction=readOnly
+++++++++++inlineTextBox name='def' restriction=readOnly
\ No newline at end of file

chrome/test/data/pdf/accessibility/directional-text-runs-expected-mac.txt

@@ -4,6 +4,4 @@ AXGroup AXDescription='Page 1'
 ++++++AXGroup
 ++++++++AXStaticText AXValue='Hello,<newline> world!<newline><newline>Goodbye,<newline> world!<newline>'
 ++++++AXGroup
-++++++++AXStaticText AXValue='This is some text '
-++++++AXGroup
-++++++++AXStaticText AXValue='abc<newline>def'
+++++++++AXStaticText AXValue='This is some text abc<newline>def'
\ No newline at end of file

chrome/test/data/pdf/accessibility/directional-text-runs-expected-uia-win.txt

@@ -4,6 +4,4 @@ group
 ++++++group
 ++++++++description Name='Hello,<newline> world!<newline><newline>Goodbye,<newline> world!<newline>'
 ++++++group
-++++++++description Name='This is some text '
-++++++group
-++++++++description Name='abc<newline>def'
+++++++++description Name='This is some text abc<newline>def'
\ No newline at end of file

chrome/test/data/pdf/accessibility/directional-text-runs-expected-win.txt

@@ -4,6 +4,4 @@ ROLE_SYSTEM_GROUPING FOCUSABLE
 ++++++IA2_ROLE_PARAGRAPH READONLY
 ++++++++ROLE_SYSTEM_STATICTEXT name='Hello,<newline> world!<newline><newline>Goodbye,<newline> world!<newline>' READONLY
 ++++++IA2_ROLE_PARAGRAPH READONLY
-++++++++ROLE_SYSTEM_STATICTEXT name='This is some text ' READONLY
-++++++IA2_ROLE_PARAGRAPH READONLY
-++++++++ROLE_SYSTEM_STATICTEXT name='abc<newline>def' READONLY
+++++++++ROLE_SYSTEM_STATICTEXT name='This is some text abc<newline>def' READONLY
\ No newline at end of file

chrome/test/data/pdf/accessibility/drop-cap-expected.txt

+This is a test with a simulated drop
+cap. A drop cap is a large initial
+letter that drops below the first line
+of a paragraph. The drop cap should be
+considered part of the first line.
+¶
+This is another test with a simulated
+drop cap. We use large
+amounts of whitespace to defeat
+the PDFium text-run logic.

chrome/test/data/pdf/accessibility/drop-cap.in

+{{header}}
+{{object 1 0}} <<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+{{object 2 0}} <<
+  /Type /Pages
+  /MediaBox [0 0 200 200]
+  /Count 1
+  /Kids [3 0 R]
+>>
+endobj
+{{object 3 0}} <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources <<
+    /Font <<
+      /F1 4 0 R
+    >>
+  >>
+  /Contents 5 0 R
+>>
+endobj
+{{object 4 0}} <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Helvetica
+>>
+endobj
+{{object 5 0}} <<
+  {{streamlen}}
+>>
+stream
+BT
+/F1 30 Tf
+10 160 Td
+(T) Tj
+/F1 10 Tf
+20 15 Td
+(his is a test with a simulated drop) Tj
+0 -10 Td
+(cap. A drop cap is a large initial) Tj
+0 -10 Td
+(letter that drops below the first line) Tj
+-20 -10 Td
+(of a paragraph. The drop cap should be) Tj
+0 -10 Td
+(considered part of the first line.) Tj
+ET
+BT
+/F1 30 Tf
+10 100 Td
+(T) Tj
+/F1 10 Tf
+20 15 Td
+(his is another      test with a simulated) Tj
+0 -10 Td
+(drop cap.       We     use      large) Tj
+0 -10 Td
+(amounts of        whitespace to defeat) Tj
+-20 -10 Td
+(the         PDFium        text-run logic.) Tj
+ET
+endstream
+endobj
+{{xref}}
+{{trailer}}
+{{startxref}}
+%%EOF

chrome/test/data/pdf/accessibility/font-change-expected.txt

+This is some simple text with a font
+change in the middle of the line.
+¶
+In this example the font size is slightly
+different from the surrounding text.
+¶
+Similarly, attributes such as bold and / or
+italics should not cause line breaks.

chrome/test/data/pdf/accessibility/font-change.in

+{{header}}
+{{object 1 0}} <<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+{{object 2 0}} <<
+  /Type /Pages
+  /MediaBox [0 0 200 200]
+  /Count 1
+  /Kids [3 0 R]
+>>
+endobj
+{{object 3 0}} <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources <<
+    /Font <<
+      /F1 4 0 R
+      /F2 5 0 R
+      /F3 6 0 R
+      /F4 7 0 R
+    >>
+  >>
+  /Contents 8 0 R
+>>
+endobj
+{{object 4 0}} <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Helvetica
+>>
+endobj
+{{object 5 0}} <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Courier
+>>
+endobj
+{{object 6 0}} <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Helvetica-Bold
+>>
+endobj
+{{object 7 0}} <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Helvetica-Oblique
+>>
+endobj
+{{object 8 0}} <<
+  {{streamlen}}
+>>
+stream
+BT
+/F1 10 Tf
+10 180 Td
+(This is some ) Tj
+/F2 10 Tf
+(simple) Tj
+/F1 10 Tf
+( text with a font) Tj
+0 -10 Td
+(change in the middle of the line.) Tj
+ET
+BT
+/F1 10 Tf
+10 150 Td
+(In this ) Tj
+/F2 12 Tf
+(example) Tj
+/F1 10 Tf
+( the font size is slightly) Tj
+0 -10 Td
+(different from the surrounding text.) Tj
+ET
+BT
+/F1 10 Tf
+10 120 Td
+(Similarly, attributes such as ) Tj
+/F3 10 Tf
+(bold) Tj
+/F1 10 Tf
+( and / or) Tj
+0 -10 Td
+/F4 10 Tf
+(italics) Tj
+/F1 10 Tf
+( should not cause line breaks.) Tj
+ET
+endstream
+endobj
+{{xref}}
+{{trailer}}
+{{startxref}}
+%%EOF

chrome/test/data/pdf/accessibility/next-on-line-expected.txt

+This is a test with multiple
+lines of text.
+¶
+Leaving a large space between words
+above defeats the a11y text-run logic and
+ensures we have multiple runs to test the
+next-on-line / previous-on-line attributes.

chrome/test/data/pdf/accessibility/next-on-line.in

@@ -35,9 +35,9 @@ stream
 BT
 /F1 10 Tf
 10 180 Td
-(<<     This     is     a     test     with     >>) Tj
+(This     is     a     test     with     multiple) Tj
 0 -10 Td
-(<<     multiple     lines     of     text.     >>) Tj
+(lines     of     text.) Tj
 0 -20 Td
 (Leaving a large space between words) Tj
 0 -10 Td

chrome/test/data/pdf/accessibility/superscript-subscript-expected.txt

+This is some textabc with a
+simulated superscript.
+¶
+This is some textabc with a
+simulated subscript.
+¶
+This is some textabcdef with both
+superscript and subscript.

chrome/test/data/pdf/accessibility/superscript-subscript.in

+{{header}}
+{{object 1 0}} <<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+{{object 2 0}} <<
+  /Type /Pages
+  /MediaBox [0 0 200 200]
+  /Count 1
+  /Kids [3 0 R]
+>>
+endobj
+{{object 3 0}} <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources <<
+    /Font <<
+      /F1 4 0 R
+    >>
+  >>
+  /Contents 5 0 R
+>>
+endobj
+{{object 4 0}} <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Courier
+>>
+endobj
+{{object 5 0}} <<
+  {{streamlen}}
+>>
+stream
+% Using Courier because the width of each string is predictable (at 10pt, the
+% width of each glyph is 6pt).
+BT
+/F1 10 Tf
+10 180 Td
+(This is some text) Tj
+/F1 5 Tf
+102 4 Td
+(abc) Tj
+9 -4 Td
+/F1 10 Tf
+( with a) Tj
+-111 -10 Td
+(simulated superscript.) Tj
+ET
+BT
+/F1 10 Tf
+10 150 Td
+(This is some text) Tj
+/F1 5 Tf
+102 -1 Td
+(abc) Tj
+9 1 Td
+/F1 10 Tf
+( with a) Tj
+-111 -10 Td
+(simulated subscript.) Tj
+ET
+BT
+/F1 10 Tf
+10 120 Td
+(This is some text) Tj
+/F1 5 Tf
+102 -1 Td
+(abc) Tj
+0 5 Td
+(def) Tj
+9 -4 Td
+/F1 10 Tf
+( with both) Tj
+-111 -10 Td
+(superscript and subscript.) Tj
+ET
+endstream
+endobj
+{{xref}}
+{{trailer}}
+{{startxref}}
+%%EOF

components/pdf/renderer/pdf_accessibility_tree.cc

@@ -29,18 +29,14 @@ namespace {
 // if the median font size is not at least this many points.
 const double kMinimumFontSize = 5;
 
-// Don't try to apply line break thresholds to automatically identify
-// line breaks if the median line break is not at least this many points.
+// Don't try to apply paragraph break thresholds to automatically identify
+// paragraph breaks if the median line break is not at least this many points.
 const double kMinimumLineSpacing = 5;
 
 // Ratio between the font size of one text run and the median on the page
 // for that text run to be considered to be a heading instead of normal text.
 const double kHeadingFontSizeRatio = 1.2;
 
-// Ratio between the delta-y between two text runs and the median on the page
-// for it to be considered a line break.
-const double kLineSpacingRatio = 0.8;
-
 // Ratio between the line spacing between two lines and the median on the
 // page for that line spacing to be considered a paragraph break.
 const double kParagraphLineSpacingRatio = 1.2;
@@ -49,6 +45,100 @@ gfx::RectF ToGfxRectF(const PP_FloatRect& r) {
   return gfx::RectF(r.point.x, r.point.y, r.size.width, r.size.height);
 }
 
+// This class is used as part of our heuristic to determine which text runs live
+// on the same "line".  As we process runs, we keep a weighted average of the
+// top and bottom coordinates of the line, and if a new run falls within that
+// range (within a threshold) it is considered part of the line.
+class LineHelper {
+ public:
+  explicit LineHelper(
+      const std::vector<PP_PrivateAccessibilityTextRunInfo>& text_runs)
+      : text_runs_(text_runs) {
+    StartNewLine(0);
+  }
+
+  void StartNewLine(size_t current_index) {
+    DCHECK(current_index == 0 || current_index < text_runs_.size());
+    start_index_ = current_index;
+    accumulated_weight_top_ = 0.0f;
+    accumulated_weight_bottom_ = 0.0f;
+    accumulated_width_ = 0.0f;
+  }
+
+  void ProcessNextRun(size_t run_index) {
+    DCHECK_LT(run_index, text_runs_.size());
+    RemoveOldRunsUpTo(run_index);
+    AddRun(text_runs_[run_index].bounds);
+  }
+
+  bool IsRunOnSameLine(size_t run_index) const {
+    DCHECK_LT(run_index, text_runs_.size());
+
+    // Calculate new top/bottom bounds for our line.
+    if (accumulated_width_ == 0.0f)
+      return false;
+
+    float line_top = accumulated_weight_top_ / accumulated_width_;
+    float line_bottom = accumulated_weight_bottom_ / accumulated_width_;
+
+    // Look at the next run, and determine how much it overlaps the line.
+    const auto& run_bounds = text_runs_[run_index].bounds;
+    if (run_bounds.size.height == 0.0f)
+      return false;
+
+    float clamped_top = std::max(line_top, run_bounds.point.y);
+    float clamped_bottom =
+        std::min(line_bottom, run_bounds.point.y + run_bounds.size.height);
+    if (clamped_bottom < clamped_top)
+      return false;
+
+    float coverage = (clamped_bottom - clamped_top) / (run_bounds.size.height);
+
+    // See if it falls within the line (within our threshold).
+    constexpr float kLineCoverageThreshold = 0.25f;
+    return coverage > kLineCoverageThreshold;
+  }
+
+ private:
+  void AddRun(const PP_FloatRect& run_bounds) {
+    float run_width = fabs(run_bounds.size.width);
+    accumulated_width_ += run_width;
+    accumulated_weight_top_ += run_bounds.point.y * run_width;
+    accumulated_weight_bottom_ +=
+        (run_bounds.point.y + run_bounds.size.height) * run_width;
+  }
+
+  void RemoveRun(const PP_FloatRect& run_bounds) {
+    float run_width = fabs(run_bounds.size.width);
+    accumulated_width_ -= run_width;
+    accumulated_weight_top_ -= run_bounds.point.y * run_width;
+    accumulated_weight_bottom_ -=
+        (run_bounds.point.y + run_bounds.size.height) * run_width;
+  }
+
+  void RemoveOldRunsUpTo(size_t stop_index) {
+    // Remove older runs from the weighted average if we've exceeded the
+    // threshold distance from them. We remove them to prevent e.g. drop-caps
+    // from unduly influencing future lines.
+    constexpr float kBoxRemoveWidthThreshold = 3.0f;
+    while (start_index_ < stop_index &&
+           accumulated_width_ > text_runs_[start_index_].bounds.size.width *
+                                    kBoxRemoveWidthThreshold) {
+      const auto& old_bounds = text_runs_[start_index_].bounds;
+      RemoveRun(old_bounds);
+      start_index_++;
+    }
+  }
+
+  const std::vector<PP_PrivateAccessibilityTextRunInfo>& text_runs_;
+  size_t start_index_;
+  float accumulated_weight_top_;
+  float accumulated_weight_bottom_;
+  float accumulated_width_;
+
+  DISALLOW_COPY_AND_ASSIGN(LineHelper);
+};
+
 template <typename T>
 bool CompareTextRuns(const T& a, const T& b) {
   return a.text_run_index < b.text_run_index;
@@ -202,16 +292,16 @@ void PdfAccessibilityTree::AddPageContent(
   DCHECK(page_node);
   double heading_font_size_threshold = 0;
   double paragraph_spacing_threshold = 0;
-  double line_spacing_threshold = 0;
   ComputeParagraphAndHeadingThresholds(text_runs, &heading_font_size_threshold,
-                                       &paragraph_spacing_threshold,
-                                       &line_spacing_threshold);
+                                       &paragraph_spacing_threshold);
 
   ui::AXNodeData* para_node = nullptr;
   ui::AXNodeData* static_text_node = nullptr;
   ui::AXNodeData* previous_on_line_node = nullptr;
   std::string static_text;
   uint32_t char_index = 0;
+  LineHelper line_helper(text_runs);
+
   for (size_t text_run_index = 0; text_run_index < text_runs.size();
        ++text_run_index) {
     const PP_PrivateAccessibilityTextRunInfo& text_run =
@@ -252,7 +342,6 @@ void PdfAccessibilityTree::AddPageContent(
           ax::mojom::IntAttribute::kPreviousOnLineId,
           previous_on_line_node->id);
     }
-    previous_on_line_node = inline_text_box_node;
 
     if (text_run_index == text_runs.size() - 1) {
       static_text_node->AddStringAttribute(ax::mojom::StringAttribute::kName,
@@ -260,24 +349,34 @@ void PdfAccessibilityTree::AddPageContent(
       break;
     }
 
-    // End a paragraph if either of the conditions are met:
-    // 1. Current and next text run have different font sizes.
-    // 2. The line spacing between current and next text run is more than the
-    // calculated threshold value.
-    double line_spacing =
-        text_runs[text_run_index + 1].bounds.point.y - text_run.bounds.point.y;
-    if (text_run.font_size != text_runs[text_run_index + 1].font_size ||
-        (paragraph_spacing_threshold > 0 &&
-         line_spacing > paragraph_spacing_threshold)) {
-      static_text_node->AddStringAttribute(ax::mojom::StringAttribute::kName,
-                                           static_text);
-      para_node = nullptr;
-      static_text_node = nullptr;
-      static_text.clear();
-      previous_on_line_node = nullptr;
-    } else if (line_spacing_threshold > 0 &&
-               line_spacing > line_spacing_threshold) {
+    if (!previous_on_line_node)
+      line_helper.StartNewLine(text_run_index);
+    line_helper.ProcessNextRun(text_run_index);
+
+    if (line_helper.IsRunOnSameLine(text_run_index + 1)) {
+      // The next run is on the same line.
+      previous_on_line_node = inline_text_box_node;
+    } else {
+      // The next run is on a new line.
       previous_on_line_node = nullptr;
+
+      // Check to see if its also a new paragraph, i.e., if the distance between
+      // lines is greater than the threshold.  If there's no threshold, that
+      // means there weren't enough lines to compute an accurate median, so
+      // we compare against the line size instead.
+      double line_spacing = fabs(text_runs[text_run_index + 1].bounds.point.y -
+                                 text_run.bounds.point.y);
+      if ((paragraph_spacing_threshold > 0 &&
+           line_spacing > paragraph_spacing_threshold) ||
+          (paragraph_spacing_threshold == 0 &&
+           line_spacing >
+               kParagraphLineSpacingRatio * text_run.bounds.size.height)) {
+        static_text_node->AddStringAttribute(ax::mojom::StringAttribute::kName,
+                                             static_text);
+        para_node = nullptr;
+        static_text_node = nullptr;
+        static_text.clear();
+      }
     }
   }
 }
@@ -354,8 +453,7 @@ void PdfAccessibilityTree::FindNodeOffset(uint32_t page_index,
 void PdfAccessibilityTree::ComputeParagraphAndHeadingThresholds(
     const std::vector<PP_PrivateAccessibilityTextRunInfo>& text_runs,
     double* out_heading_font_size_threshold,
-    double* out_paragraph_spacing_threshold,
-    double* out_line_spacing_threshold) {
+    double* out_paragraph_spacing_threshold) {
   // Scan over the font sizes and line spacing within this page and
   // set heuristic thresholds so that text larger than the median font
   // size can be marked as a heading, and spacing larger than the median
@@ -379,13 +477,10 @@ void PdfAccessibilityTree::ComputeParagraphAndHeadingThresholds(
           median_font_size * kHeadingFontSizeRatio;
     }
   }
-  if (line_spacings.size() > 0) {
+  if (line_spacings.size() > 4) {
     std::sort(line_spacings.begin(), line_spacings.end());
     double median_line_spacing = line_spacings[line_spacings.size() / 2];
     if (median_line_spacing > kMinimumLineSpacing) {
-      *out_line_spacing_threshold = median_line_spacing * kLineSpacingRatio;
-    }
-    if (line_spacings.size() > 4) {
       *out_paragraph_spacing_threshold =
           median_line_spacing * kParagraphLineSpacingRatio;
     }

components/pdf/renderer/pdf_accessibility_tree.h

@@ -99,8 +99,7 @@ class PdfAccessibilityTree : public content::PluginAXTreeSource {
   void ComputeParagraphAndHeadingThresholds(
       const std::vector<PP_PrivateAccessibilityTextRunInfo>& text_runs,
       double* out_heading_font_size_threshold,
-      double* out_paragraph_spacing_threshold,
-      double* out_line_spacing_threshold);
+      double* out_paragraph_spacing_threshold);
   std::string GetTextRunCharsAsUTF8(
       const PP_PrivateAccessibilityTextRunInfo& text_run,
       const std::vector<PP_PrivateAccessibilityCharInfo>& chars,