gpu scheduler: Optimize priorities for circular stream waits.

This changes streams to only propagate inherited prioirities to
already existing wait fences.

New wait fences initially inherit the default priority of the waiter.
If the waiter gets a priority bump, that priority is propagated to
existing wait fences at that time.

BUG=786820

Cq-Include-Trybots: master.tryserver.chromium.android:android_optional_gpu_tests_rel;master.tryserver.chromium.linux:linux_optional_gpu_tests_rel;master.tryserver.chromium.mac:mac_optional_gpu_tests_rel;master.tryserver.chromium.win:win_optional_gpu_tests_rel
Change-Id: Iab04f21985e69aba7f3b90b7ffe2545d3c8510e6
Reviewed-on: https://chromium-review.googlesource.com/778422
Commit-Queue: Victor Miura <vmiura@chromium.org>
Reviewed-by: Antoine Labour <piman@chromium.org>
Reviewed-by: Sunny Sachanandani <sunnyps@chromium.org>
Cr-Commit-Position: refs/heads/master@{#518453}

gpu/command_buffer/service/scheduler.cc

@@ -47,16 +47,16 @@ Scheduler::Sequence::Task::~Task() = default;
 Scheduler::Sequence::Task& Scheduler::Sequence::Task::operator=(Task&& other) =
     default;
 
-Scheduler::Sequence::Fence::Fence(const SyncToken& sync_token,
+Scheduler::Sequence::WaitFence::WaitFence(const SyncToken& sync_token,
                                           uint32_t order_num,
                                           SequenceId release_sequence_id)
     : sync_token(sync_token),
       order_num(order_num),
       release_sequence_id(release_sequence_id) {}
-Scheduler::Sequence::Fence::Fence(Fence&& other) = default;
-Scheduler::Sequence::Fence::~Fence() = default;
-Scheduler::Sequence::Fence& Scheduler::Sequence::Fence::operator=(
-    Fence&& other) = default;
+Scheduler::Sequence::WaitFence::WaitFence(WaitFence&& other) = default;
+Scheduler::Sequence::WaitFence::~WaitFence() = default;
+Scheduler::Sequence::WaitFence& Scheduler::Sequence::WaitFence::operator=(
+    WaitFence&& other) = default;
 
 Scheduler::Sequence::Sequence(Scheduler* scheduler,
                               SequenceId sequence_id,
@@ -69,13 +69,11 @@ Scheduler::Sequence::Sequence(Scheduler* scheduler,
       order_data_(std::move(order_data)) {}
 
 Scheduler::Sequence::~Sequence() {
-  for (auto& fence : wait_fences_) {
+  for (auto& kv : wait_fences_) {
     Sequence* release_sequence =
-        scheduler_->GetSequence(fence.release_sequence_id);
-    if (!release_sequence)
-      continue;
-
-    release_sequence->RemoveWaitingPriority(current_priority());
+        scheduler_->GetSequence(kv.first.release_sequence_id);
+    if (release_sequence)
+      release_sequence->RemoveWaitingPriority(kv.second);
   }
 
   order_data_->Destroy();
@@ -99,19 +97,7 @@ void Scheduler::Sequence::UpdateSchedulingPriority() {
                  "sequence_id", sequence_id_.GetUnsafeValue(), "new_priority",
                  SchedulingPriorityToString(priority));
 
-    SchedulingPriority old_priority = current_priority_;
     current_priority_ = priority;
-
-    // Update priorities on sequences we're waiting on.
-    for (auto& wait_fence : wait_fences_) {
-      Sequence* release_sequence =
-          scheduler_->GetSequence(wait_fence.release_sequence_id);
-      if (release_sequence) {
-        release_sequence->ChangeWaitingPriority(old_priority,
-                                                current_priority_);
-      }
-    }
-
     scheduler_->TryScheduleSequence(this);
   }
 }
@@ -124,7 +110,7 @@ bool Scheduler::Sequence::NeedsRescheduling() const {
 bool Scheduler::Sequence::IsRunnable() const {
   return enabled_ && !tasks_.empty() &&
          (wait_fences_.empty() ||
-          wait_fences_.front().order_num > tasks_.front().order_num);
+          wait_fences_.begin()->first.order_num > tasks_.front().order_num);
 }
 
 bool Scheduler::Sequence::ShouldYieldTo(const Sequence* other) const {
@@ -202,30 +188,51 @@ void Scheduler::Sequence::AddWaitFence(const SyncToken& sync_token,
                                        uint32_t order_num,
                                        SequenceId release_sequence_id,
                                        Sequence* release_sequence) {
+  auto it =
+      wait_fences_.find(WaitFence{sync_token, order_num, release_sequence_id});
+  if (it != wait_fences_.end())
+    return;
+
   DCHECK(release_sequence);
-  release_sequence->AddWaitingPriority(current_priority());
+  release_sequence->AddWaitingPriority(default_priority_);
 
-  wait_fences_.emplace_back(sync_token, order_num, release_sequence_id);
+  wait_fences_.emplace(
+      std::make_pair(WaitFence(sync_token, order_num, release_sequence_id),
+                     default_priority_));
 }
 
 void Scheduler::Sequence::RemoveWaitFence(const SyncToken& sync_token,
                                           uint32_t order_num,
                                           SequenceId release_sequence_id) {
-  auto it = std::find(wait_fences_.begin(), wait_fences_.end(),
-                      Fence{sync_token, order_num, release_sequence_id});
-  DCHECK(it != wait_fences_.end());
-
+  auto it =
+      wait_fences_.find(WaitFence{sync_token, order_num, release_sequence_id});
   if (it != wait_fences_.end()) {
+    SchedulingPriority wait_priority = it->second;
     wait_fences_.erase(it);
 
     Sequence* release_sequence = scheduler_->GetSequence(release_sequence_id);
     if (release_sequence)
-      release_sequence->RemoveWaitingPriority(current_priority());
+      release_sequence->RemoveWaitingPriority(wait_priority);
 
     scheduler_->TryScheduleSequence(this);
   }
 }
 
+void Scheduler::Sequence::PropagatePriority(SchedulingPriority priority) {
+  for (auto& kv : wait_fences_) {
+    if (kv.second > priority) {
+      SchedulingPriority old_priority = kv.second;
+      kv.second = priority;
+
+      Sequence* release_sequence =
+          scheduler_->GetSequence(kv.first.release_sequence_id);
+      if (release_sequence) {
+        release_sequence->ChangeWaitingPriority(old_priority, priority);
+      }
+    }
+  }
+}
+
 void Scheduler::Sequence::AddWaitingPriority(SchedulingPriority priority) {
   TRACE_EVENT2("gpu", "Scheduler::Sequence::RemoveWaitingPriority",
                "sequence_id", sequence_id_.GetUnsafeValue(), "new_priority",
@@ -236,6 +243,8 @@ void Scheduler::Sequence::AddWaitingPriority(SchedulingPriority priority) {
   if (priority < current_priority_) {
     UpdateSchedulingPriority();
   }
+
+  PropagatePriority(priority);
 }
 
 void Scheduler::Sequence::RemoveWaitingPriority(SchedulingPriority priority) {
@@ -263,11 +272,14 @@ void Scheduler::Sequence::ChangeWaitingPriority(
        waiting_priority_counts_[static_cast<int>(old_priority)] == 0)) {
     UpdateSchedulingPriority();
   }
+
+  PropagatePriority(new_priority);
 }
 
 void Scheduler::Sequence::AddClientWait(CommandBufferId command_buffer_id) {
   client_waits_.insert(command_buffer_id);
   UpdateSchedulingPriority();
+  PropagatePriority(SchedulingPriority::kHigh);
 }
 
 void Scheduler::Sequence::RemoveClientWait(CommandBufferId command_buffer_id) {

gpu/command_buffer/service/scheduler.h

@@ -192,21 +192,26 @@ class GPU_EXPORT Scheduler {
    private:
     enum RunningState { IDLE, SCHEDULED, RUNNING };
 
-    struct Fence {
-      Fence(Fence&& other);
-      Fence(const SyncToken& sync_token,
+    struct WaitFence {
+      WaitFence(WaitFence&& other);
+      WaitFence(const SyncToken& sync_token,
                 uint32_t order_num,
                 SequenceId release_sequence_id);
-      ~Fence();
-      Fence& operator=(Fence&& other);
+      ~WaitFence();
+      WaitFence& operator=(WaitFence&& other);
 
       SyncToken sync_token;
       uint32_t order_num;
       SequenceId release_sequence_id;
 
-      bool operator==(const Fence& other) const {
-        return std::tie(sync_token, order_num, release_sequence_id) ==
-               std::tie(other.sync_token, other.order_num, release_sequence_id);
+      bool operator==(const WaitFence& other) const {
+        return std::tie(order_num, release_sequence_id, sync_token) ==
+               std::tie(other.order_num, release_sequence_id, other.sync_token);
+      }
+
+      bool operator<(const WaitFence& other) const {
+        return std::tie(order_num, release_sequence_id, sync_token) <
+               std::tie(other.order_num, release_sequence_id, other.sync_token);
       }
     };
 
@@ -220,6 +225,27 @@ class GPU_EXPORT Scheduler {
       uint32_t order_num;
     };
 
+    // Description of Stream priority propagation: Each Stream has an initial
+    // priority ('default_priority_').  When a Stream has other Streams waiting
+    // on it via a 'WaitFence', it computes it's own priority based on those
+    // fences, by keeping count of the priority of each incoming WaitFence's
+    // priority in 'waiting_priority_counts_'.
+    //
+    // 'wait_fences_' maps each 'WaitFence' to it's current priority.  Initially
+    // WaitFences take the priority of the waiting Stream, and propagate their
+    // priority to the releasing Stream via AddWaitingPriority().
+    //
+    // A higher priority waiting stream or ClientWait, can recursively pass on
+    // it's priority to existing 'ClientWaits' via PropagatePriority(), which
+    // updates the releasing stream via ChangeWaitingPriority().
+    //
+    // When a 'WaitFence' is removed either by the SyncToken being released,
+    // or when the waiting Stream is Destroyed, it removes it's priority from
+    // the releasing stream via RemoveWaitingPriority().
+
+    // Propagate a priority to all wait fences.
+    void PropagatePriority(SchedulingPriority priority);
+
     // Add a waiting priority.
     void AddWaitingPriority(SchedulingPriority priority);
 
@@ -256,11 +282,11 @@ class GPU_EXPORT Scheduler {
     // continued, it is inserted at the front with the same order number.
     base::circular_deque<Task> tasks_;
 
-    // List of fences that this sequence is waiting on. Fences are inserted in
+    // Map of fences that this sequence is waiting on. Fences are ordered in
     // increasing order number but may be removed out of order. Tasks are
     // blocked if there's a wait fence with order number less than or equal to
     // the task's order number.
-    std::vector<Fence> wait_fences_;
+    base::flat_map<WaitFence, SchedulingPriority> wait_fences_;
 
     // Counts of pending releases bucketed by scheduling priority.
     int waiting_priority_counts_[static_cast<int>(SchedulingPriority::kLast) +

gpu/command_buffer/service/scheduler_unittest.cc

@@ -555,17 +555,17 @@ TEST_F(SchedulerTest, StreamPriorities) {
   EXPECT_EQ(SchedulingPriority::kHigh, seq3->current_priority());
 
   // Release priority propagate.
-  seq3->RemoveWaitFence(sync_token2, 2, seq_id2);
-  EXPECT_EQ(SchedulingPriority::kNormal, seq1->current_priority());
-  EXPECT_EQ(SchedulingPriority::kNormal, seq2->current_priority());
+  seq2->RemoveWaitFence(sync_token1, 1, seq_id1);
+  EXPECT_EQ(SchedulingPriority::kLow, seq1->current_priority());
+  EXPECT_EQ(SchedulingPriority::kHigh, seq2->current_priority());
   EXPECT_EQ(SchedulingPriority::kHigh, seq3->current_priority());
 
   seq2->RemoveWaitFence(sync_token1, 1, seq_id1);
-  EXPECT_EQ(SchedulingPriority::kNormal, seq1->current_priority());
-  EXPECT_EQ(SchedulingPriority::kNormal, seq2->current_priority());
+  EXPECT_EQ(SchedulingPriority::kLow, seq1->current_priority());
+  EXPECT_EQ(SchedulingPriority::kHigh, seq2->current_priority());
   EXPECT_EQ(SchedulingPriority::kHigh, seq3->current_priority());
 
-  seq2->RemoveWaitFence(sync_token1, 1, seq_id1);
+  seq3->RemoveWaitFence(sync_token2, 2, seq_id2);
   EXPECT_EQ(SchedulingPriority::kLow, seq1->current_priority());
   EXPECT_EQ(SchedulingPriority::kNormal, seq2->current_priority());
   EXPECT_EQ(SchedulingPriority::kHigh, seq3->current_priority());
@@ -611,7 +611,7 @@ TEST_F(SchedulerTest, StreamDestroyRemovesPriorities) {
     scheduler()->DestroySequence(seq_id3);
   }
 
-  EXPECT_EQ(SchedulingPriority::kNormal, seq1->current_priority());
+  EXPECT_EQ(SchedulingPriority::kHigh, seq1->current_priority());
   EXPECT_EQ(SchedulingPriority::kNormal, seq2->current_priority());
 
   {
@@ -654,25 +654,25 @@ TEST_F(SchedulerTest, StreamPriorityChangeWhileReleasing) {
   EXPECT_EQ(SchedulingPriority::kNormal, seq2->current_priority());
   EXPECT_EQ(SchedulingPriority::kHigh, seq3->current_priority());
 
-  // Begin releasing fences.
+  // All matching wait fences are removed together.
   seq2->RemoveWaitFence(sync_token1, 1, seq_id1);
-  EXPECT_EQ(SchedulingPriority::kNormal, seq1->current_priority());
+  EXPECT_EQ(SchedulingPriority::kLow, seq1->current_priority());
   EXPECT_EQ(SchedulingPriority::kNormal, seq2->current_priority());
   EXPECT_EQ(SchedulingPriority::kHigh, seq3->current_priority());
 
   // Add wait fence with higher priority.  This replicates a possible race.
-  seq3->AddWaitFence(sync_token2, 1, seq_id2, seq2);
-  EXPECT_EQ(SchedulingPriority::kHigh, seq1->current_priority());
+  seq3->AddWaitFence(sync_token2, 2, seq_id2, seq2);
+  EXPECT_EQ(SchedulingPriority::kLow, seq1->current_priority());
   EXPECT_EQ(SchedulingPriority::kHigh, seq2->current_priority());
   EXPECT_EQ(SchedulingPriority::kHigh, seq3->current_priority());
 
-  // Finish removing fences.
+  // This should be a No-op.
   seq2->RemoveWaitFence(sync_token1, 1, seq_id1);
   EXPECT_EQ(SchedulingPriority::kLow, seq1->current_priority());
   EXPECT_EQ(SchedulingPriority::kHigh, seq2->current_priority());
   EXPECT_EQ(SchedulingPriority::kHigh, seq3->current_priority());
 
-  seq3->RemoveWaitFence(sync_token2, 1, seq_id2);
+  seq3->RemoveWaitFence(sync_token2, 2, seq_id2);
   EXPECT_EQ(SchedulingPriority::kLow, seq1->current_priority());
   EXPECT_EQ(SchedulingPriority::kNormal, seq2->current_priority());
   EXPECT_EQ(SchedulingPriority::kHigh, seq3->current_priority());
@@ -727,17 +727,17 @@ TEST_F(SchedulerTest, CircularPriorities) {
   seq2->AddWaitFence(sync_token_seq3_1, 4, seq_id3, seq3);
   EXPECT_EQ(SchedulingPriority::kHigh, seq1->current_priority());
   EXPECT_EQ(SchedulingPriority::kHigh, seq2->current_priority());
-  EXPECT_EQ(SchedulingPriority::kHigh, seq3->current_priority());
+  EXPECT_EQ(SchedulingPriority::kNormal, seq3->current_priority());
 
   seq3->RemoveWaitFence(sync_token_seq2_1, 1, seq_id2);
   EXPECT_EQ(SchedulingPriority::kHigh, seq1->current_priority());
   EXPECT_EQ(SchedulingPriority::kHigh, seq2->current_priority());
-  EXPECT_EQ(SchedulingPriority::kHigh, seq3->current_priority());
+  EXPECT_EQ(SchedulingPriority::kNormal, seq3->current_priority());
 
   seq1->RemoveWaitFence(sync_token_seq2_2, 2, seq_id2);
   EXPECT_EQ(SchedulingPriority::kHigh, seq1->current_priority());
-  EXPECT_EQ(SchedulingPriority::kHigh, seq2->current_priority());
-  EXPECT_EQ(SchedulingPriority::kHigh, seq3->current_priority());
+  EXPECT_EQ(SchedulingPriority::kNormal, seq2->current_priority());
+  EXPECT_EQ(SchedulingPriority::kNormal, seq3->current_priority());
 
   seq3->RemoveWaitFence(sync_token_seq2_3, 3, seq_id2);
   EXPECT_EQ(SchedulingPriority::kHigh, seq1->current_priority());