Commit 355fefda authored by Wenbin Zhang's avatar Wenbin Zhang Committed by Commit Bot

[benchmarking] update sharding algorithm

The current sharding algorithm calculates the expected shard time and stick to it. When a test has a long runtime, it could cause its sibling shard to be empty.

Updating the algorithm to always update the expect time for the remaining after each shard. Also will ensure each shard should have at least one test.

Bug: chromium:1130157
Change-Id: I3c335cfc58e35b48e5fd374c9d7546184cd68c56
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2481006Reviewed-by: default avatarJohn Chen <johnchen@chromium.org>
Commit-Queue: Wenbin Zhang <wenbinzhang@google.com>
Cr-Commit-Position: refs/heads/master@{#827022}
parent e66d5383
...@@ -44,72 +44,73 @@ def generate_sharding_map(benchmarks_to_shard, timing_data, num_shards, debug): ...@@ -44,72 +44,73 @@ def generate_sharding_map(benchmarks_to_shard, timing_data, num_shards, debug):
story_timing_list = _gather_timing_data( story_timing_list = _gather_timing_data(
benchmarks_to_shard, timing_data, True) benchmarks_to_shard, timing_data, True)
all_stories = {} stories_by_benchmark = {}
for b in benchmarks_to_shard: for b in benchmarks_to_shard:
all_stories[b.name] = b.stories stories_by_benchmark[b.name] = b.stories
total_time = sum(p[1] for p in story_timing_list)
expected_time_per_shard = total_time/num_shards
total_time_scheduled = 0
sharding_map = collections.OrderedDict() sharding_map = collections.OrderedDict()
debug_map = collections.OrderedDict() num_stories = len(story_timing_list)
min_shard_time = sys.maxint min_shard_time = sys.maxint
min_shard_index = None min_shard_index = None
max_shard_time = 0 max_shard_time = 0
max_shard_index = None max_shard_index = None
predicted_shard_timings = [] predicted_shard_timings = []
debug_timing = collections.OrderedDict()
# The algorithm below removes all the stories from |story_timing_list| one by # The algorithm below removes all the stories from |story_timing_list| one by
# one and add them to the current shard until the shard's total time is # one and add them to the current shard until the shard's total time is
# approximately equals to |expected_time_per_shard|. After that point, # approximately equals to |expected_shard_time|. After that point,
# it moves to the next shard. # it moves to the next shard.
# For efficient removal of |story_timing_list|'s elements & to keep the # For efficient removal of |story_timing_list|'s elements & to keep the
# ordering of benchmark alphabetically sorted in the shards' assignment, we # ordering of benchmark alphabetically sorted in the shards' assignment, we
# reverse the |story_timing_list|. # reverse the |story_timing_list|.
total_time = sum(p[1] for p in story_timing_list)
expected_shard_time = total_time / num_shards
story_timing_list.reverse() story_timing_list.reverse()
num_stories = len(story_timing_list)
final_shard_index = num_shards - 1
for i in range(num_shards): for i in range(num_shards):
shard_name = 'shard #%i' % i shard_name = 'shard #%i' % i
sharding_map[str(i)] = {'benchmarks': collections.OrderedDict()} sharding_map[str(i)] = {'benchmarks': collections.OrderedDict()}
debug_map[shard_name] = collections.OrderedDict() debug_timing[shard_name] = collections.OrderedDict()
time_per_shard = 0 shard_time = 0
stories_in_shard = [] stories_in_shard = []
expected_total_time = expected_time_per_shard * (i + 1)
last_diff = abs(total_time_scheduled - expected_total_time) # Keep adding stories to the current shard if:
# Keep adding story to the current shard until either: # 1. Adding the next story does not makes the shard time further from
# * The absolute difference between the total time of shards so far and # the expected;
# expected total time is minimal. # Or
# * The shard is final shard, and there is no more stories to add. # 2. The current shard is the last shard.
#
# Note: we do check for the final shard in case due to rounding error,
# the last_diff can be minimal even if we don't add all the stories to the
# final shard.
while story_timing_list: while story_timing_list:
candidate_story, candidate_story_duration = story_timing_list[-1] # Add one story anyway to avoid empty shard
new_diff = abs(total_time_scheduled + candidate_story_duration - current_story, current_duration = story_timing_list[-1]
expected_total_time) story_timing_list.pop()
if new_diff <= last_diff or i == final_shard_index: shard_time += current_duration
story_timing_list.pop() stories_in_shard.append(current_story)
total_time_scheduled += candidate_story_duration debug_timing[shard_name][current_story] = current_duration
time_per_shard += candidate_story_duration _add_benchmarks_to_shard(sharding_map, i, stories_in_shard,
stories_in_shard.append(candidate_story) stories_by_benchmark, benchmark_name_to_config)
debug_map[shard_name][candidate_story] = candidate_story_duration
last_diff = abs(total_time_scheduled - expected_total_time) if not story_timing_list:
_add_benchmarks_to_shard(sharding_map, i, stories_in_shard, all_stories, # All stories sharded
benchmark_name_to_config)
else:
break break
debug_map[shard_name]['expected_total_time'] = time_per_shard
if time_per_shard > max_shard_time: _, next_duration = story_timing_list[-1]
max_shard_time = time_per_shard if (abs(shard_time + next_duration - expected_shard_time) >
abs(shard_time - expected_shard_time)) and i != num_shards - 1:
# it is not the last shard and we should not add the next story
break
if i != num_shards - 1:
total_time -= shard_time
expected_shard_time = total_time / (num_shards - i - 1)
if shard_time > max_shard_time:
max_shard_time = shard_time
max_shard_index = i max_shard_index = i
if time_per_shard < min_shard_time: if shard_time < min_shard_time:
min_shard_time = time_per_shard min_shard_time = shard_time
min_shard_index = i min_shard_index = i
predicted_shard_timings.append((shard_name, time_per_shard)) predicted_shard_timings.append((shard_name, shard_time))
debug_timing[shard_name]['expected_total_time'] = shard_time
sharding_map['extra_infos'] = collections.OrderedDict([ sharding_map['extra_infos'] = collections.OrderedDict([
('num_stories', num_stories), ('num_stories', num_stories),
...@@ -120,7 +121,7 @@ def generate_sharding_map(benchmarks_to_shard, timing_data, num_shards, debug): ...@@ -120,7 +121,7 @@ def generate_sharding_map(benchmarks_to_shard, timing_data, num_shards, debug):
]) ])
if debug: if debug:
sharding_map['extra_infos'].update(debug_map) sharding_map['extra_infos'].update(debug_timing)
else: else:
sharding_map['extra_infos'].update(predicted_shard_timings) sharding_map['extra_infos'].update(predicted_shard_timings)
return sharding_map return sharding_map
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment