Remove duplicate licenses from about:credits

Sometimes the same library is checked into multiple directories.
Using simple de-duplication logic removes 42 duplicates from
about:credits.

Bug: 1093004
Change-Id: I054b5aa6f7f273309a70c6b2909e554835f2be10
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2239327Reviewed-by: Nico Weber <thakis@chromium.org>
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#777007}

tools/licenses.py

@@ -664,6 +664,7 @@ def GenerateCredits(
   entries.append(
       MetadataToTemplateEntry(chromium_license_metadata, entry_template))
 
+  entries_by_name = {}
   for path in third_party_dirs:
     try:
       metadata = ParseDir(path, _REPOSITORY_ROOT)
@@ -680,7 +681,15 @@ def GenerateCredits(
       # updated to provide --gn-target to this script.
       if path in KNOWN_NON_IOS_LIBRARIES:
         continue
-    entries.append(MetadataToTemplateEntry(metadata, entry_template))
+
+    new_entry = MetadataToTemplateEntry(metadata, entry_template)
+    # Skip entries that we've already seen (it exists in multiple directories).
+    prev_entry = entries_by_name.setdefault(new_entry['name'], new_entry)
+    if prev_entry is not new_entry and (
+        prev_entry['content'] == new_entry['content']):
+      continue
+
+    entries.append(new_entry)
 
   entries.sort(key=lambda entry: (entry['name'].lower(), entry['content']))
   for entry_id, entry in enumerate(entries):