Add a tool to upload translation screenshots to content addressed Cloud Storage

This tool scans for any screenshots (png only) associated with grd files and uploads them to Google Cloud Storage using upload_to_google_storage.py.

upload_to_google_storage.py is a tool that maintains content addressed files. It stores files with their sha1 hashes as filenames.

The storage bucket name used in the tool is tentative.

For details, see the design doc at http://go/chrome-translation-screenshots#heading=h.cyt0azox2105

Change-Id: Ide06aa679489a65887df9c58948c8adde12d3eba
Bug: 814901
Reviewed-on: https://chromium-review.googlesource.com/922686
Commit-Queue: Mustafa Emre Acer <meacer@chromium.org>
Reviewed-by: anthonyvd <anthonyvd@chromium.org>
Reviewed-by: Aaron Gable <agable@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Cr-Commit-Position: refs/heads/master@{#541898}

tools/gritsettings/translation_expectations.pyl

@@ -88,6 +88,8 @@
     "tools/grit/grit/testdata/substitute_tmpl.grd": "Test data",
     "tools/grit/grit/testdata/whitelist_resources.grd": "Test data",
     "tools/grit/grit/testdata/whitelist_strings.grd": "Test data",
+    "tools/translation/testdata/not_translated.grd": "Test data",
+    "tools/translation/testdata/test.grd": "Test data",
     "ui/strings/app_locale_settings.grd": "Not UI strings; localized separately",
   },
 }

tools/translation/OWNERS

+anthonyvd@chromium.org
+meacer@chromium.org

tools/translation/PRESUBMIT.py

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+def _CommonChecks(input_api, output_api, run_tests=False):
+  results = []
+
+  # Run Pylint over the files in the directory.
+  pylint_checks = input_api.canned_checks.GetPylint(input_api, output_api)
+  results.extend(input_api.RunTests(pylint_checks))
+
+  # Run unittests.
+  if run_tests:
+    results.extend(input_api.canned_checks.RunUnitTestsInDirectory(
+        input_api, output_api, '.', [ r'^.+_unittest\.py$']))
+
+  return results
+
+
+def CheckChangeOnUpload(input_api, output_api):
+  return _CommonChecks(input_api, output_api)
+
+
+def CheckChangeOnCommit(input_api, output_api):
+  return _CommonChecks(input_api, output_api, run_tests=True)

tools/translation/README.md

+# Translation Tools
+
+This directory contains tools for Chrome's UI translations.
+
+## upload_screenshots.py
+
+This tool uploads translation screenshots to a content-addressed Google Cloud
+Storage bucket.
+
+Translation screenshots are .png files provided by Chrome Developers to give
+translators more context about UI changes. Developers take screenshots of their
+UI changes, add them under a specific directory (derived from the path of the
+.grd or .grdp file that contains the UI string) and run this tool. The tool
+uploads the images, generates SHA1 fingerprints of the screenshots, and asks the
+developer if they want to add the fingerprints to the CL.
+
+Example: For a file at path/to/test.grd, the screenshot directory will be
+path/to/test_grd. In the upstream Chrome repository, this directory will only
+contain .png.sha1 files previously generated by this tool. In local working
+repositories, it may contain .png files generated by Chrome developers, such as
+path/to/test_grd/IDS_MESSAGE.png.
+
+For more information, see go/chrome-translation-screenshots (Google internal
+link).

tools/translation/helper/translation_helper.py

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Helpers for dealing with translation files."""
+
+import ast
+import os
+import re
+import xml.etree.cElementTree as ElementTree
+
+
+class GRDFile(object):
+  """Class representing a grd xml file.
+
+  Attributes:
+    path: the path to the grd file.
+    dir: the path to the the grd's parent directery.
+    name: the base name of the grd file.
+    grdp_paths: the list of grdp files included in the grd via <part>.
+    structure_paths: the paths of any <structure> elements in the grd file.
+    xtb_paths: the xtb paths where the grd's translations live.
+    lang_to_xtb_path: maps each language to the xtb path for that language.
+    appears_translatable: whether the contents of the grd indicate that it's
+        supposed to be translated.
+    expected_languages: the languages that this grd is expected to have
+        translations for, based on the translation expectations file.
+  """
+
+  def __init__(self, path):
+    self.path = path
+    self.dir, self.name = os.path.split(path)
+    dom, self.grdp_paths = _parse_grd_file(path)
+    self.structure_paths = [os.path.join(self.dir, s.get('file'))
+                            for s in dom.findall('.//structure')]
+    self.xtb_paths = [os.path.join(self.dir, f.get('path'))
+                      for f in dom.findall('.//file')]
+    self.lang_to_xtb_path = {}
+    self.appears_translatable = (len(self.xtb_paths) != 0 or
+                                 dom.find('.//message') is not None)
+    self.expected_languages = None
+
+  def _populate_lang_to_xtb_path(self, errors):
+    """Populates the lang_to_xtb_path attribute."""
+    grd_root = os.path.splitext(self.name)[0]
+    lang_pattern = re.compile(r'%s_([^_]+)\.xtb$' % re.escape(grd_root))
+    for xtb_path in self.xtb_paths:
+      xtb_basename = os.path.basename(xtb_path)
+      xtb_lang_match = re.match(lang_pattern, xtb_basename)
+      if not xtb_lang_match:
+        errors.append('%s: invalid xtb name: %s. xtb name must be %s_<lang>'
+                      '.xtb where <lang> is the language code.' %
+                      (self.name, xtb_basename, grd_root))
+        continue
+      xtb_lang = xtb_lang_match.group(1)
+      if xtb_lang in self.lang_to_xtb_path:
+        errors.append('%s: %s is listed twice' % (self.name, xtb_basename))
+        continue
+      self.lang_to_xtb_path[xtb_lang] = xtb_path
+
+    return errors
+
+
+def get_translatable_grds(repo_root, all_grd_paths,
+                          translation_expectations_path):
+  """Returns all the grds that should be translated as a list of GRDFiles.
+
+  This verifies that every grd file that appears translatable is listed in
+  the translation expectations, and that every grd in the translation
+  expectations actually exists.
+
+  Args:
+    repo_root: The path to the root of the repository.
+    all_grd_paths: All grd paths in the repository relative to repo_root.
+    translation_expectations_path: The path to the translation expectations
+        file, which specifies which grds to translate and into which languages.
+  """
+  grd_to_langs, untranslated_grds = _parse_translation_expectations(
+      translation_expectations_path)
+
+  # Check that every grd that appears translatable is listed in
+  # the translation expectations.
+  grds_with_expectations = set(grd_to_langs.keys()).union(untranslated_grds)
+  all_grds = {p: GRDFile(os.path.join(repo_root, p)) for p in all_grd_paths}
+  errors = []
+  for path, grd in all_grds.iteritems():
+    if grd.appears_translatable:
+      if path not in grds_with_expectations:
+        errors.append('%s appears to be translatable (because it contains '
+            '<file> or <message> elements), but is not listed in the '
+            'translation expectations.' % path)
+
+  # Check that every file in translation_expectations exists.
+  for path in grds_with_expectations:
+    if path not in all_grd_paths:
+      errors.append('%s is listed in the translation expectations, but this '
+          'grd file does not exist.' % path)
+
+  if errors:
+    raise Exception('%s needs to be updated. Please fix these issues:\n - %s' %
+                    (translation_expectations_path, '\n - '.join(errors)))
+
+  translatable_grds = []
+  for path, expected_languages_list in grd_to_langs.iteritems():
+    grd = all_grds[path]
+    grd.expected_languages = expected_languages_list
+    grd._populate_lang_to_xtb_path(errors)
+    translatable_grds.append(grd)
+
+    # Ensure each grd lists the expected languages.
+    expected_languages = set(expected_languages_list)
+    actual_languages = set(grd.lang_to_xtb_path.keys())
+    if expected_languages.difference(actual_languages):
+      errors.append('%s: missing translations for these languages: %s. Add '
+                    '<file> and <output> elements to the grd file, or update '
+                    'the translation expectations.' % (grd.name,
+                    sorted(expected_languages.difference(actual_languages))))
+    if actual_languages.difference(expected_languages):
+      errors.append('%s: references translations for unexpected languages: %s. '
+                    'Remove the offending <file> and <output> elements from the'
+                    ' grd file, or update the translation expectations.'
+                    % (grd.name,
+                    sorted(actual_languages.difference(expected_languages))))
+
+  if errors:
+    raise Exception('Please fix these issues:\n - %s' %
+                    ('\n - '.join(errors)))
+
+  return translatable_grds
+
+
+def _parse_grd_file(grd_path):
+  """Reads a grd(p) file and any subfiles included via <part file="..." />.
+
+  Args:
+    grd_path: The path of the .grd or .grdp file.
+  Returns:
+    A tuple (grd_dom, grdp_paths). dom is an ElementTree DOM for the grd file,
+    with the <part> elements inlined. grdp_paths is the list of grdp files that
+    were included via <part> elements.
+  """
+  grdp_paths = []
+  grd_dom = ElementTree.parse(grd_path)
+  # We modify grd in the loop, so listify this iterable to be safe.
+  part_nodes = list(grd_dom.findall('.//part'))
+  for part_node in part_nodes:
+    grdp_rel_path = part_node.get('file')
+    grdp_path = os.path.join(os.path.dirname(grd_path), grdp_rel_path)
+    grdp_paths.append(grdp_path)
+    grdp_dom, grdp_grpd_paths = _parse_grd_file(grdp_path)
+    grdp_paths.extend(grdp_grpd_paths)
+    part_node.append(grdp_dom.getroot())
+  return grd_dom, grdp_paths
+
+
+def _parse_translation_expectations(path):
+  """Parses a translations expectations file.
+
+  Example translations expectations file:
+  {
+    "desktop_grds": {
+      "languages": ["es", "fr"],
+      "files": [
+        "ash/ash_strings.grd",
+        "ui/strings/ui_strings.grd",
+      ],
+    },
+    "android_grds": {
+      "languages": ["de", "pt-BR"],
+      "files": [
+        "chrome/android/android_chrome_strings.grd",
+    },
+    "untranslated_grds": {
+      "chrome/locale_settings.grd": "Not UI strings; localized separately",
+      "chrome/locale_settings_mac.grd": "Not UI strings; localized separately",
+    },
+  }
+
+  Returns:
+    A tuple (grd_to_langs, untranslated_grds). grd_to_langs maps each grd path
+    to the list of languages into which that grd should be translated.
+    untranslated_grds is a list of grds that "appear translatable" but should
+    not be translated.
+  """
+  with open(path) as f:
+    file_contents = f.read()
+
+  def assert_list_of_strings(l, name):
+    assert isinstance(l, list) and all(isinstance(s, basestring) for s in l), (
+        '%s must be a list of strings' % name)
+
+  try:
+    translations_expectations = ast.literal_eval(file_contents)
+    assert isinstance(translations_expectations, dict), (
+        '%s must be a python dict' % path)
+
+    grd_to_langs = {}
+    untranslated_grds = []
+
+    for group_name, settings in translations_expectations.iteritems():
+      if group_name == 'untranslated_grds':
+        untranslated_grds = list(settings.keys())
+        assert_list_of_strings(untranslated_grds, 'untranslated_grds')
+        continue
+
+      languages = settings['languages']
+      files = settings['files']
+      assert_list_of_strings(languages, group_name + '.languages')
+      assert_list_of_strings(files, group_name + '.files')
+      for grd in files:
+        grd_to_langs[grd] = languages
+
+    return grd_to_langs, untranslated_grds
+
+  except Exception:
+    print 'Error: failed to parse', path
+    raise

tools/translation/helper/translation_helper_unittest.py

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Unit tests for translation_helper.py."""
+import unittest
+import os
+import sys
+
+# pylint: disable=relative-import
+import translation_helper
+
+here = os.path.realpath(__file__)
+testdata_path = os.path.normpath(os.path.join(here, '..', '..', 'testdata'))
+
+
+class TcHelperTest(unittest.TestCase):
+
+  def test_get_translatable_grds(self):
+    grds = translation_helper.get_translatable_grds(
+        testdata_path, ['test.grd', 'not_translated.grd'],
+        os.path.join(testdata_path, 'translation_expectations.pyl'))
+    self.assertEqual(1, len(grds))
+
+    # There should be no references to not_translated.grd (mentioning the
+    # filename here so that it doesn't appear unused).
+    grd = grds[0]
+    self.assertEqual(os.path.join(testdata_path, 'test.grd'), grd.path)
+    self.assertEqual(testdata_path, grd.dir)
+    self.assertEqual('test.grd', grd.name)
+    self.assertEqual([os.path.join(testdata_path, 'part.grdp')], grd.grdp_paths)
+    self.assertEqual([], grd.structure_paths)
+    self.assertEqual([os.path.join(testdata_path, 'test_en-GB.xtb')],
+                     grd.xtb_paths)
+    self.assertEqual({
+        'en-GB': os.path.join(testdata_path, 'test_en-GB.xtb')
+    }, grd.lang_to_xtb_path)
+    self.assertTrue(grd.appears_translatable)
+    self.assertEquals(['en-GB'], grd.expected_languages)
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/translation/testdata/not_translated.grd

+<?xml version="1.0" encoding="UTF-8"?>
+<grit latest_public_release="0" current_release="1" output_all_resource_defines="false">
+  <outputs>
+    <output filename="locale_settings_en-GB.pak" type="data_package" lang="en-GB" />
+  </outputs>
+  <translations>
+    <file path="locale_settings_en-GB.xtb" lang="en-GB" />
+  </translations>
+  <release seq="1" allow_pseudo="false">
+    <messages fallback_to_english="true">
+      <message name="IDS_NOT_TRANSLATED" use_name_for_id="true">
+        Not translated
+      </message>
+    </messages>
+  </release>
+</grit>
\ No newline at end of file

tools/translation/testdata/part.grdp

+<?xml version="1.0" encoding="utf-8"?>
+<grit-part>
+  <message name="IDS_PART_STRING1">
+    This is part string 1
+  </message>
+  <message name="IDS_PART_STRING2">
+    This is part string 2
+  </message>
+  <message name="IDS_PART_STRING_NON_TRANSLATEABLE" translateable="false">
+    This is a non translateable part string
+  </message>
+</grit-part>

tools/translation/testdata/test.grd

+<?xml version="1.0" encoding="UTF-8"?>
+
+<grit latest_public_release="1" current_release="1">
+  <translations>
+      <file path="test_en-GB.xtb" lang="en-GB" />
+  </translations>
+  <release seq="1">
+    <messages>
+      <message name="IDS_TEST_STRING1">
+        Test string 1
+      </message>
+      <message name="IDS_TEST_STRING2">
+        Test string 2
+      </message>
+      <message name="IDS_TEST_STRING_NON_TRANSLATEABLE" translateable="false">
+        This is a non translateable string
+      </message>
+      <part file="part.grdp" />
+    </messages>
+  </release>
+</grit>

tools/translation/testdata/test_en-GB.xtb

+<?xml version="1.0" ?>
+<!DOCTYPE translationbundle>
+<translationbundle lang="en-GB">
+<translation id="123">Test String 1</translation>
+<translation id="456">Test String 2</translation>
+</translationbundle>
\ No newline at end of file

tools/translation/testdata/translation_expectations.pyl

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# This is a .pyl, or "Python Literal", file. You can treat it just like a
+# .json file, with the following exceptions:
+# * all keys must be quoted (use single quotes, please);
+# * comments are allowed, using '#' syntax; and
+# * trailing commas are allowed.
+#
+# Specifies which grd files should be translated and into which languages they
+# should be translated. Used by the internal translation process.
+{
+  "desktop_grds": {
+    "languages": [
+      "en-GB"
+    ],
+    "files": [
+      "test.grd",
+    ],
+  },
+  # Grd files that contain <message> or <translations> elements, but that
+  # shouldn't be translated as part of the normal translation process. Each
+  # entry needs an explanation for why it shouldn't be translated.
+  "untranslated_grds": {
+    "not_translated.grd": "Not translated"
+  },
+}
\ No newline at end of file

tools/translation/upload_screenshots.py

+#!/usr/bin/env python
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""A tool to upload translation screenshots to Google Cloud Storage.
+
+This tool searches the current repo for .png files associated with .grd or
+.grdp files. It uploads the images to a Cloud Storage bucket and generates .sha1
+files. Finally, it asks the user if they want to add the .sha1 files to their
+CL.
+
+Images must be named the same as the UI strings they represent
+(e.g. IDS_HELLO.png for IDS_HELLO). The tool does NOT try to parse .grd/.grdp
+files, so it doesn't know whether an image file corresponds to a message or not.
+It will attempt to upload the image anyways.
+"""
+
+import argparse
+import sys
+import os
+import subprocess
+
+import helper.translation_helper as translation_helper
+
+here = os.path.dirname(os.path.realpath(__file__))
+src_path = os.path.normpath(os.path.join(here, '..', '..'))
+
+depot_tools_path = os.path.normpath(
+    os.path.join(src_path, 'third_party', 'depot_tools'))
+sys.path.insert(0, depot_tools_path)
+
+import upload_to_google_storage
+import download_from_google_storage
+
+sys.path.remove(depot_tools_path)
+
+# Translation expectations file for the Chromium repo.
+TRANSLATION_EXPECTATIONS_PATH = os.path.join('tools', 'gritsettings',
+                                             'translation_expectations.pyl')
+
+# URL of the bucket used for storing screenshots.
+BUCKET_URL = 'gs://chrome-screenshots'
+
+
+
+def query_yes_no(question, default='yes'):
+  """Ask a yes/no question via raw_input() and return their answer.
+
+  "question" is a string that is presented to the user.
+  "default" is the presumed answer if the user just hits <Enter>.
+      It must be "yes" (the default), "no" or None (meaning
+      an answer is required of the user).
+
+  The "answer" return value is True for "yes" or False for "no".
+  """
+  if default is None:
+    prompt = '[y/n] '
+  elif default == 'yes':
+    prompt = '[Y/n] '
+  elif default == 'no':
+    prompt = '[y/N] '
+  else:
+    raise ValueError("invalid default answer: '%s'" % default)
+
+  valid = {'yes': True, 'y': True, 'ye': True, 'no': False, 'n': False}
+  while True:
+    print question, prompt
+    choice = raw_input().lower()
+    if default is not None and choice == '':
+      return valid[default]
+    elif choice in valid:
+      return valid[choice]
+    else:
+      print "Please respond with 'yes' or 'no' (or 'y' or 'n')."
+
+
+def list_grds_in_repository(repo_path):
+  """Returns a list of all the grd files in the current git repository."""
+  # This works because git does its own glob expansion even though there is no
+  # shell to do it.
+  output = subprocess.check_output(
+      ['git', 'ls-files', '--', '*.grd'], cwd=repo_path)
+  return output.strip().splitlines()
+
+
+def git_add(files, repo_root):
+  """Adds relative paths given in files to the current CL."""
+  # Upload in batches in order to not exceed command line length limit.
+  BATCH_SIZE = 50
+  added_count = 0
+  while added_count < len(files):
+    batch = files[added_count:added_count+BATCH_SIZE]
+    command = ['git', 'add'] + batch
+    subprocess.check_call(command, cwd=repo_root)
+    added_count += len(batch)
+
+
+def find_screenshots(repo_root, translation_expectations):
+  grd_files = translation_helper.get_translatable_grds(
+      repo_root, list_grds_in_repository(repo_root), translation_expectations)
+
+  screenshots = []
+  for grd_file in grd_files:
+    grd_path = grd_file.path
+    # Convert grd_path.grd to grd_path_grd/ directory.
+    name, ext = os.path.splitext(os.path.basename(grd_path))
+    relative_screenshots_dir = os.path.relpath(
+        os.path.dirname(grd_path), repo_root)
+    screenshots_dir = os.path.realpath(
+        os.path.join(repo_root,
+                     os.path.join(relative_screenshots_dir,
+                                  name + ext.replace('.', '_'))))
+    # Grab all the .png files under the screenshot directory. On a clean
+    # checkout this should be an empty list, as the repo should only contain
+    # .sha1 files of previously uploaded screenshots.
+    if not os.path.exists(screenshots_dir):
+      continue
+    for f in os.listdir(screenshots_dir):
+      if f.endswith('.sha1'):
+        continue
+      if not f.endswith('.png'):
+        print 'File with unexpected extension: %s in %s' % (f, screenshots_dir)
+        continue
+      screenshots.append(os.path.join(screenshots_dir, f))
+  return screenshots
+
+
+def main():
+  parser = argparse.ArgumentParser(
+      description='Upload translation screenshots to Google Cloud Storage')
+  parser.add_argument(
+      '-n',
+      '--dry-run',
+      action='store_true',
+      help='Don\'t actually upload the images')
+  args = parser.parse_args()
+
+  screenshots = find_screenshots(src_path,
+                                 os.path.join(src_path,
+                                              TRANSLATION_EXPECTATIONS_PATH))
+  if not screenshots:
+    print 'No screenshots found, exiting.'
+
+  print 'Found %d updated screenshot(s): ' % len(screenshots)
+  for s in screenshots:
+    print '  %s' % s
+  print
+  if not query_yes_no('Do you want to upload these to Google Cloud Storage?'):
+    exit(0)
+
+  # Creating a standard gsutil object, assuming there are depot_tools
+  # and everything related is set up already.
+  gsutil_path = os.path.abspath(os.path.join(depot_tools_path, 'gsutil.py'))
+  gsutil = download_from_google_storage.Gsutil(gsutil_path, boto_path=None)
+
+  if not args.dry_run:
+    if upload_to_google_storage.upload_to_google_storage(
+        input_filenames=screenshots,
+        base_url=BUCKET_URL,
+        gsutil=gsutil,
+        force=False,
+        use_md5=False,
+        num_threads=1,
+        skip_hashing=False,
+        gzip=None) != 0:
+      print 'Error uploading screenshots, exiting.'
+      exit(1)
+
+  print
+  print 'Images are uploaded and their signatures are calculated:'
+
+  signatures = ['%s.sha1' % s for s in screenshots]
+  for s in signatures:
+    print '  %s' % s
+  print
+
+  # Always ask if the .sha1 files should be added to the CL, even if they are
+  # already part of the CL. If the files are not modified, adding again is a
+  # no-op.
+  if not query_yes_no('Do you want to add these files to your CL?'):
+    exit(0)
+
+  if not args.dry_run:
+    git_add(signatures, src_path)
+
+  print 'DONE.'
+
+
+if __name__ == '__main__':
+  main()

tools/translation/upload_screenshots_unittest.py

+#!/usr/bin/env python
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+import os
+import sys
+
+here = os.path.realpath(__file__)
+testdata_path = os.path.normpath(os.path.join(here, '..', 'testdata'))
+
+import upload_screenshots
+
+
+class UploadTests(unittest.TestCase):
+
+  def test_find_screenshots(self):
+    screenshots = upload_screenshots.find_screenshots(
+        testdata_path,
+        os.path.join(testdata_path, 'translation_expectations.pyl'))
+    self.assertEquals(1, len(screenshots))
+    self.assertEquals(
+        os.path.join(testdata_path, 'test_grd', 'IDS_TEST_STRING1.png'),
+        screenshots[0])
+
+
+if __name__ == '__main__':
+  unittest.main()