Skip to content

GitLab

T
tangled
Commit d0ed6939 authored by vabr's avatar vabr Committed by Commit bot
Browse files
Options

[Password manager Python tests] Re-arrange tests 

This CL changes the structure of the tests. Instead of many repetitions, there are now just 3 types of tests: failed login, successful login, and autofill test.

The Environment and WebsiteTest classes are also being refactored for clarity.

BUG=369521

Review URL: https://codereview.chromium.org/1026833003

Cr-Commit-Position: refs/heads/master@{#322574}
parent 9973bb23
Hide whitespace changes
Inline Side-by-side
Showing with 438 additions and 530 deletions
components/test/data/password_manager/automated_tests/README
View file @ d0ed6939
...@@ -75,10 +75,6 @@ class NewWebsiteTest(WebsiteTest): ...@@ -75,10 +75,6 @@ class NewWebsiteTest(WebsiteTest):
self.FillPasswordInto("Password CSS selector") self.FillPasswordInto("Password CSS selector")
self.Submit("Password CSS selector") self.Submit("Password CSS selector")
def Logout(self):
# Add logout steps for the website, for example:
self.Click("Logout button CSS selector")
Then, to create the new test, you need just to add: Then, to create the new test, you need just to add:
environment.AddWebsiteTest(NewWebsiteTest("website name")) environment.AddWebsiteTest(NewWebsiteTest("website name"))
...@@ -149,7 +145,7 @@ manager. When this bug is solved, all the tests that were failing because of ...@@ -149,7 +145,7 @@ manager. When this bug is solved, all the tests that were failing because of
it are going to be moved to working tests. it are going to be moved to working tests.
Other files: Other files:
* websites.xml : a private file where you can find all the passwords. You can * websites.xml: a private file where you can find all the passwords. You can
ask someone to give it to you or just create your own with your personal ask someone to give it to you or just create your own with your personal
accounts. accounts.
<websites> <websites>
......
components/test/data/password_manager/automated_tests/environment.py
View file @ d0ed6939
...@@ -2,7 +2,12 @@ ...@@ -2,7 +2,12 @@
# Use of this source code is governed by a BSD-style license that can be # Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file. # found in the LICENSE file.
"""The testing Environment class.""" """The testing Environment class.
It holds the WebsiteTest instances, provides them with credentials,
provides clean browser environment, runs the tests, and gathers the
results.
"""
import os import os
import shutil import shutil
...@@ -13,34 +18,18 @@ from selenium import webdriver ...@@ -13,34 +18,18 @@ from selenium import webdriver
from selenium.webdriver.chrome.options import Options from selenium.webdriver.chrome.options import Options
# Message strings to look for in chrome://password-manager-internals # Message strings to look for in chrome://password-manager-internals.
MESSAGE_ASK = "Message: Decision: ASK the user" MESSAGE_ASK = "Message: Decision: ASK the user"
MESSAGE_SAVE = "Message: Decision: SAVE the password" MESSAGE_SAVE = "Message: Decision: SAVE the password"
INTERNALS_PAGE_URL = "chrome://password-manager-internals/"
class TestResult:
"""Stores the information related to a test result. """
def __init__(self, name, test_type, successful, message):
"""Creates a new TestResult.
Args:
name: The tested website name.
test_type: The test type.
successful: Whether or not the test was successful.
message: The error message of the test.
"""
self.name = name
self.test_type = test_type
self.successful = successful
self.message = message
class Environment: class Environment:
"""Sets up the testing Environment. """ """Sets up the testing Environment. """
def __init__(self, chrome_path, chromedriver_path, profile_path, def __init__(self, chrome_path, chromedriver_path, profile_path,
passwords_path, enable_automatic_password_saving): passwords_path, enable_automatic_password_saving):
"""Creates a new testing Environment. """Creates a new testing Environment, starts Chromedriver.
Args: Args:
chrome_path: The chrome binary file. chrome_path: The chrome binary file.
...@@ -51,6 +40,8 @@ class Environment: ...@@ -51,6 +40,8 @@ class Environment:
saved without showing the prompt. saved without showing the prompt.
Raises: Raises:
IOError: When the passwords file cannot be accessed.
ParseError: When the passwords file cannot be parsed.
Exception: An exception is raised if |profile_path| folder could not be Exception: An exception is raised if |profile_path| folder could not be
removed. removed.
""" """
...@@ -58,39 +49,35 @@ class Environment: ...@@ -58,39 +49,35 @@ class Environment:
# Cleaning the chrome testing profile folder. # Cleaning the chrome testing profile folder.
if os.path.exists(profile_path): if os.path.exists(profile_path):
shutil.rmtree(profile_path) shutil.rmtree(profile_path)
options = Options() options = Options()
self.enable_automatic_password_saving = enable_automatic_password_saving
if enable_automatic_password_saving: if enable_automatic_password_saving:
options.add_argument("enable-automatic-password-saving") options.add_argument("enable-automatic-password-saving")
# Chrome path. # TODO(vabr): show_prompt is used in WebsiteTest for asserting that
# Chrome set-up corresponds to the test type. Remove that knowledge
# about Environment from the WebsiteTest.
self.show_prompt = not enable_automatic_password_saving
options.binary_location = chrome_path options.binary_location = chrome_path
# Chrome testing profile path.
options.add_argument("user-data-dir=%s" % profile_path) options.add_argument("user-data-dir=%s" % profile_path)
# The webdriver. It's possible to choose the port the service is going to # The webdriver. It's possible to choose the port the service is going to
# run on. If it's left to 0, a free port will be found. # run on. If it's left to 0, a free port will be found.
self.driver = webdriver.Chrome(chromedriver_path, 0, options) self.driver = webdriver.Chrome(chromedriver_path, 0, options)
# The password internals window.
# Password internals page tab/window handle.
self.internals_window = self.driver.current_window_handle self.internals_window = self.driver.current_window_handle
if passwords_path:
# An xml tree filled with logins and passwords. # An xml tree filled with logins and passwords.
self.passwords_tree = ElementTree.parse(passwords_path).getroot() self.passwords_tree = ElementTree.parse(passwords_path).getroot()
else:
raise Exception("Error: |passwords_path| needs to be provided if" self.website_window = self._OpenNewTab()
"|chrome_path| is provided, otherwise the tests could not be run")
# Password internals page.
self.internals_page = "chrome://password-manager-internals/"
# The Website window.
self.website_window = None
# The WebsiteTests list.
self.websitetests = [] self.websitetests = []
# Map messages to the number of their appearance in the log. # Map messages to the number of their appearance in the log.
self.message_count = { MESSAGE_ASK: 0, MESSAGE_SAVE: 0 } self.message_count = { MESSAGE_ASK: 0, MESSAGE_SAVE: 0 }
# The tests needs two tabs to work. A new tab is opened with the first
# GoTo. This is why we store here whether or not it's the first time to # A list of (test_name, test_type, test_success, failure_log).
# execute GoTo.
self.first_go_to = True
# List of all tests results.
self.tests_results = [] self.tests_results = []
def AddWebsiteTest(self, websitetest): def AddWebsiteTest(self, websitetest):
...@@ -110,248 +97,197 @@ class Environment: ...@@ -110,248 +97,197 @@ class Environment:
# TODO(vabr): Make driver a property of WebsiteTest. # TODO(vabr): Make driver a property of WebsiteTest.
websitetest.driver = self.driver websitetest.driver = self.driver
if not websitetest.username: if not websitetest.username:
username_tag = ( username_tag = (self.passwords_tree.find(
self.passwords_tree.find( ".//*[@name='%s']/username" % websitetest.name))
".//*[@name='%s']/username" % websitetest.name))
websitetest.username = username_tag.text websitetest.username = username_tag.text
if not websitetest.password: if not websitetest.password:
password_tag = ( password_tag = (self.passwords_tree.find(
self.passwords_tree.find( ".//*[@name='%s']/password" % websitetest.name))
".//*[@name='%s']/password" % websitetest.name))
websitetest.password = password_tag.text websitetest.password = password_tag.text
self.websitetests.append(websitetest) self.websitetests.append(websitetest)
def ClearCache(self, clear_passwords): def _ClearBrowserDataInit(self):
"""Clear the browser cookies. If |clear_passwords| is true, clear all the """Opens and resets the chrome://settings/clearBrowserData dialog.
saved passwords too.
Args: It unchecks all checkboxes, and sets the time range to the "beginning of
clear_passwords : Clear all the passwords if the bool value is true. time".
""" """
self.driver.get("chrome://settings/clearBrowserData") self.driver.get("chrome://settings/clearBrowserData")
self.driver.switch_to_frame("settings") self.driver.switch_to_frame("settings")
script = (
"if (!document.querySelector('#delete-cookies-checkbox').checked)" time_range_selector = "#clear-browser-data-time-period"
" document.querySelector('#delete-cookies-checkbox').click();" # TODO(vabr): Wait until time_range_selector is displayed instead.
)
negation = ""
if clear_passwords:
negation = "!"
script += (
"if (%sdocument.querySelector('#delete-passwords-checkbox').checked)"
" document.querySelector('#delete-passwords-checkbox').click();"
% negation)
script += "document.querySelector('#clear-browser-data-commit').click();"
self.driver.execute_script(script)
time.sleep(2) time.sleep(2)
# Every time we do something to the cache let's enable password saving. set_time_range = (
"var range = document.querySelector('{0}');".format(
time_range_selector) +
"range.value = 4" # 4 == the beginning of time
)
self.driver.execute_script(set_time_range)
all_cboxes_selector = (
"#clear-data-checkboxes [type=\"checkbox\"]")
uncheck_all = (
"var checkboxes = document.querySelectorAll('{0}');".format(
all_cboxes_selector ) +
"for (var i = 0; i < checkboxes.length; ++i) {"
" checkboxes[i].checked = false;"
"}"
)
self.driver.execute_script(uncheck_all)
def _ClearDataForCheckbox(self, selector):
"""Causes the data associated with |selector| to be cleared.
Opens chrome://settings/clearBrowserData, unchecks all checkboxes, then
checks the one described by |selector|, then clears the corresponding
browsing data for the full time range.
Args:
selector: describes the checkbox through which to delete the data.
"""
self._ClearBrowserDataInit()
check_cookies_and_submit = (
"document.querySelector('{0}').checked = true;".format(selector) +
"document.querySelector('#clear-browser-data-commit').click();"
)
self.driver.execute_script(check_cookies_and_submit)
def _EnablePasswordSaving(self):
"""Make sure that password manager is enabled."""
# TODO(melandory): We should check why it's off in a first place. # TODO(melandory): We should check why it's off in a first place.
# TODO(melandory): Investigate, maybe there is no need to enable it that # TODO(melandory): Investigate, maybe there is no need to enable it that
# often. # often.
self.EnablePasswordsSaving()
def EnablePasswordsSaving(self):
self.driver.get("chrome://settings") self.driver.get("chrome://settings")
self.driver.switch_to_frame("settings") self.driver.switch_to_frame("settings")
script = "document.getElementById('advanced-settings-expander').click();" script = "document.getElementById('advanced-settings-expander').click();"
self.driver.execute_script(script) self.driver.execute_script(script)
# TODO(vabr): Wait until element is displayed instead.
time.sleep(2) time.sleep(2)
script = ( script = (
"if (!document.querySelector('#password-manager-enabled').checked)" "document.querySelector('#password-manager-enabled').checked = true;")
"{ document.querySelector('#password-manager-enabled').click();}")
self.driver.execute_script(script) self.driver.execute_script(script)
time.sleep(2) time.sleep(2)
def OpenTabAndGoToInternals(self, url): def _OpenNewTab(self):
"""If there is no |self.website_window|, opens a new tab and navigates to """Open a new tab, and loads the internals page in the old tab.
|url| in the new tab. Navigates to the passwords internals page in the
first tab. Raises an exception otherwise.
Args:
url: Url to go to in the new tab.
Raises: Returns:
Exception: An exception is raised if |self.website_window| already A handle to the new tab.
exists.
""" """
if self.website_window:
raise Exception("Error: The window was already opened.")
self.driver.get("chrome://newtab") number_old_tabs = len(self.driver.window_handles)
# There is no straightforward way to open a new tab with chromedriver. # There is no straightforward way to open a new tab with chromedriver.
# One work-around is to go to a website, insert a link that is going # One work-around is to go to a website, insert a link that is going
# to be opened in a new tab, click on it. # to be opened in a new tab, and click on it.
self.driver.get("about:blank")
a = self.driver.execute_script( a = self.driver.execute_script(
"var a = document.createElement('a');" "var a = document.createElement('a');"
"a.target = '_blank';" "a.target = '_blank';"
"a.href = arguments[0];" "a.href = 'about:blank';"
"a.innerHTML = '.';" "a.innerHTML = '.';"
"document.body.appendChild(a);" "document.body.appendChild(a);"
"return a;", "return a;")
url)
a.click() a.click()
time.sleep(1) while number_old_tabs == len(self.driver.window_handles):
time.sleep(1) # Wait until the new tab is opened.
self.website_window = self.driver.window_handles[-1] new_tab = self.driver.window_handles[-1]
self.driver.get(self.internals_page) self.driver.get(INTERNALS_PAGE_URL)
self.driver.switch_to_window(self.website_window) self.driver.switch_to_window(new_tab)
return new_tab
def SwitchToInternals(self): def _DidStringAppearUntilTimeout(self, strings, timeout):
"""Switches from the Website window to internals tab.""" """Checks whether some of |strings| appeared in the current page.
self.driver.switch_to_window(self.internals_window)
def SwitchFromInternals(self): Waits for up to |timeout| seconds until at least one of |strings| is
"""Switches from internals tab to the Website window.""" shown in the current page. Updates self.message_count with the current
self.driver.switch_to_window(self.website_window) number of occurrences of the shown string. Assumes that at most
one of |strings| is newly shown.
def _DidMessageAppearUntilTimeout(self, log_message, timeout):
"""Checks whether the save password prompt is shown.
Args: Args:
log_message: Log message to look for in the password internals. strings: A list of strings to look for.
timeout: There is some delay between the login and the password timeout: If any such string does not appear within the first |timeout|
internals update. The method checks periodically during the first seconds, it is considered a no-show.
|timeout| seconds if the internals page reports the prompt being
shown. If the prompt is not reported shown within the first
|timeout| seconds, it is considered not shown at all.
Returns: Returns:
True if the save password prompt is shown. True if one of |strings| is observed until |timeout|, False otherwise.
False otherwise.
""" """
log = self.driver.find_element_by_css_selector("#log-entries")
count = log.text.count(log_message)
if count > self.message_count[log_message]: log = self.driver.find_element_by_css_selector("#log-entries")
self.message_count[log_message] = count while timeout:
return True for string in strings:
elif timeout > 0: count = log.text.count(string)
if count > self.message_count[string]:
self.message_count[string] = count
return True
time.sleep(1) time.sleep(1)
return self._DidMessageAppearUntilTimeout(log_message, timeout - 1) timeout -= 1
else: return False
return False
def CheckForNewMessage(self, log_message, message_should_show_up, def CheckForNewString(self, strings, string_should_show_up, error):
error_message, timeout=15): """Checks that |strings| show up on the internals page as it should.
"""Detects whether the save password prompt is shown.
Args: Switches to the internals page and looks for a new instances of |strings|
log_message: Log message to look for in the password internals. The being shown up there. It checks that |string_should_show_up| is true if
only valid values are the constants MESSAGE_* defined at the and only if at leas one string from |strings| shows up, and throws an
beginning of this file. Exception if that check fails.
message_should_show_up: Whether or not the message is expected to be
shown.
error_message: Error message for the exception.
timeout: There is some delay between the login and the password
internals update. The method checks periodically during the first
|timeout| seconds if the internals page reports the prompt being
shown. If the prompt is not reported shown within the first
|timeout| seconds, it is considered not shown at all.
Raises:
Exception: An exception is raised in case the result does not match the
expectation
"""
if (self._DidMessageAppearUntilTimeout(log_message, timeout) !=
message_should_show_up):
raise Exception(error_message)
def AllTests(self, prompt_test):
"""Runs the tests on all the WebsiteTests.
TODO(vabr): Currently, "all tests" always means one.
Args: Args:
prompt_test: If True, tests caring about showing the save-password strings: A list of strings to look for in the internals page.
prompt are going to be run, otherwise tests which don't care about string_should_show_up: Whether or not at least one string from |strings|
the prompt are going to be run. is expected to be shown.
error: Error message for the exception.
Raises: Raises:
Exception: An exception is raised if the tests fail. Exception: (See above.)
""" """
if prompt_test:
self.PromptTestList(self.websitetests)
else:
self.TestList(self.websitetests)
def Test(self, tests, prompt_test): self.driver.switch_to_window(self.internals_window)
"""Runs the tests on websites named in |tests|. try:
if (self._DidStringAppearUntilTimeout(strings, 15) !=
string_should_show_up):
raise Exception(error)
finally:
self.driver.switch_to_window(self.website_window)
Args: def DeleteCookies(self):
tests: A list of the names of the WebsiteTests that are going to be """Deletes cookies via the settings page."""
tested.
prompt_test: If True, tests caring about showing the save-password
prompt are going to be run, otherwise tests which don't care about
the prompt are going to be executed.
Raises: self._ClearDataForCheckbox("#delete-cookies-checkbox")
Exception: An exception is raised if the tests fail.
"""
websitetests = []
for websitetest in self.websitetests:
if websitetest.name in tests:
websitetests.append(websitetest)
if prompt_test: def RunTestsOnSites(self, test_type):
self.PromptTestList(websitetests) """Runs the specified test on the known websites.
else:
self.TestList(websitetests)
def TestList(self, websitetests): Also saves the test results in the environment. Note that test types
"""Runs the tests on the websites in |websitetests|. differ in their requirements on whether the save password prompt
should be displayed. Make sure that such requirements are consistent
with the enable_automatic_password_saving argument passed to |self|
on construction.
Args: Args:
websitetests: A list of WebsiteTests that are going to be tested. test_type: A test identifier understood by WebsiteTest.run_test().
Raises:
Exception: An exception is raised if the tests fail.
""" """
self.ClearCache(True)
for websitetest in websitetests:
successful = True
error = ""
try:
websitetest.was_run = True
websitetest.WrongLoginTest()
websitetest.SuccessfulLoginTest()
self.ClearCache(False)
websitetest.SuccessfulLoginWithAutofilledPasswordTest()
self.ClearCache(True)
websitetest.SuccessfulLoginTest()
self.ClearCache(True)
except Exception as e:
successful = False
error = e.message
self.tests_results.append(TestResult(websitetest.name, "normal",
successful, error))
self.DeleteCookies()
self._ClearDataForCheckbox("#delete-passwords-checkbox")
self._EnablePasswordSaving()
def PromptTestList(self, websitetests): for websitetest in self.websitetests:
"""Runs the prompt tests on the websites in |websitetests|.
Args:
websitetests: A list of WebsiteTests that are going to be tested.
Raises:
Exception: An exception is raised if the tests fail.
"""
self.ClearCache(True)
for websitetest in websitetests:
successful = True successful = True
error = "" error = ""
try: try:
websitetest.was_run = True websitetest.RunTest(test_type)
websitetest.PromptTest()
except Exception as e: except Exception as e:
successful = False successful = False
error = e.message error = e.message
self.tests_results.append(TestResult(websitetest.name, "prompt", self.tests_results.append(
successful, error)) (websitetest.name, test_type, successful, error))
def Quit(self): def Quit(self):
"""Closes the tests.""" """Shuts down the driver."""
# Close the webdriver.
self.driver.quit() self.driver.quit()
components/test/data/password_manager/automated_tests/run_tests.py
View file @ d0ed6939
...@@ -67,12 +67,11 @@ class TestRunner(object): ...@@ -67,12 +67,11 @@ class TestRunner(object):
# TODO(vabr): Ideally we would replace timeout with something allowing # TODO(vabr): Ideally we would replace timeout with something allowing
# calling tests directly inside Python, and working on other platforms. # calling tests directly inside Python, and working on other platforms.
# #
# The website test runs in two passes, each pass has an internal # The website test runs multiple scenarios, each one has an internal
# timeout of 200s for waiting (see |remaining_time_to_wait| and # timeout of 200s for waiting (see |remaining_time_to_wait| and
# Wait() in websitetest.py). Accounting for some more time spent on # Wait() in websitetest.py). Expecting that not every scenario should
# the non-waiting execution, 300 seconds should be the upper bound on # take 200s, the maximum time allocated for all of them is 300s.
# the runtime of one pass, thus 600 seconds for the whole test. self.test_cmd = ["timeout", "300"] + self.test_cmd
self.test_cmd = ["timeout", "600"] + self.test_cmd
self.logger.log(SCRIPT_DEBUG, self.logger.log(SCRIPT_DEBUG,
"TestRunner set up for test %s, command '%s', " "TestRunner set up for test %s, command '%s', "
...@@ -110,21 +109,26 @@ class TestRunner(object): ...@@ -110,21 +109,26 @@ class TestRunner(object):
def _check_if_test_passed(self): def _check_if_test_passed(self):
"""Returns True if and only if the test passed.""" """Returns True if and only if the test passed."""
success = False
if os.path.isfile(self.results_path): if os.path.isfile(self.results_path):
with open(self.results_path, "r") as results: with open(self.results_path, "r") as results:
count = 0 # Count the number of successful tests. # TODO(vabr): Parse the results to make sure all scenarios succeeded
# instead of hard-coding here the number of tests scenarios from
# test.py:main.
NUMBER_OF_TEST_SCENARIOS = 3
passed_scenarios = 0
for line in results: for line in results:
self.failures.append(line) self.failures.append(line)
count += line.count("successful='True'") passed_scenarios += line.count("successful='True'")
success = passed_scenarios == NUMBER_OF_TEST_SCENARIOS
# There is only two tests running for every website: the prompt and if success:
# the normal test. If both of the tests were successful, the tests break
# would be stopped for the current website.
self.logger.log(SCRIPT_DEBUG, "Test run of %s: %s", self.logger.log(
self.test_name, "pass" if count == 2 else "fail") SCRIPT_DEBUG,
if count == 2: "Test run of {0} succeded: {1}".format(self.test_name, success))
return True return success
return False
def _run_test(self): def _run_test(self):
"""Executes the command to run the test.""" """Executes the command to run the test."""
......
components/test/data/password_manager/automated_tests/tests.py
View file @ d0ed6939
...@@ -475,7 +475,7 @@ all_tests = { ...@@ -475,7 +475,7 @@ all_tests = {
} }
def saveResults(environment_tests_results, environment_save_path): def SaveResults(environment_tests_results, environment_save_path):
"""Save the test results in an xml file. """Save the test results in an xml file.
Args: Args:
...@@ -488,17 +488,16 @@ def saveResults(environment_tests_results, environment_save_path): ...@@ -488,17 +488,16 @@ def saveResults(environment_tests_results, environment_save_path):
""" """
if environment_save_path: if environment_save_path:
xml = "<result>" xml = "<result>"
for test_result in environment_tests_results: for (name, test_type, success, failure_log) in environment_tests_results:
xml += ("<test name='%s' successful='%s' type='%s'>%s</test>" xml += (
% (test_result.name, str(test_result.successful), "<test name='{0}' successful='{1}' type='{2}'>{3}</test>".format(
test_result.test_type, test_result.message)) name, success, test_type, failure_log))
xml += "</result>" xml += "</result>"
with open(environment_save_path, "w") as save_file: with open(environment_save_path, "w") as save_file:
save_file.write(xml) save_file.write(xml)
def RunTest(chrome_path, chromedriver_path, profile_path, def RunTest(chrome_path, chromedriver_path, profile_path,
environment_passwords_path, enable_automatic_password_saving, environment_passwords_path, website_test_name, test_type):
website_test_name):
"""Runs the test for the specified website. """Runs the test for the specified website.
Args: Args:
...@@ -506,8 +505,6 @@ def RunTest(chrome_path, chromedriver_path, profile_path, ...@@ -506,8 +505,6 @@ def RunTest(chrome_path, chromedriver_path, profile_path,
chromedriver_path: The chromedriver binary file. chromedriver_path: The chromedriver binary file.
profile_path: The chrome testing profile folder. profile_path: The chrome testing profile folder.
environment_passwords_path: The usernames and passwords file. environment_passwords_path: The usernames and passwords file.
enable_automatic_password_saving: If True, the passwords are going to be
saved without showing the prompt.
website_test_name: Name of the website to test (refer to keys in website_test_name: Name of the website to test (refer to keys in
all_tests above). all_tests above).
...@@ -519,26 +516,22 @@ def RunTest(chrome_path, chromedriver_path, profile_path, ...@@ -519,26 +516,22 @@ def RunTest(chrome_path, chromedriver_path, profile_path,
fails, or if the website name is not known. fails, or if the website name is not known.
""" """
enable_automatic_password_saving = (
test_type == WebsiteTest.TEST_TYPE_SAVE_AND_AUTOFILL)
environment = Environment(chrome_path, chromedriver_path, profile_path, environment = Environment(chrome_path, chromedriver_path, profile_path,
environment_passwords_path, environment_passwords_path,
enable_automatic_password_saving) enable_automatic_password_saving)
# Test which care about the save-password prompt need the prompt
# to be shown. Automatic password saving results in no prompt.
run_prompt_tests = not enable_automatic_password_saving
if website_test_name in all_tests: if website_test_name in all_tests:
environment.AddWebsiteTest(all_tests[website_test_name]) environment.AddWebsiteTest(all_tests[website_test_name])
else: else:
raise Exception("Test name {} is unknown.".format(website_test_name)) raise Exception("Test name {} is unknown.".format(website_test_name))
environment.AllTests(run_prompt_tests) environment.RunTestsOnSites(test_type)
environment.Quit() environment.Quit()
return environment.tests_results return environment.tests_results
# Tests setup. def main():
if __name__ == "__main__":
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Password Manager automated tests help.") description="Password Manager automated tests help.")
...@@ -568,22 +561,19 @@ if __name__ == "__main__": ...@@ -568,22 +561,19 @@ if __name__ == "__main__":
if args.save_path: if args.save_path:
save_path = args.save_path save_path = args.save_path
# Run the test without enable-automatic-password-saving to check whether or tests_results = RunTest(
# not the prompt is shown in the way we expected. args.chrome_path, args.chromedriver_path, args.profile_path,
tests_results = RunTest(args.chrome_path, args.passwords_path, args.test, WebsiteTest.TEST_TYPE_PROMPT_FAIL)
args.chromedriver_path,
args.profile_path, tests_results += RunTest(
args.passwords_path, args.chrome_path, args.chromedriver_path, args.profile_path,
False, args.passwords_path, args.test, WebsiteTest.TEST_TYPE_PROMPT_SUCCESS)
args.test)
tests_results += RunTest(
# Run the test with enable-automatic-password-saving to check whether or not args.chrome_path, args.chromedriver_path, args.profile_path,
# the passwords is stored in the the way we expected. args.passwords_path, args.test, WebsiteTest.TEST_TYPE_SAVE_AND_AUTOFILL)
tests_results += RunTest(args.chrome_path,
args.chromedriver_path, SaveResults(tests_results, save_path)
args.profile_path,
args.passwords_path, if __name__ == "__main__":
True, main()
args.test)
saveResults(tests_results, save_path)
components/test/data/password_manager/automated_tests/websitetest.py
View file @ d0ed6939
...@@ -12,390 +12,372 @@ from selenium.webdriver.common.keys import Keys ...@@ -12,390 +12,372 @@ from selenium.webdriver.common.keys import Keys
import environment import environment
SCRIPT_DEBUG = 9 # TODO(vabr) -- make this consistent with run_tests.py.
def _IsOneSubstringOfAnother(s1, s2): class WebsiteTest:
"""Checks if one of the string arguements is substring of the other. """WebsiteTest testing class.
Args:
s1: The first string.
s2: The second string.
Returns:
True if one of the string arguements is substring of the other. Represents one website, defines some generic operations on that site.
False otherwise. To customise for a particular website, this class needs to be inherited
and the Login() method overridden.
""" """
return s1 in s2 or s2 in s1
# Possible values of self.autofill_expectation.
AUTOFILLED = 1 # Expect password and username to be autofilled.
NOT_AUTOFILLED = 2 # Expect password and username not to be autofilled.
class WebsiteTest: # The maximal accumulated time to spend in waiting for website UI
"""Handles a tested WebsiteTest.""" # interaction.
MAX_WAIT_TIME_IN_SECONDS = 200
class Mode:
"""Test mode."""
# Password and username are expected to be autofilled.
AUTOFILLED = 1
# Password and username are not expected to be autofilled.
NOT_AUTOFILLED = 2
def __init__(self): # Types of test to be passed to self.RunTest().
pass TEST_TYPE_PROMPT_FAIL = 1
TEST_TYPE_PROMPT_SUCCESS = 2
TEST_TYPE_SAVE_AND_AUTOFILL = 3
def __init__(self, name, username_not_auto=False): def __init__(self, name, username_not_auto=False):
"""Creates a new WebsiteTest. """Creates a new WebsiteTest.
Args: Args:
name: The website name. name: The website name, identifying it in the test results.
username_not_auto: Username inputs in some websites (like wikipedia) are username_not_auto: Expect that the tested website fills username field
sometimes filled with some messages and thus, the usernames are not on load, and Chrome cannot autofill in that case.
automatically autofilled. This flag handles that and disables us from
checking if the state of the DOM is the same as the username of
website.
""" """
# Name of the website
self.name = name self.name = name
# Username of the website.
self.username = None self.username = None
# Password of the website.
self.password = None self.password = None
# Username is not automatically filled.
self.username_not_auto = username_not_auto self.username_not_auto = username_not_auto
# Autofilling mode.
self.mode = self.Mode.NOT_AUTOFILLED # Specify, whether it is expected that credentials get autofilled.
# The |remaining_time_to_wait| limits the total time in seconds spent in self.autofill_expectation = WebsiteTest.NOT_AUTOFILLED
# potentially infinite loops. self.remaining_seconds_to_wait = WebsiteTest.MAX_WAIT_TIME_IN_SECONDS
self.remaining_time_to_wait = 200 # The testing Environment, if added to any.
# The testing Environment.
self.environment = None self.environment = None
# The webdriver. # The webdriver from the environment.
self.driver = None self.driver = None
# Whether or not the test was run.
self.was_run = False
# Mouse/Keyboard actions. # Mouse/Keyboard actions.
def Click(self, selector): def Click(self, selector):
"""Clicks on an element. """Clicks on the element described by |selector|.
Args: Args:
selector: The element CSS selector. selector: The clicked element's CSS selector.
""" """
logging.info("action: Click %s" % selector)
self.WaitUntilDisplayed(selector) logging.log(SCRIPT_DEBUG, "action: Click %s" % selector)
element = self.driver.find_element_by_css_selector(selector) element = self.WaitUntilDisplayed(selector)
element.click() element.click()
def ClickIfClickable(self, selector): def ClickIfClickable(self, selector):
"""Clicks on an element if it's clickable: If it doesn't exist in the DOM, """Clicks on the element described by |selector| if it is clickable.
it's covered by another element or it's out viewing area, nothing is
done and False is returned. Otherwise, even if the element is 100% The driver's find_element_by_css_selector method defines what is clickable
transparent, the element is going to receive a click and a True is -- anything for which it does not throw, is clickable. To be clickable,
returned. the element must:
* exist in the DOM,
* be not covered by another element
* be inside the visible area.
Note that transparency does not influence clickability.
Args: Args:
selector: The element CSS selector. selector: The clicked element's CSS selector.
Returns: Returns:
True if the click happens. True if the element is clickable (and was clicked on).
False otherwise. False otherwise.
""" """
logging.info("action: ClickIfVisible %s" % selector)
self.WaitUntilDisplayed(selector) logging.log(SCRIPT_DEBUG, "action: ClickIfVisible %s" % selector)
element = self.WaitUntilDisplayed(selector)
try: try:
element = self.driver.find_element_by_css_selector(selector)
element.click() element.click()
return True return True
except Exception: except Exception:
return False return False
def GoTo(self, url): def GoTo(self, url):
"""Navigates the main frame to the |url|. """Navigates the main frame to |url|.
Args: Args:
url: The URL. url: The URL of where to go to.
""" """
logging.info("action: GoTo %s" % self.name)
if self.environment.first_go_to: logging.log(SCRIPT_DEBUG, "action: GoTo %s" % self.name)
self.environment.OpenTabAndGoToInternals(url) self.driver.get(url)
self.environment.first_go_to = False
else:
self.driver.get(url)
def HoverOver(self, selector): def HoverOver(self, selector):
"""Hovers over an element. """Hovers over the element described by |selector|.
Args: Args:
selector: The element CSS selector. selector: The CSS selector of the element to hover over.
""" """
logging.info("action: Hover %s" % selector)
self.WaitUntilDisplayed(selector) logging.log(SCRIPT_DEBUG, "action: Hover %s" % selector)
element = self.driver.find_element_by_css_selector(selector) element = self.WaitUntilDisplayed(selector)
hover = ActionChains(self.driver).move_to_element(element) hover = ActionChains(self.driver).move_to_element(element)
hover.perform() hover.perform()
# Waiting/Displaying actions. # Waiting/Displaying actions.
def IsDisplayed(self, selector): def _ReturnElementIfDisplayed(self, selector):
"""Returns False if an element doesn't exist in the DOM or is 100% """Returns the element described by |selector|, if displayed.
transparent. Otherwise, returns True even if it's covered by another
element or it's out viewing area. Note: This takes neither overlapping among elements nor position with
regards to the visible area into account.
Args: Args:
selector: The element CSS selector. selector: The CSS selector of the checked element.
Returns:
The element if displayed, None otherwise.
""" """
logging.info("action: IsDisplayed %s" % selector)
try: try:
element = self.driver.find_element_by_css_selector(selector) element = self.driver.find_element_by_css_selector(selector)
return element.is_displayed() return element if element.is_displayed() else None
except Exception: except Exception:
return False return None
def IsDisplayed(self, selector):
"""Check if the element described by |selector| is displayed.
Note: This takes neither overlapping among elements nor position with
regards to the visible area into account.
Args:
selector: The CSS selector of the checked element.
Returns:
True if the element is in the DOM and less than 100% transparent.
False otherwise.
"""
logging.log(SCRIPT_DEBUG, "action: IsDisplayed %s" % selector)
return self._ReturnElementIfDisplayed(selector) is not None
def Wait(self, duration): def Wait(self, duration):
"""Wait for a duration in seconds. This needs to be used in potentially """Wait for |duration| in seconds.
infinite loops, to limit their running time.
To avoid deadlocks, the accummulated waiting time for the whole object does
not exceed MAX_WAIT_TIME_IN_SECONDS.
Args: Args:
duration: The time to wait in seconds. duration: The time to wait in seconds.
Raises:
Exception: In case the accummulated waiting limit is exceeded.
""" """
logging.info("action: Wait %s" % duration)
logging.log(SCRIPT_DEBUG, "action: Wait %s" % duration)
self.remaining_seconds_to_wait -= duration
if self.remaining_seconds_to_wait < 0:
raise Exception("Waiting limit exceeded for website: %s" % self.name)
time.sleep(duration) time.sleep(duration)
self.remaining_time_to_wait -= 1
if self.remaining_time_to_wait < 0:
raise Exception("Tests took more time than expected for the following "
"website : %s \n" % self.name)
def WaitUntilDisplayed(self, selector, timeout=10): # TODO(vabr): Pull this out into some website-utils and use in Environment
"""Waits until an element is displayed. # also?
def WaitUntilDisplayed(self, selector):
"""Waits until the element described by |selector| is displayed.
Args: Args:
selector: The element CSS selector. selector: The CSS selector of the element to wait for.
timeout: The maximum waiting time in seconds before failing.
Returns:
The displayed element.
""" """
if not self.IsDisplayed(selector):
element = self._ReturnElementIfDisplayed(selector)
while not element:
self.Wait(1) self.Wait(1)
timeout = timeout - 1 element = self._ReturnElementIfDisplayed(selector)
if (timeout <= 0): return element
raise Exception("Error: Element %s not shown before timeout is "
"finished for the following website: %s"
% (selector, self.name))
else:
self.WaitUntilDisplayed(selector, timeout)
# Form actions. # Form actions.
def FillPasswordInto(self, selector): def FillPasswordInto(self, selector):
"""If the testing mode is the Autofilled mode, compares the website """Ensures that the selected element's value is the saved password.
password to the DOM state.
If the testing mode is the NotAutofilled mode, checks that the DOM state Depending on self.autofill_expectation, this either checks that the
is empty. element already has the password autofilled, or checks that the value
Then, fills the input with the Website password. is empty and replaces it with the password.
Args: Args:
selector: The password input CSS selector. selector: The CSS selector for the filled element.
Raises: Raises:
Exception: An exception is raised if the DOM value of the password is Exception: An exception is raised if the element's value is different
different than the one we expected. from the expectation.
""" """
logging.info("action: FillPasswordInto %s" % selector)
self.WaitUntilDisplayed(selector) logging.log(SCRIPT_DEBUG, "action: FillPasswordInto %s" % selector)
password_element = self.driver.find_element_by_css_selector(selector) password_element = self.WaitUntilDisplayed(selector)
# Chrome protects the password inputs and doesn't fill them until # Chrome protects the password inputs and doesn't fill them until
# the user interacts with the page. To be sure that such thing has # the user interacts with the page. To be sure that such thing has
# happened we perform |Keys.CONTROL| keypress. # happened we perform |Keys.CONTROL| keypress.
action_chains = ActionChains(self.driver) action_chains = ActionChains(self.driver)
action_chains.key_down(Keys.CONTROL).key_up(Keys.CONTROL).perform() action_chains.key_down(Keys.CONTROL).key_up(Keys.CONTROL).perform()
if self.mode == self.Mode.AUTOFILLED:
autofilled_password = password_element.get_attribute("value")
if autofilled_password != self.password:
raise Exception("Error: autofilled password is different from the one "
"we just saved for the following website : %s p1: %s "
"p2:%s \n" % (self.name,
password_element.get_attribute("value"),
self.password))
elif self.mode == self.Mode.NOT_AUTOFILLED:
autofilled_password = password_element.get_attribute("value")
if autofilled_password:
raise Exception("Error: password is autofilled when it shouldn't be "
"for the following website : %s \n"
% self.name)
if self.autofill_expectation == WebsiteTest.AUTOFILLED:
if password_element.get_attribute("value") != self.password:
raise Exception("Error: autofilled password is different from the saved"
" one on website: %s" % self.name)
elif self.autofill_expectation == WebsiteTest.NOT_AUTOFILLED:
if password_element.get_attribute("value"):
raise Exception("Error: password value unexpectedly not empty on"
"website: %s" % self.name)
password_element.send_keys(self.password) password_element.send_keys(self.password)
def FillUsernameInto(self, selector): def FillUsernameInto(self, selector):
"""If the testing mode is the Autofilled mode, compares the website """Ensures that the selected element's value is the saved username.
username to the input value. Then, fills the input with the website
Depending on self.autofill_expectation, this either checks that the
element already has the username autofilled, or checks that the value
is empty and replaces it with the password. If self.username_not_auto
is true, it skips the checks and just overwrites the value with the
username. username.
Args: Args:
selector: The username input CSS selector. selector: The CSS selector for the filled element.
Raises: Raises:
Exception: An exception is raised if the DOM value of the username is Exception: An exception is raised if the element's value is different
different that the one we expected. from the expectation.
""" """
logging.info("action: FillUsernameInto %s" % selector)
self.WaitUntilDisplayed(selector) logging.log(SCRIPT_DEBUG, "action: FillUsernameInto %s" % selector)
username_element = self.driver.find_element_by_css_selector(selector) username_element = self.WaitUntilDisplayed(selector)
if (self.mode == self.Mode.AUTOFILLED and not self.username_not_auto): if not self.username_not_auto:
if not (username_element.get_attribute("value") == self.username): if self.autofill_expectation == WebsiteTest.AUTOFILLED:
raise Exception("Error: autofilled username is different form the one " if username_element.get_attribute("value") != self.username:
"we just saved for the following website : %s \n" % raise Exception("Error: filled username different from the saved"
self.name) " one on website: %s" % self.name)
else: return
username_element.clear() if self.autofill_expectation == WebsiteTest.NOT_AUTOFILLED:
username_element.send_keys(self.username) if username_element.get_attribute("value"):
raise Exception("Error: username value unexpectedly not empty on"
"website: %s" % self.name)
username_element.clear()
username_element.send_keys(self.username)
def Submit(self, selector): def Submit(self, selector):
"""Finds an element using CSS Selector and calls its submit() handler. """Finds an element using CSS |selector| and calls its submit() handler.
Args: Args:
selector: The input CSS selector. selector: The CSS selector for the element to call submit() on.
""" """
logging.info("action: Submit %s" % selector)
self.WaitUntilDisplayed(selector) logging.log(SCRIPT_DEBUG, "action: Submit %s" % selector)
element = self.driver.find_element_by_css_selector(selector) element = self.WaitUntilDisplayed(selector)
element.submit() element.submit()
# Login/Logout Methods # Login/Logout methods
def Login(self): def Login(self):
"""Login Method. Has to be overloaded by the WebsiteTest test.""" """Login Method. Has to be overridden by the WebsiteTest test."""
raise NotImplementedError("Login is not implemented.") raise NotImplementedError("Login is not implemented.")
def LoginWhenAutofilled(self): def LoginWhenAutofilled(self):
"""Logs in and checks that the password is autofilled.""" """Logs in and checks that the password is autofilled."""
self.mode = self.Mode.AUTOFILLED
self.autofill_expectation = WebsiteTest.AUTOFILLED
self.Login() self.Login()
def LoginWhenNotAutofilled(self): def LoginWhenNotAutofilled(self):
"""Logs in and checks that the password is not autofilled.""" """Logs in and checks that the password is not autofilled."""
self.mode = self.Mode.NOT_AUTOFILLED
self.autofill_expectation = WebsiteTest.NOT_AUTOFILLED
self.Login() self.Login()
def Logout(self): def Logout(self):
"""Logout Method.""" self.environment.DeleteCookies()
# Tests # Test scenarios
def WrongLoginTest(self): def PromptFailTest(self):
"""Does the wrong login test: Tries to login with a wrong password and """Checks that prompt is not shown on a failed login attempt.
checks that the password is not saved.
Tries to login with a wrong password and checks that the password
is not offered for saving.
Raises: Raises:
Exception: An exception is raised if the test fails: If there is a Exception: An exception is raised if the test fails.
problem when performing the login (ex: the login button is not
available ...), if the state of the username and password fields is
not like we expected or if the password is saved.
""" """
logging.info("\nWrong Login Test for %s \n" % self.name)
try: logging.log(SCRIPT_DEBUG, "PromptFailTest for %s" % self.name)
correct_password = self.password correct_password = self.password
# Hardcoded random wrong password. Chosen by fair `pwgen` call. # Hardcoded random wrong password. Chosen by fair `pwgen` call.
# For details, see: http://xkcd.com/221/. # For details, see: http://xkcd.com/221/.
self.password = "ChieF2ae" self.password = "ChieF2ae"
self.LoginWhenNotAutofilled() self.LoginWhenNotAutofilled()
self.password = correct_password self.password = correct_password
self.Wait(2) self.environment.CheckForNewString(
self.environment.SwitchToInternals() [environment.MESSAGE_ASK, environment.MESSAGE_SAVE],
self.environment.CheckForNewMessage( False,
environment.MESSAGE_SAVE, "Error: did not detect wrong login on website: %s" % self.name)
False,
"Error: password manager thinks that a login with wrong password was " def PromptSuccessTest(self):
"successful for the following website : %s \n" % self.name) """Checks that prompt is shown on a successful login attempt.
finally:
self.environment.SwitchFromInternals() Tries to login with a correct password and checks that the password
is offered for saving. Chrome cannot have the auto-save option on
def SuccessfulLoginTest(self): when running this test.
"""Does the successful login when the password is not expected to be
autofilled test: Checks that the password is not autofilled, tries to login
with a right password and checks if the password is saved. Then logs out.
Raises: Raises:
Exception: An exception is raised if the test fails: If there is a Exception: An exception is raised if the test fails.
problem when performing the login and the logout (ex: the login
button is not available ...), if the state of the username and
password fields is not like we expected or if the password is not
saved.
""" """
logging.info("\nSuccessful Login Test for %s \n" % self.name)
try: logging.log(SCRIPT_DEBUG, "PromptSuccessTest for %s" % self.name)
self.LoginWhenNotAutofilled() if not self.environment.show_prompt:
self.Wait(2) raise Exception("Switch off auto-save during PromptSuccessTest.")
self.environment.SwitchToInternals() self.LoginWhenNotAutofilled()
self.environment.CheckForNewMessage( self.environment.CheckForNewString(
environment.MESSAGE_SAVE, [environment.MESSAGE_ASK],
True, True,
"Error: password manager hasn't detected a successful login for the " "Error: did not detect login success on website: %s" % self.name)
"following website : %s \n"
% self.name) def SaveAndAutofillTest(self):
finally: """Checks that a correct password is saved and autofilled.
self.environment.SwitchFromInternals()
self.Logout() Tries to login with a correct password and checks that the password
is saved and autofilled on next visit. Chrome must have the auto-save
def SuccessfulLoginWithAutofilledPasswordTest(self): option on when running this test.
"""Does the successful login when the password is expected to be autofilled
test: Checks that the password is autofilled, tries to login with the
autofilled password and checks if the password is saved. Then logs out.
Raises: Raises:
Exception: An exception is raised if the test fails: If there is a Exception: An exception is raised if the test fails.
problem when performing the login and the logout (ex: the login
button is not available ...), if the state of the username and
password fields is not like we expected or if the password is not
saved.
""" """
logging.info("\nSuccessful Login With Autofilled Password"
" Test %s \n" % self.name) logging.log(SCRIPT_DEBUG, "SaveAndAutofillTest for %s" % self.name)
try: if self.environment.show_prompt:
self.LoginWhenAutofilled() raise Exception("Switch off auto-save during PromptSuccessTest.")
self.Wait(2) self.LoginWhenNotAutofilled()
self.environment.SwitchToInternals() self.environment.CheckForNewString(
self.environment.CheckForNewMessage( [environment.MESSAGE_SAVE],
environment.MESSAGE_SAVE, True,
True, "Error: did not detect login success on website: %s" % self.name)
"Error: password manager hasn't detected a successful login for the " self.Logout()
"following website : %s \n" self.LoginWhenAutofilled()
% self.name) self.environment.CheckForNewString(
finally: [environment.MESSAGE_SAVE],
self.environment.SwitchFromInternals() True,
self.Logout() "Error: failed autofilled login on website: %s" % self.name)
def PromptTest(self): def RunTest(self, test_type):
"""Does the prompt test: Tries to login with a wrong password and """Runs test according to the |test_type|.
checks that the prompt is not shown. Then tries to login with a right
password and checks that the prompt is not shown.
Raises: Raises:
Exception: An exception is raised if the test fails: If there is a Exception: If |test_type| is not one of the TEST_TYPE_* constants.
problem when performing the login (ex: the login button is not
available ...), if the state of the username and password fields is
not like we expected or if the prompt is not shown for the right
password or is shown for a wrong one.
""" """
logging.info("\nPrompt Test for %s \n" % self.name)
try: if test_type == WebsiteTest.TEST_TYPE_PROMPT_FAIL:
correct_password = self.password self.PromptFailTest()
self.password = self.password + "1" elif test_type == WebsiteTest.TEST_TYPE_PROMPT_SUCCESS:
self.LoginWhenNotAutofilled() self.PromptSuccessTest()
self.password = correct_password elif test_type == WebsiteTest.TEST_TYPE_SAVE_AND_AUTOFILL:
self.Wait(2) self.SaveAndAutofillTest()
self.environment.SwitchToInternals() else:
self.environment.CheckForNewMessage( raise Exception("Unknown test type {}.".format(test_type))
environment.MESSAGE_ASK,
False,
"Error: password manager thinks that a login with wrong password was "
"successful for the following website : %s \n" % self.name)
self.environment.SwitchFromInternals()
self.LoginWhenNotAutofilled()
self.Wait(2)
self.environment.SwitchToInternals()
self.environment.CheckForNewMessage(
environment.MESSAGE_ASK,
True,
"Error: password manager hasn't detected a successful login for the "
"following website : %s \n" % self.name)
finally:
self.environment.SwitchFromInternals()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment