Skip to content

GitLab

T
tangled
Commit 856e676d authored by Yiming Zhou's avatar Yiming Zhou Committed by Commit Bot
Browse files
Options

Introducing improvements for the Action Recorder Extension. 

1. Improved the xpath generator. The generator will produce even more concise xpaths. This improvement helps to make test recipes readable to humans.
2. Changed the way the extension queries for an iframe's context. Prior to this change, the extension would construct a path from the iframe to the top level frame by making a query at each ancestor frame node. However, InProcessBrowserTest does not need a path to go from the top level frame to a descendant iframe. Therefore, in this change the extension only makes one query at the parent frame node.
3. Moved the frame context query action to the front of the start recording workflow. This move cuts one message roundtrip from the extension's background script to the extension's content script.
4. Made the extension jot down a target element's visibility state when recording an action. Prior to this change, the extension assumes that every target element is visible, enabled and on the top of the page. However, the extension's complex action recording logic sometimes catches actions on invisible or partially obscured elements.
5. Added refined logic to the extension to distinguish between a user typing action and a Chrome autofill action. Prior to this change the extension simply assumes that Chrome always autofills every autofill-able field. If a user types inside an autofillable field, the extension will mistakenly record an autofill action. With this change the extension uses keyboard events to detect when a user types inside a field, eliminating the false positive.
6. Began implementing new features to capture Chrome Password Manager actions.

Bug: 855284
Change-Id: Ic7ff3af95cdc9f308c3ad061a3506fced150b4f8
Reviewed-on: https://chromium-review.googlesource.com/1132540
Commit-Queue: Yiming Zhou <uwyiming@google.com>
Reviewed-by: default avatarJared Saul <jsaul@google.com>
Reviewed-by: default avatarMathieu Perreault <mathp@chromium.org>
Cr-Commit-Position: refs/heads/master@{#574385}
parent c18d1990
Show whitespace changes
Inline Side-by-side
Showing with 285 additions and 144 deletions
components/test/data/autofill/automated_integration/action_recorder_extension/background/background.js
View file @ 856e676d
......@@ -288,26 +288,80 @@
});
}
function getIframeContext(tabId, frameId, iframeLocation) {
function getIframeContext(tabId, frameId) {
return new Promise((resolve, reject) => {
if (frameId === 0) {
resolve({ isIframe: false });
} else {
chrome.webNavigation.getFrame(
{ tabId: tabId, frameId: frameId }, (details) => {
if (chrome.runtime.lastError) {
reject(`Unable to query for frame info on frame ${frameId}`);
} else {
let context = { isIframe: true };
getAllFramesInTab(tabId)
.then((details) => {
let targetFrame;
for (let index = 0; index < details.length; index++) {
if (details[index].frameId === frameId) {
targetFrame = details[index];
break;
}
}
// Send a message to the parent frame and see if the iframe has a
// 'name' attribute.
sendMessageToTab(tabId, {
type: RecorderMsgEnum.GET_IFRAME_XPATH,
frameId: frameId,
location: iframeLocation
}, { frameId: details.parentFrameId }).then((context) => {
type: RecorderMsgEnum.GET_IFRAME_NAME,
url: targetFrame.url
}, {
frameId: targetFrame.parentFrameId
})
.then((frameName) => {
if (frameName !== '') {
context.browserTest = { name: frameName };
resolve(context);
}).catch((message) => {
reject(message);
});
} else {
const targetFrameUrl = new URL(targetFrame.url);
// The frame does not have a 'name' attribute. Check if the
// frame has a unique combination of scheme, host and port.
//
// The Captured Site automation framework can identify an
// iframe by its scheme + host + port, provided this
// information combination is unique. Identifying an iframe
// through its scheme + host + port is more preferable than
// identifying an iframe through its URL. An URL will
// frequently contain parameters, and many websites use random
// number generator or date generator to create these
// parameters. For example, in the following URL
//
// https://payment.bhphotovideo.com/static/desktop/v2.0/
// index.html
// #paypageId=aLGNuLSTJVwgEiCn&cartID=333334444
// &receiverID=77777777-7777-4777-b777-777777888888
// &uuid=77777777-7777-4777-b777-778888888888
//
// The site created the parameters cartID, receiverID and uuid
// using random number generators. These parameters will have
// different values every time the browser loads the page.
// Therefore automation will not be able to identify an iframe
// that loads this URL.
let frameHostAndSchemeIsUnique = true;
for (let index = 0; index < details.length; index++) {
const url = new URL(details[index].url);
if (details[index].frameId !== targetFrame.frameId &&
targetFrameUrl.protocol === url.protocol &&
targetFrameUrl.host === url.host) {
frameHostAndSchemeIsUnique = false;
break;
}
}
if (frameHostAndSchemeIsUnique) {
context.browserTest = {
schemeAndHost:
`${targetFrameUrl.protocol}//${targetFrameUrl.host}`
};
resolve(context);
} else {
context.browserTest = { url: targetFrame.url };
resolve(context);
}
}
});
});
}
});
......@@ -354,8 +408,14 @@
function startRecordingOnTabAndFrame(tabId, frameId) {
const ret =
sendMessageToTab(tabId, { type: RecorderMsgEnum.START },
{ frameId: frameId })
getIframeContext(tabId, frameId)
.then((context) => {
return sendMessageToTab(tabId,
{ type: RecorderMsgEnum.START,
frameContext: context
},
{ frameId: frameId });
})
.then((response) => {
if (!response) {
return Promise.reject(
......@@ -546,9 +606,9 @@
.then((details) => {
let recordingStartedOnRootFramePromise;
details.forEach((frame) => {
// Th extension has no need and no permission to inject script
// into a blank page.
if (frame.url !== 'about:blank') {
// The extension has no need and no permission to inject script
// into 'about:' pages, such as the 'about:blank' page.
if (!frame.url.startsWith('about:')) {
const promise =
startRecordingOnTabAndFrame(tab.id, frame.frameId);
if (frame.frameId === 0) {
......@@ -596,10 +656,10 @@
chrome.webNavigation.onCompleted.addListener((details) => {
getRecordingTabId().then((tabId) => {
if (details.tabId === tabId &&
// Skip recording on about:blank. No meaningful user interaction will
// occur on a blank page. Plus, this extension has no permission to
// access about:blank.
details.url !== 'about:blank') {
// Skip recording on 'about:' pages. No meaningful user interaction
// occur on 'about:'' pages such as the blank page. Plus, this
// extension has no permission to access 'about:' pages.
!details.url.startsWith('about:')) {
startRecordingOnTabAndFrame(tabId, details.frameId)
.then(() => getRecordingState())
.then((state) => {
......
components/test/data/autofill/automated_integration/action_recorder_extension/constants.js
View file @ 856e676d
......@@ -31,9 +31,9 @@ const RecorderMsgEnum = {
START: 'start-recording',
STOP: 'stop-recording',
CANCEL: 'cancel-recording',
GET_FRAME_CONTEXT: 'get-frame-context',
GET_IFRAME_XPATH: 'get-iframe-xpath',
GET_IFRAME_NAME: 'get-iframe-name',
ADD_ACTION: 'record-action',
MEMORIZE_PASSWORD_FORM: 'memorize-password-form',
};
const Local_Storage_Vars = {
......
components/test/data/autofill/automated_integration/action_recorder_extension/content/action_handler.js
View file @ 856e676d
......@@ -33,12 +33,18 @@
// local name results in a unique XPath.
let nodeXPath = buildXPathForSingleNode(node);
let testXPath = `//${nodeXPath}${xPath}`;
if (countNumOfMatches(testXPath) === 1) {
let numMatches = countNumberOfMatches(testXPath);
if (numMatches === 1) {
return { isUnique: true, xPath: testXPath };
}
// Build a list of potential classifiers using
// * The element's explicit attributes.
// * The element's text content, if the element contains text.
let classifiers = [];
// Try adding each explicit attribute of the element to the XPath.
let attributes = [];
let attrRawValues = [];
// Populate the attribute list with the element's explicit attributes.
for (let index = 0; index < node.attributes.length; index++) {
const attr = node.attributes[index].name;
if (attr === 'style') {
......@@ -55,48 +61,45 @@
// attribute.
continue;
} else {
let classifier = buildClassifier(node, `@${attr}`,
node.attributes[index].value);
// Add the classifier and see if it generates a unique XPath.
classifiers.push(classifier);
nodeXPath = buildXPathForSingleNode(node, classifiers);
let testXPath = `//${nodeXPath}${xPath}`;
let numMatches = countNumOfMatches(testXPath);
if (numMatches === 0) {
// The classifier is faulty, log an error and remove the
// classifier.
console.warn('Encountered faulty classifier: ' +
classifiers.pop());
} else if (numMatches === 1) {
// The current XPath is unique, exit.
return { isUnique: true, xPath: testXPath };
}
attributes.push(`@${attr}`);
attrRawValues.push(node.attributes[index].value);
}
}
// For HTML elements that contains text content, try to construct
// an XPath using the text content.
// Add the element's text content to the attribute list.
switch (node.localName) {
case 'a':
case 'span':
case 'button':
let classifier = buildClassifier(node, 'text()',
node.textContent);
attributes.push('text()');
attrRawValues.push(node.textContent);
break;
default:
}
// Iterate through each attribute.
for (let index = 0; index < attributes.length; index++) {
let classifier = buildClassifier(node, attributes[index],
attrRawValues[index]);
// Add the classifier and see if adding it generates a unique XPath.
let numMatchesBefore = numMatches;
classifiers.push(classifier);
nodeXPath = buildXPathForSingleNode(node, classifiers);
let testXPath = `//${nodeXPath}${xPath}`;
let numMatches = countNumOfMatches(testXPath);
testXPath = `//${nodeXPath}${xPath}`;
numMatches = countNumberOfMatches(testXPath);
if (numMatches === 0) {
// The classifier is faulty, log an error and remove the
// classifier.
// The classifier is faulty, log an error and remove the classifier.
console.warn('Encountered faulty classifier: ' +
classifiers.pop());
} else if (numMatches === 1) {
// The current XPath is unique, exit.
return { isUnique: true, xPath: testXPath };
} else if (numMatches === numMatchesBefore) {
// Adding this classifier to the XPath does not narrow down the
// number of elements this XPath matches. Therefore the XPath does not
// need this classifier. Remove the classifier.
classifiers.pop();
}
break;
default:
}
// A XPath with the current node as the root is not unique.
......@@ -162,7 +165,7 @@
return xPathSelector;
}
function countNumOfMatches(xPath) {
function countNumberOfMatches(xPath) {
const queryResult = document.evaluate(
`count(${xPath})`, document, null, XPathResult.NUMBER_TYPE, null);
return queryResult.numberValue;
......@@ -172,6 +175,7 @@
};
let autofillTriggerElementSelector = null;
let lastTypingEventTargetValue = null;
let frameContext;
let mutationObserver = null;
let started = false;
......@@ -189,6 +193,10 @@
canTriggerAutofill(element);
}
function isPasswordInputElement(element) {
return element.getAttribute('type') === 'password';
}
function canTriggerAutofill(element) {
return (element.localName === 'input' &&
['checkbox', 'radio', 'button', 'submit', 'hidden', 'reset']
......@@ -219,46 +227,87 @@
});
}
function memorizePasswordFormInputs(passwordField) {
// Extract the 'form signature' value from the password's 'title'
// attribute.
const title = passwordField.getAttribute('title');
if (!title)
return;
let formSign = null;
const attrs = title.split('\n');
for (let index = 0; index < attrs.length; index++) {
if (attrs[index].startsWith('form signature')) {
formSign = attrs[index];
}
}
if (formSign)
return;
// Identify the 'user name' field and grab the field value.
const fields = document.querySelectorAll(`*[title*='${formSign}']`);
for (let index = 0; index < fields.length; index++) {
const field = fields[index];
const type = field.getAttribute('autofill-prediction');
if (type === 'HTML_TYPE_EMAIL') {
// (TODO: uwyiming@) Send the password to the UI content script. A
// user may then add password manager events through the Recorder
// Extension UI, without having to retype the password.
//sendRuntimeMessageToBackgroundScript({
// type: RecorderMsgEnum.MEMORIZE_PASSWORD_FORM,
// password: passwordField.value,
// user:
//});
}
}
}
function onInputChangeActionHandler(event) {
const selector = buildXPathForElement(event.target);
if (isAutofillableElement(event.target)) {
const elementReadyState = automation_helper.getElementState(event.target);
const autofillPrediction =
event.target.getAttribute('autofill-prediction');
// If a field that can trigger autofill has previously been
// clicked, fire a trigger autofill event.
if (autofillTriggerElementSelector !== null) {
console.log(`Triggered autofill on ${autofillTriggerElementSelector}`);
addActionToRecipe({
selector: selector,
context: frameContext,
type: 'autofill'
});
autofillTriggerElementSelector = null;
}
addActionToRecipe({
let action = {
selector: selector,
context: frameContext,
expectedAutofillType: autofillPrediction,
expectedValue: event.target.value,
type: 'validateField'
});
} else if (event.target.localName === 'select') {
visibility: elementReadyState
};
if (event.target.localName === 'select') {
if (document.activeElement === event.target) {
const index = event.target.options.selectedIndex;
console.log(`Select detected on: ${selector} with '${index}'`);
addActionToRecipe({
selector: selector,
context: frameContext,
index: index,
type: 'select'
});
action.type = 'select';
action.index = index;
addActionToRecipe(action);
} else {
action.type = 'validateField';
action.expectedValue = event.target.value;
if (autofillPrediction) {
action.expectedAutofillType = autofillPrediction;
}
addActionToRecipe(action);
}
} else if (lastTypingEventTargetValue === event.target.value) {
console.log(`Typing detected on: ${selector}`);
addActionToRecipe({
selector: selector,
context: frameContext,
value: event.target.value,
type: 'type'
});
action.type = 'type';
action.value = event.target.value;
addActionToRecipe(action);
} else {
// If the user has previously clicked on a field that can trigger
// autofill, add a trigger autofill action.
if (autofillTriggerElementSelector !== null) {
console.log(`Triggered autofill on ${autofillTriggerElementSelector}`);
action.type = 'autofill';
addActionToRecipe(action);
autofillTriggerElementSelector = null;
}
action.type = 'validateField';
action.expectedValue = event.target.value;
if (autofillPrediction) {
action.expectedAutofillType = autofillPrediction;
}
addActionToRecipe(action);
}
}
......@@ -287,6 +336,8 @@
// clicks on a scroll bar.
event.target.localName !== 'html') {
const element = event.target;
const elementReadyState =
automation_helper.getElementState(event.target);
const selector = buildXPathForElement(element);
console.log(`Left-click detected on: ${selector}`);
......@@ -300,6 +351,7 @@
} else {
addActionToRecipe({
selector: selector,
visibility: elementReadyState,
context: frameContext,
type: 'click'
});
......@@ -308,15 +360,26 @@
} else if (event.button === Buttons.RIGHT_BUTTON) {
const element = event.target;
const selector = buildXPathForElement(element);
const elementReadyState =
automation_helper.getElementState(event.target);
console.log(`Right-click detected on: ${selector}`);
addActionToRecipe({
selector: selector,
visibility: elementReadyState,
context: frameContext,
type: 'hover'
});
}
}
function onKeyUpActionHandler(event) {
if (isEditableInputElement(event.target)) {
lastTypingEventTargetValue = event.target.value;
} else {
lastTypingEventTargetValue = null;
}
}
function startRecording() {
const promise =
// First, obtain the current frame's context.
......@@ -327,7 +390,7 @@
frameContext = context;
// Register on change listeners on all the input elements.
registerOnInputChangeActionListener(document);
// Register a mouse up listener on the entire dom.
// Register a mouse up listener on the entire document.
//
// The content script registers a 'Mouse Up' listener rather than a
// 'Mouse Down' to correctly handle the following scenario:
......@@ -342,6 +405,8 @@
// To capture the correct sequence of actions, the content script
// should tie left mouse click actions to the mouseup event.
document.addEventListener('mouseup', onClickActionHander);
// Register a key press listener on the entire document.
document.addEventListener('keyup', onKeyUpActionHandler);
// Setup mutation observer to listen for event on nodes added after
// recording starts.
mutationObserver = new MutationObserver((mutations) => {
......@@ -367,19 +432,15 @@
if (started) {
mutationObserver.disconnect();
document.removeEventListener('mousedown', onClickActionHander);
document.removeEventListener('keyup', onKeyUpActionHandler);
deRegisterOnInputChangeActionListener(document);
}
}
function queryIframeContext(frameId, frameLocation) {
// Check if we cached the XPath for this iframe.
if (iframeContextMap[frameId]) {
return Promise.resolve(iframeContextMap[frameId]);
}
function queryIframeName(iframeUrl) {
let iframe = null;
const frameLocation = new URL(iframeUrl);
const iframes = document.querySelectorAll('iframe');
let numIframeWithSameSchemeAndHost = 0;
// Find the target iframe.
for (let index = 0; index < iframes.length; index++) {
const url = new URL(iframes[index].src,
......@@ -402,51 +463,18 @@
frameLocation.search === url.search) {
iframe = iframes[index];
}
// Count the number of iframes with the same protocol and the same host.
if (frameLocation.protocol === url.protocol &&
frameLocation.host === url.host) {
numIframeWithSameSchemeAndHost++;
}
}
if (iframe === null) {
return Promise.reject(
new Error('Unable to find iframe with url ' +
`'${frameLocation.href}', for frame ${frameId}`));
new Error(`Unable to find iframe with url '${iframeUrl}'!`));
}
let context = { isIframe: true, browser_test: {} };
if (iframe.name) {
context.browser_test.name = iframe.name;
}
else if (numIframeWithSameSchemeAndHost === 1) {
// Register the iframe's scheme, host and port.
// The Captured Site automation framework can identify an iframe by its
// scheme + host + port, provided this information combination is unique.
// Identifying an iframe through its scheme + host + port is preferable
// than identityfing an iframe through its URL. An URL will frequently
// contain parameters, and many websites use random number generator
// or date generator to create these parameters. For example, in the
// following URL
//
// https://payment.bhphotovideo.com/static/desktop/v2.0/index.html
// #paypageId=aLGNuLSTJVwgEiCn&cartID=333334444
// &receiverID=77777777-7777-4777-b777-777777888888
// &uuid=77777777-7777-4777-b777-778888888888
//
// The site created the parameters cartID, receiverID and uuid using
// random number generators. These parameters will have different values
// every time the browser loads the page. Therefore automation will not
// be able to identify an iframe that loads this URL.
context.browser_test.schemeAndHost =
`${frameLocation.protocol}//${frameLocation.host}`;
return Promise.resolve(iframe.name);
} else {
context.browser_test.url = frameLocation.href;
return Promise.resolve('');
}
iframeContextMap[frameId] = context;
return Promise.resolve(context);
}
chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
......@@ -466,8 +494,8 @@
stopRecording();
sendResponse(true);
break;
case RecorderMsgEnum.GET_IFRAME_XPATH:
queryIframeContext(request.frameId, request.location)
case RecorderMsgEnum.GET_IFRAME_NAME:
queryIframeName(request.url)
.then((context) => {
sendResponse(context);
}).catch((error) => {
......@@ -482,4 +510,57 @@
// to persist.
return false;
});
if (typeof automation_helper === 'undefined') {
var automation_helper = {
// An enum specifying the state of an element on the page.
DomElementReadyState:
Object.freeze({
"present": 0,
"visible": 1 << 0,
"enabled": 1 << 1,
"on_top": 1 << 2,
}),
};
automation_helper.getElementState = function(element) {
let state_flags = this.DomElementReadyState.present;
// Check if the element is disabled.
if (!element.disabled) {
state_flags |= this.DomElementReadyState.enabled;
}
// Check if the element is visible.
if (element.offsetParent !== null &&
element.offsetWidth > 0 &&
element.offsetHeight > 0) {
state_flags |= this.DomElementReadyState.visible;
// Check if the element is also on top.
const rect = element.getBoundingClientRect();
const topElement = document.elementFromPoint(
// As coordinates, use the center of the element, minus
// the window offset in case the element is outside the
// view.
rect.left + rect.width / 2 - window.pageXOffset,
rect.top + rect.height / 2 - window.pageYOffset);
if (isSelfOrDescendant(element, topElement)) {
state_flags |= this.DomElementReadyState.on_top;
}
}
return state_flags;
}
function isSelfOrDescendant(parent, child) {
var node = child;
while (node != null) {
if (node == parent) {
return true;
}
node = node.parentNode;
}
return false;
}
return automation_helper;
}
})();
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment