diff --git a/firefox-ios/Client/Assets/CC_Script/FieldScanner.sys.mjs b/firefox-ios/Client/Assets/CC_Script/FieldScanner.sys.mjs index 85f771dd0c01..7d6a87d255cc 100644 --- a/firefox-ios/Client/Assets/CC_Script/FieldScanner.sys.mjs +++ b/firefox-ios/Client/Assets/CC_Script/FieldScanner.sys.mjs @@ -6,6 +6,7 @@ const lazy = {}; ChromeUtils.defineESModuleGetters(lazy, { FormAutofill: "resource://autofill/FormAutofill.sys.mjs", FormAutofillUtils: "resource://gre/modules/shared/FormAutofillUtils.sys.mjs", + MLAutofill: "resource://autofill/MLAutofill.sys.mjs", }); /** @@ -98,6 +99,8 @@ export class FieldDetail { fathomLabel = null, fathomConfidence = null, isVisible = true, + mlHeaderInput = null, + mlButtonInput = null, } = {} ) { const fieldDetail = new FieldDetail(element); @@ -169,7 +172,9 @@ export class FieldDetail { lazy.FormAutofill.isMLExperimentEnabled && ["input", "select"].includes(element.localName) ) { - fieldDetail.htmlMarkup = element.outerHTML.substring(0, 1024); + fieldDetail.mlinput = lazy.MLAutofill.getMLMarkup(fieldDetail.element); + fieldDetail.mlHeaderInput = mlHeaderInput; + fieldDetail.mlButtonInput = mlButtonInput; fieldDetail.fathomLabel = fathomLabel; fieldDetail.fathomConfidence = fathomConfidence; } diff --git a/firefox-ios/Client/Assets/CC_Script/FormAutofill.sys.mjs b/firefox-ios/Client/Assets/CC_Script/FormAutofill.sys.mjs index 82889760649a..e55ee9e77c01 100644 --- a/firefox-ios/Client/Assets/CC_Script/FormAutofill.sys.mjs +++ b/firefox-ios/Client/Assets/CC_Script/FormAutofill.sys.mjs @@ -317,6 +317,13 @@ XPCOMUtils.defineLazyPreferenceGetter( false ); +XPCOMUtils.defineLazyPreferenceGetter( + FormAutofill, + "MLModelRevision", + "extensions.formautofill.ml.experiment.modelRevision", + null +); + ChromeUtils.defineLazyGetter(FormAutofill, "countries", () => AddressMetaDataLoader.getCountries() ); diff --git a/firefox-ios/Client/Assets/CC_Script/FormAutofillChild.ios.sys.mjs b/firefox-ios/Client/Assets/CC_Script/FormAutofillChild.ios.sys.mjs index b0a956a19304..f094b0fe96a1 100644 --- a/firefox-ios/Client/Assets/CC_Script/FormAutofillChild.ios.sys.mjs +++ b/firefox-ios/Client/Assets/CC_Script/FormAutofillChild.ios.sys.mjs @@ -71,7 +71,7 @@ export class FormAutofillChild { if (!handler.hasIdentifiedFields() || handler.updateFormIfNeeded(element)) { // If we found newly identified fields, run section classification heuristic - const detectedFields = FormAutofillHandler.collectFormFields( + const detectedFields = FormAutofillHandler.collectFormFieldDetails( handler.form ); diff --git a/firefox-ios/Client/Assets/CC_Script/FormAutofillHandler.sys.mjs b/firefox-ios/Client/Assets/CC_Script/FormAutofillHandler.sys.mjs index a7aec3ac1f64..1989b41a94ff 100644 --- a/firefox-ios/Client/Assets/CC_Script/FormAutofillHandler.sys.mjs +++ b/firefox-ios/Client/Assets/CC_Script/FormAutofillHandler.sys.mjs @@ -217,20 +217,38 @@ export class FormAutofillHandler { * Collect , fields. * * @returns {Array} * An array containing eliglble fields for autofill, also * including iframe. */ - static collectFormFields(form) { - const fieldDetails = lazy.FormAutofillHeuristics.getFormInfo(form) ?? []; + static collectFormFieldDetails( + formLike, + includeIframe, + ignoreInvisibleInput = true + ) { + const fieldDetails = + lazy.FormAutofillHeuristics.getFormInfo(formLike, ignoreInvisibleInput) ?? + []; + + // 'FormLike' only contains & fields. * @returns {Array} * all sections within its field details in the form. */ - getFormInfo(form) { - const elements = Array.from(form.elements).filter(element => + getFormInfo(formLike, ignoreInvisibleInput) { + const elements = Array.from(formLike.elements).filter(element => lazy.FormAutofillUtils.isCreditCardOrAddressFieldType(element) ); + let closestHeaders; + let closestButtons; + if (FormAutofill.isMLExperimentEnabled && elements.length) { + closestHeaders = lazy.MLAutofill.closestHeaderAbove(elements); + closestButtons = lazy.MLAutofill.closestButtonBelow(elements); + } + const fieldDetails = []; - for (const element of elements) { + for (let idx = 0; idx < elements.length; idx++) { + const element = elements[idx]; // Ignore invisible , we still keep invisible // to store the value. const isVisible = lazy.FormAutofillUtils.isFieldVisible(element); - if (!HTMLSelectElement.isInstance(element) && !isVisible) { + if ( + !HTMLSelectElement.isInstance(element) && + !isVisible && + ignoreInvisibleInput + ) { continue; } @@ -742,11 +757,13 @@ export const FormAutofillHeuristics = { } fieldDetails.push( - lazy.FieldDetail.create(element, form, fieldName, { + lazy.FieldDetail.create(element, formLike, fieldName, { autocompleteInfo: inferInfo.autocompleteInfo, fathomLabel: inferInfo.fathomLabel, fathomConfidence: inferInfo.fathomConfidence, isVisible, + mlHeaderInput: closestHeaders[idx], + mlButtonInput: closestButtons[idx], }) ); } diff --git a/firefox-ios/Client/Assets/CC_Script/FormAutofillSection.sys.mjs b/firefox-ios/Client/Assets/CC_Script/FormAutofillSection.sys.mjs index c6aa40a773f7..84f54648304c 100644 --- a/firefox-ios/Client/Assets/CC_Script/FormAutofillSection.sys.mjs +++ b/firefox-ios/Client/Assets/CC_Script/FormAutofillSection.sys.mjs @@ -25,14 +25,19 @@ class FormSection { fieldDetails.forEach(field => this.addField(field)); - const fieldName = fieldDetails[0].fieldName; - if (lazy.FormAutofillUtils.isAddressField(fieldName)) { - this.type = FormSection.ADDRESS; - } else if (lazy.FormAutofillUtils.isCreditCardField(fieldName)) { - this.type = FormSection.CREDIT_CARD; - } else { - throw new Error("Unknown field type to create a section."); + for (const fieldDetail of fieldDetails) { + if (lazy.FormAutofillUtils.isAddressField(fieldDetail.fieldName)) { + this.type = FormSection.ADDRESS; + break; + } else if ( + lazy.FormAutofillUtils.isCreditCardField(fieldDetail.fieldName) + ) { + this.type = FormSection.CREDIT_CARD; + break; + } } + + this.type ||= FormSection.ADDRESS; } get fieldDetails() { @@ -140,21 +145,40 @@ export class FormAutofillSection { * The result is an array contains the sections with its belonging field details. * * @param {Array} fieldDetails field detail array to be classified - * @param {boolean} ignoreInvalid - * True to keep invalid section in the return array. Only used by tests now. + * @param {object} options + * @param {boolean} [options.ignoreInvalidSection = false] + * True to keep invalid section in the return array. Only used by tests now + * @param {boolean} [options.ignoreUnknownField = true] + * False to keep unknown field in a section. Only used by developer tools now * @returns {Array} The array with the sections. */ - static classifySections(fieldDetails, ignoreInvalid = false) { - const addressSections = FormAutofillSection.groupFields( - fieldDetails.filter(f => - lazy.FormAutofillUtils.isAddressField(f.fieldName) - ) - ); - const creditCardSections = FormAutofillSection.groupFields( - fieldDetails.filter(f => - lazy.FormAutofillUtils.isCreditCardField(f.fieldName) - ) - ); + static classifySections( + fieldDetails, + { ignoreInvalidSection = false, ignoreUnknownField = true } = {} + ) { + const addressFields = []; + const creditCardFields = []; + + // 'current' refers to the last list where an field was added to. + // It helps determine the appropriate list for unknown fields, defaulting to the address + // field list for simplicity + let current = addressFields; + for (const fieldDetail of fieldDetails) { + if (lazy.FormAutofillUtils.isAddressField(fieldDetail.fieldName)) { + current = addressFields; + } else if ( + lazy.FormAutofillUtils.isCreditCardField(fieldDetail.fieldName) + ) { + current = creditCardFields; + } else if (ignoreUnknownField) { + continue; + } + current.push(fieldDetail); + } + + const addressSections = FormAutofillSection.groupFields(addressFields); + const creditCardSections = + FormAutofillSection.groupFields(creditCardFields); const sections = [...addressSections, ...creditCardSections].sort( (a, b) => @@ -173,7 +197,7 @@ export class FormAutofillSection { ? new FormAutofillAddressSection(section.fieldDetails) : new FormAutofillCreditCardSection(section.fieldDetails); - if (ignoreInvalid && !autofillableSection.isValidSection()) { + if (ignoreInvalidSection && !autofillableSection.isValidSection()) { continue; }