mozilla-mobile · github-actions · Jan 10, 2025
@@ -6,6 +6,7 @@ const lazy = {};
 ChromeUtils.defineESModuleGetters(lazy, {
   FormAutofill: "resource://autofill/FormAutofill.sys.mjs",
   FormAutofillUtils: "resource://gre/modules/shared/FormAutofillUtils.sys.mjs",
+  MLAutofill: "resource://autofill/MLAutofill.sys.mjs",
 });
 
 /**
@@ -98,6 +99,8 @@ export class FieldDetail {
       fathomLabel = null,
       fathomConfidence = null,
       isVisible = true,
+      mlHeaderInput = null,
+      mlButtonInput = null,
     } = {}
   ) {
     const fieldDetail = new FieldDetail(element);
@@ -169,7 +172,9 @@ export class FieldDetail {
       lazy.FormAutofill.isMLExperimentEnabled &&
       ["input", "select"].includes(element.localName)
     ) {
-      fieldDetail.htmlMarkup = element.outerHTML.substring(0, 1024);
+      fieldDetail.mlinput = lazy.MLAutofill.getMLMarkup(fieldDetail.element);
+      fieldDetail.mlHeaderInput = mlHeaderInput;
+      fieldDetail.mlButtonInput = mlButtonInput;
       fieldDetail.fathomLabel = fathomLabel;
       fieldDetail.fathomConfidence = fathomConfidence;
     }

@@ -83,10 +83,22 @@ export const FormAutofill = {
     }
     return false;
   },
+
+  /**
+   * Return true if address autofill is available for a specific country.
+   */
   isAutofillAddressesAvailableInCountry(country) {
-    return FormAutofill._addressAutofillSupportedCountries.includes(
-      country.toUpperCase()
-    );
+    if (FormAutofill._isAutofillAddressesAvailableInExperiment) {
+      return true;
+    }
+
+    let available = FormAutofill._isAutofillAddressesAvailable;
+    if (country && available == "detect") {
+      return FormAutofill._addressAutofillSupportedCountries.includes(
+        country.toUpperCase()
+      );
+    }
+    return available == "on";
   },
   get isAutofillEnabled() {
     return this.isAutofillAddressesEnabled || this.isAutofillCreditCardsEnabled;
@@ -317,6 +329,13 @@ XPCOMUtils.defineLazyPreferenceGetter(
   false
 );
 
+XPCOMUtils.defineLazyPreferenceGetter(
+  FormAutofill,
+  "MLModelRevision",
+  "extensions.formautofill.ml.experiment.modelRevision",
+  null
+);
+
 ChromeUtils.defineLazyGetter(FormAutofill, "countries", () =>
   AddressMetaDataLoader.getCountries()
 );
@@ -71,7 +71,7 @@ export class FormAutofillChild {
 
     if (!handler.hasIdentifiedFields() || handler.updateFormIfNeeded(element)) {
       // If we found newly identified fields, run section classification heuristic
-      const detectedFields = FormAutofillHandler.collectFormFields(
+      const detectedFields = FormAutofillHandler.collectFormFieldDetails(
         handler.form
       );
 

@@ -217,20 +217,38 @@ export class FormAutofillHandler {
    * Collect <input>, <select>, and <iframe> elements from the specified form
    * and return the correspond 'FieldDetail' objects.
    *
-   * @param {HTMLFormElement} form
+   * @param {formLike} formLike
    *        The form that we collect information from.
+   * @param {boolean} includeIframe
+   *        True to add <iframe> to the returned FieldDetails array.
+   * @param {boolean} ignoreInvisibleInput
+   *        True to NOT run heuristics on invisible <input> fields.
    *
    * @returns {Array<FieldDeail>}
    *        An array containing eliglble fields for autofill, also
    *        including iframe.
    */
-  static collectFormFields(form) {
-    const fieldDetails = lazy.FormAutofillHeuristics.getFormInfo(form) ?? [];
+  static collectFormFieldDetails(
+    formLike,
+    includeIframe,
+    ignoreInvisibleInput = true
+  ) {
+    const fieldDetails =
+      lazy.FormAutofillHeuristics.getFormInfo(formLike, ignoreInvisibleInput) ??
+      [];
+
+    // 'FormLike' only contains <input> & <select>, so in order to include <iframe>
+    // in the list of 'FieldDetails', we need to search for <iframe> in the form.
+    if (!includeIframe) {
+      return fieldDetails;
+    }
 
-    let index = 0;
+    // Insert <iframe> elements into the fieldDetails array, maintaining the element order.
     const fieldDetailsIncludeIframe = [];
-    const elements = form.rootElement.querySelectorAll("input, select, iframe");
-
+    let index = 0;
+    const elements = formLike.rootElement.querySelectorAll(
+      "input, select, iframe"
+    );
     for (const element of elements) {
       if (fieldDetails[index]?.element == element) {
         fieldDetailsIncludeIframe.push(fieldDetails[index]);
@@ -239,8 +257,17 @@ export class FormAutofillHandler {
         element.localName == "iframe" &&
         FormAutofillUtils.isFieldVisible(element)
       ) {
-        const iframeFd = lazy.FieldDetail.create(element, form, "iframe");
-        fieldDetailsIncludeIframe.push(iframeFd);
+        // Add the <iframe> only if it is under the `formLike` element.
+        // While we use formLike.rootElement.querySelectorAll, it is still possible
+        // we find an <iframe> inside a <form> within this rootElement. In this
+        // case, we don't want to include the <iframe> in the field list.
+        if (
+          lazy.AutofillFormFactory.findRootForField(element) ==
+          formLike.rootElement
+        ) {
+          const iframeFd = lazy.FieldDetail.create(element, formLike, "iframe");
+          fieldDetailsIncludeIframe.push(iframeFd);
+        }
       }
     }
     return fieldDetailsIncludeIframe;

@@ -13,6 +13,7 @@ ChromeUtils.defineESModuleGetters(lazy, {
   FieldScanner: "resource://gre/modules/shared/FieldScanner.sys.mjs",
   FormAutofillUtils: "resource://gre/modules/shared/FormAutofillUtils.sys.mjs",
   LabelUtils: "resource://gre/modules/shared/LabelUtils.sys.mjs",
+  MLAutofill: "resource://autofill/MLAutofill.sys.mjs",
 });
 
 /**
@@ -704,23 +705,37 @@ export const FormAutofillHeuristics = {
    * in the belonging section. The details contain the autocomplete info
    * (e.g. fieldName, section, etc).
    *
-   * @param {HTMLFormElement} form
+   * @param {formLike} formLike
    *        the elements in this form to be predicted the field info.
+   * @param {boolean} ignoreInvisibleInput
+   *        True to NOT run heuristics on invisible <input> fields.
    * @returns {Array<FormSection>}
    *        all sections within its field details in the form.
    */
-  getFormInfo(form) {
-    const elements = Array.from(form.elements).filter(element =>
+  getFormInfo(formLike, ignoreInvisibleInput) {
+    const elements = Array.from(formLike.elements).filter(element =>
       lazy.FormAutofillUtils.isCreditCardOrAddressFieldType(element)
     );
 
+    let closestHeaders;
+    let closestButtons;
+    if (FormAutofill.isMLExperimentEnabled && elements.length) {
+      closestHeaders = lazy.MLAutofill.closestHeaderAbove(elements);
+      closestButtons = lazy.MLAutofill.closestButtonBelow(elements);
+    }
+
     const fieldDetails = [];
-    for (const element of elements) {
+    for (let idx = 0; idx < elements.length; idx++) {
+      const element = elements[idx];
       // Ignore invisible <input>, we still keep invisible <select> since
       // some websites implements their custom dropdown and use invisible <select>
       // to store the value.
       const isVisible = lazy.FormAutofillUtils.isFieldVisible(element);
-      if (!HTMLSelectElement.isInstance(element) && !isVisible) {
+      if (
+        !HTMLSelectElement.isInstance(element) &&
+        !isVisible &&
+        ignoreInvisibleInput
+      ) {
         continue;
       }
 
@@ -742,11 +757,13 @@ export const FormAutofillHeuristics = {
       }
 
       fieldDetails.push(
-        lazy.FieldDetail.create(element, form, fieldName, {
+        lazy.FieldDetail.create(element, formLike, fieldName, {
           autocompleteInfo: inferInfo.autocompleteInfo,
           fathomLabel: inferInfo.fathomLabel,
           fathomConfidence: inferInfo.fathomConfidence,
           isVisible,
+          mlHeaderInput: closestHeaders?.[idx] ?? null,
+          mlButtonInput: closestButtons?.[idx] ?? null,
         })
       );
     }

@@ -25,14 +25,19 @@ class FormSection {
 
     fieldDetails.forEach(field => this.addField(field));
 
-    const fieldName = fieldDetails[0].fieldName;
-    if (lazy.FormAutofillUtils.isAddressField(fieldName)) {
-      this.type = FormSection.ADDRESS;
-    } else if (lazy.FormAutofillUtils.isCreditCardField(fieldName)) {
-      this.type = FormSection.CREDIT_CARD;
-    } else {
-      throw new Error("Unknown field type to create a section.");
+    for (const fieldDetail of fieldDetails) {
+      if (lazy.FormAutofillUtils.isAddressField(fieldDetail.fieldName)) {
+        this.type = FormSection.ADDRESS;
+        break;
+      } else if (
+        lazy.FormAutofillUtils.isCreditCardField(fieldDetail.fieldName)
+      ) {
+        this.type = FormSection.CREDIT_CARD;
+        break;
+      }
     }
+
+    this.type ||= FormSection.ADDRESS;
   }
 
   get fieldDetails() {
@@ -140,21 +145,40 @@ export class FormAutofillSection {
    * The result is an array contains the sections with its belonging field details.
    *
    * @param   {Array<FieldDetails>} fieldDetails field detail array to be classified
-   * @param   {boolean} ignoreInvalid
-   *          True to keep invalid section in the return array. Only used by tests now.
+   * @param   {object} options
+   * @param   {boolean} [options.ignoreInvalidSection = false]
+   *          True to keep invalid section in the return array. Only used by tests now
+   * @param   {boolean} [options.ignoreUnknownField = true]
+   *          False to keep unknown field in a section. Only used by developer tools now
    * @returns {Array<FormSection>} The array with the sections.
    */
-  static classifySections(fieldDetails, ignoreInvalid = false) {
-    const addressSections = FormAutofillSection.groupFields(
-      fieldDetails.filter(f =>
-        lazy.FormAutofillUtils.isAddressField(f.fieldName)
-      )
-    );
-    const creditCardSections = FormAutofillSection.groupFields(
-      fieldDetails.filter(f =>
-        lazy.FormAutofillUtils.isCreditCardField(f.fieldName)
-      )
-    );
+  static classifySections(
+    fieldDetails,
+    { ignoreInvalidSection = false, ignoreUnknownField = true } = {}
+  ) {
+    const addressFields = [];
+    const creditCardFields = [];
+
+    // 'current' refers to the last list where an field was added to.
+    // It helps determine the appropriate list for unknown fields, defaulting to the address
+    // field list for simplicity
+    let current = addressFields;
+    for (const fieldDetail of fieldDetails) {
+      if (lazy.FormAutofillUtils.isAddressField(fieldDetail.fieldName)) {
+        current = addressFields;
+      } else if (
+        lazy.FormAutofillUtils.isCreditCardField(fieldDetail.fieldName)
+      ) {
+        current = creditCardFields;
+      } else if (ignoreUnknownField) {
+        continue;
+      }
+      current.push(fieldDetail);
+    }
+
+    const addressSections = FormAutofillSection.groupFields(addressFields);
+    const creditCardSections =
+      FormAutofillSection.groupFields(creditCardFields);
 
     const sections = [...addressSections, ...creditCardSections].sort(
       (a, b) =>
@@ -173,7 +197,7 @@ export class FormAutofillSection {
           ? new FormAutofillAddressSection(section.fieldDetails)
           : new FormAutofillCreditCardSection(section.fieldDetails);
 
-      if (ignoreInvalid && !autofillableSection.isValidSection()) {
+      if (ignoreInvalidSection && !autofillableSection.isValidSection()) {
         continue;
       }
 
@@ -420,10 +444,7 @@ export class FormAutofillAddressSection extends FormAutofillSection {
     const country = lazy.FormAutofillUtils.identifyCountryCode(
       record.country || record["country-name"]
     );
-    if (
-      country &&
-      !lazy.FormAutofill.isAutofillAddressesAvailableInCountry(country)
-    ) {
+    if (!lazy.FormAutofill.isAutofillAddressesAvailableInCountry(country)) {
       // We don't want to save data in the wrong fields due to not having proper
       // heuristic regexes in countries we don't yet support.
       this.log.warn(