use time step embedding from file (#928)

* feat: pass task-specific config to backend (#922)

* Add TaskConfig.CustomConfig and pass them to backend

* Add CustomConfig for main.cc

* Use seed and num_steps from CustomConfig for TFLite backend

* Replace std::cout with LOG(INFO)

* Format files

* feat: add ConvertOutputs() API (#927)

* Add ConvertOutputs() API

* Add ConvertOutputs() for mobile_back_tflite

* Set minimum macos version

* Set minimum macos version to 13.1

* Update _kIphoneOnGitHubAction

* feat: timestamp-embedding-parser (WIP)

* disabled bitcode to be able compile with new XCode

* chore: formatting

* refactor: use custom setting in Core ML backend to detect NCHW input. (#924)

* Add GetConfigValue()

* Add custom setting data-format for Core ML

* Use GetConfigValue() to get stable_diffusion_seed and stable_diffusion_num_steps

* fix: resolve crash due to permission denied on Android Play Store version (#930)

* Set android:extractNativeLibs="true"

* Set android.bundle.enableUncompressedNativeLibs=false

* chore: increase Android minSdkVersion from 21 to 30 (#859)

Increase minSdkVersion to 30

* feat: finalized SD pipeline to use embedding from the binary file.

* refactor: updated embedding_utils to parse pkl file

* chore: linting

* fix: fixed lint issue in neuron

* chore: BUILD cleanup

* chore: cleanup

* chore: ignore .fvm

* chore: updated model paths and checksums for stable diffusion benchmark: tflite_settings_android.pbtxt

* chore: reverse timesteps and embeddings to support descending order of the timesteps and embeddings

* chore: fixed formatting

* chore: added links to the sd models and timestep embeddings file

* chore: add the proper name for the embedding_timesteps file

* chore: added missed declaration for backend_convert_outputs

* chore: clang formatting

* chore: added missed files

* chore: fixed build file for the pixel backend

* chore: bazel formatting

* fix: added missed interface implementation for pixel

* chore: clang formatting

---------

Co-authored-by: Anh <[email protected]>

.bazelrc

@@ -43,7 +43,6 @@ build:android_x86_64 --fat_apk_cpu=x86_64
 
 # iOS configs
 build:ios --apple_platform_type=ios
-build:ios --apple_bitcode=embedded --copt=-fembed-bitcode
 build:ios --copt=-Wno-c++11-narrowing
 build:ios --cxxopt=-fobjc-arc
 

.gitignore

@@ -4,6 +4,7 @@
 .ijwb
 .idea
 .vscode
+.fvm
 /bazel-*
 /output
 /output_logs

WORKSPACE

@@ -1,7 +1,7 @@
 workspace(name = "mlperf_app")
 
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 http_archive(
     name = "bazel_skylib",
@@ -49,11 +49,11 @@ http_archive(
     ],
 )
 
-load("@rules_python//python:repositories.bzl", "python_register_toolchains")
 load(
     "@org_tensorflow//tensorflow/tools/toolchains/python:python_repo.bzl",
     "python_repository",
 )
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
 
 python_repository(name = "python_version_repo")
 

flutter/android/app/build.gradle

@@ -42,7 +42,7 @@ android {
 
     defaultConfig {
         applicationId "org.mlcommons.android.mlperfbench"
-        minSdkVersion 21
+        minSdkVersion 30
         targetSdkVersion 34
         versionCode flutterVersionCode.toInteger()
         versionName flutterVersionName

flutter/android/app/src/main/AndroidManifest.xml

@@ -11,7 +11,8 @@
                    android:maxSdkVersion="29"/>
 
   <application android:label="MLPerf Mobile"
-               android:icon="@mipmap/ic_launcher">
+               android:icon="@mipmap/ic_launcher"
+               android:extractNativeLibs="true">
 
     <!-- TFLite -->
     <uses-library android:name="libOpenCL.so"

flutter/android/gradle.properties

@@ -1,3 +1,4 @@
 org.gradle.jvmargs=-Xmx1536M
 android.useAndroidX=true
 android.enableJetifier=true
+android.bundle.enableUncompressedNativeLibs=false

flutter/assets/tasks.pbtxt

@@ -247,4 +247,12 @@ task {
     id: "stable_diffusion"
     name: "StableDiffusion"
   }
+  custom_config {
+    id: "stable_diffusion_seed"
+    value: "633994880"
+  }
+  custom_config {
+    id: "stable_diffusion_num_steps"
+    value: "20"
+  }
 }

flutter/cpp/backend.h

@@ -74,6 +74,10 @@ class Backend {
   virtual void ConvertInputs(int bytes, int image_width, int image_height,
                              uint8_t* data) = 0;
 
+  // Allow backend to do output layout change
+  virtual void ConvertOutputs(int bytes, int image_width, int image_height,
+                              uint8_t* data) = 0;
+
  private:
   BackendSetting settings_;
 };

flutter/cpp/backends/external.cc

@@ -187,7 +187,9 @@ BackendFunctions::BackendFunctions(const std::string& lib_path) {
   // Backends may need to change the format of the inputs (e.g. channel order)
   convert_inputs = reinterpret_cast<decltype(convert_inputs)>(
       CheckSymbol("mlperf_backend_convert_inputs"));
-
+  // Backends may need to change the format of the outputs (e.g. channel order)
+  convert_outputs = reinterpret_cast<decltype(convert_outputs)>(
+      CheckSymbol("mlperf_backend_convert_outputs"));
   // If both functions are defined, then update
   if (get_buffer && release_buffer) {
     LOG(INFO) << "Using backend allocator";

flutter/cpp/backends/external.h

@@ -67,6 +67,8 @@ struct BackendFunctions {
       mlperf_backend_ptr_t, uint32_t, int32_t, void**)>::type;
   using ConvertInputsPtr = std::add_pointer<void(mlperf_backend_ptr_t, int, int,
                                                  int, uint8_t*)>::type;
+  using ConvertOutputsPtr = std::add_pointer<void(mlperf_backend_ptr_t, int,
+                                                  int, int, uint8_t*)>::type;
 
   // Required functions.
   BackendMatchesPtr match{nullptr};
@@ -91,6 +93,7 @@ struct BackendFunctions {
   AllocatorMgr::GetBufferFn get_buffer{nullptr};
   AllocatorMgr::ReleaseBufferFn release_buffer{nullptr};
   ConvertInputsPtr convert_inputs{nullptr};
+  ConvertOutputsPtr convert_outputs{nullptr};
 
   bool isLoaded() { return isloaded; }
 
@@ -210,6 +213,15 @@ class ExternalBackend : public Backend {
     }
   }
 
+  // Optional function to do output data re-formatting
+  void ConvertOutputs(int bytes, int width, int height,
+                      uint8_t* data) override {
+    if (backend_functions_.convert_outputs) {
+      backend_functions_.convert_outputs(backend_ptr_, bytes, width, height,
+                                         data);
+    }
+  }
+
  private:
   std::string backend_name_;
   std::string vendor_;

flutter/cpp/binary/main.cc

@@ -132,7 +132,7 @@ int Main(int argc, char *argv[]) {
   command_line += " " + backend_name + " " + benchmark_id;
 
   // Command Line Flags for mlperf.
-  std::string mode, scenario = "SingleStream", output_dir;
+  std::string mode, scenario = "SingleStream", output_dir, custom_config;
   int min_query_count = 100, min_duration_ms = 100,
       max_duration_ms = 10 * 60 * 1000,
       single_stream_expected_latency_ns = 1000000;
@@ -157,8 +157,9 @@ int Main(int argc, char *argv[]) {
                         "A hint used by the loadgen to pre-generate "
                         "enough samples to meet the minimum test duration."),
        Flag::CreateFlag("output_dir", &output_dir,
-                        "The output directory of mlperf.", Flag::kRequired)});
-
+                        "The output directory of mlperf.", Flag::kRequired),
+       Flag::CreateFlag("custom_config", &custom_config,
+                        "Custom config in form key1:val1,key2:val2.")});
   // Command Line Flags for backend.
   std::unique_ptr<Backend> backend;
   std::unique_ptr<Dataset> dataset;
@@ -207,9 +208,8 @@ int Main(int argc, char *argv[]) {
             }
           }
         }
-
         SettingList setting_list =
-            createSettingList(backend_setting, benchmark_id);
+            CreateSettingList(backend_setting, custom_config, benchmark_id);
 
         ExternalBackend *external_backend = new ExternalBackend(
             model_file_path, lib_path, setting_list, native_lib_path);

flutter/cpp/c/backend_c.h

@@ -82,6 +82,8 @@ mlperf_status_t mlperf_backend_get_output(mlperf_backend_ptr_t backend_ptr,
 // Optional functions
 void mlperf_backend_convert_inputs(mlperf_backend_ptr_t backend_ptr, int bytes,
                                    int width, int height, uint8_t* data);
+void mlperf_backend_convert_outputs(mlperf_backend_ptr_t backend_ptr, int bytes,
+                                    int width, int height, uint8_t* data);
 
 #ifdef __cplusplus
 }

flutter/cpp/flutter/BUILD

@@ -55,6 +55,7 @@ apple_xcframework(
     },
     minimum_os_versions = {
         "ios": "13.1",
+        "macos": "13.1",
     },
     deps = [
         "//flutter/cpp/flutter:bridge",

flutter/cpp/proto/mlperf_task.proto

@@ -31,7 +31,7 @@ message MLPerfConfig {
 // Config of the mlperf tasks.
 // A task is basically a combination of models and a dataset.
 //
-// Next ID: 11
+// Next ID: 12
 message TaskConfig {
   // Must be unique in one task file. Ex: image_classification
   // used to match backend settings
@@ -52,6 +52,7 @@ message TaskConfig {
   required string scenario = 7;
   required DatasetConfig datasets = 8;
   required ModelConfig model = 9;
+  repeated CustomConfig custom_config = 11;
 }
 
 // Datasets for a task
@@ -107,3 +108,14 @@ message ModelConfig {
   // Number of detection classes if applicable
   optional int32 num_classes = 6;
 }
+
+// CustomConfig are task specific configuration.
+// The TaskConfig.CustomConfig will be converted to
+// BenchmarkSetting.CustomSetting and passed to the backend.
+// To avoid name collision, the id should be prefixed with TaskConfig.id.
+message CustomConfig {
+  // Id of this config.
+  required string id = 1;
+  // Value of this config.
+  required string value = 2;
+}

flutter/cpp/proto/test.cc

@@ -139,9 +139,11 @@ int test_proto() {
   std::list<std::string> benchmarks;
   benchmarks.push_back("image_classification");
   benchmarks.push_back("image_classification_offline");
+  std::string custom_config = "key1:val1,key2:val2";
   for (auto benchmark_id : benchmarks) {
     // Convert to SettingList
-    SettingList setting_list = createSettingList(backend_setting, benchmark_id);
+    SettingList setting_list =
+        CreateSettingList(backend_setting, custom_config, benchmark_id);
 
     std::cout << "SettingList for " << benchmark_id << ":\n";
     dumpSettingList(setting_list);

flutter/cpp/utils.cc

@@ -125,27 +125,106 @@ mlperf_backend_configuration_t CppToCSettings(const SettingList &settings) {
   return c_settings;
 }
 
-SettingList createSettingList(const BackendSetting &backend_setting,
-                              std::string benchmark_id) {
+// Split the string by a given delimiter
+std::vector<std::string> _splitString(const std::string &str, char delimiter) {
+  std::vector<std::string> tokens;
+  std::stringstream ss(str);
+  std::string token;
+  while (std::getline(ss, token, delimiter)) {
+    tokens.push_back(token);
+  }
+  return tokens;
+}
+
+// Parse the key:value string list
+std::unordered_map<std::string, std::string> _parseKeyValueList(
+    const std::string &input) {
+  std::unordered_map<std::string, std::string> keyValueMap;
+  std::vector<std::string> pairs = _splitString(input, ',');  // Split by comma
+
+  for (const std::string &pair : pairs) {
+    std::vector<std::string> keyValue =
+        _splitString(pair, ':');  // Split by colon
+    if (keyValue.size() == 2) {
+      keyValueMap[keyValue[0]] = keyValue[1];
+    } else {
+      LOG(ERROR) << "Invalid key:value pair: " << pair;
+    }
+  }
+  return keyValueMap;
+}
+
+// Create the setting list for backend
+SettingList CreateSettingList(const BackendSetting &backend_setting,
+                              const std::string &custom_config,
+                              const std::string &benchmark_id) {
   SettingList setting_list;
   int setting_index = 0;
-
-  for (auto setting : backend_setting.common_setting()) {
+  for (const auto &setting : backend_setting.common_setting()) {
     setting_list.add_setting();
     (*setting_list.mutable_setting(setting_index)) = setting;
     setting_index++;
   }
 
   // Copy the benchmark specific settings
-  setting_index = 0;
-  for (auto bm_setting : backend_setting.benchmark_setting()) {
+  for (const auto &bm_setting : backend_setting.benchmark_setting()) {
     if (bm_setting.benchmark_id() == benchmark_id) {
       setting_list.mutable_benchmark_setting()->CopyFrom(bm_setting);
+
+      auto parsed = _parseKeyValueList(custom_config);
+      for (const auto &kv : parsed) {
+        CustomSetting custom_setting = CustomSetting();
+        custom_setting.set_id(kv.first);
+        custom_setting.set_value(kv.second);
+        setting_list.mutable_benchmark_setting()->mutable_custom_setting()->Add(
+            std::move(custom_setting));
+      }
+      break;
     }
   }
   LOG(INFO) << "setting_list:" << std::endl << setting_list.DebugString();
   return setting_list;
 }
 
+template <typename T>
+T GetConfigValue(mlperf_backend_configuration_t *configs, const char *key,
+                 T defaultValue);
+
+template <>
+int GetConfigValue<int>(mlperf_backend_configuration_t *configs,
+                        const char *key, int defaultValue) {
+  for (int i = 0; i < configs->count; ++i) {
+    if (strcmp(configs->keys[i], key) == 0) {
+      const char *valueStr = configs->values[i];
+      char *endptr = nullptr;
+      errno = 0;
+      long value =
+          strtol(valueStr, &endptr, 10);  // Base 10 for decimal conversion
+      if (errno == ERANGE || value < INT_MIN || value > INT_MAX) {
+        LOG(ERROR) << "Value out of range for int: " << valueStr;
+        return defaultValue;
+      }
+      if (endptr == valueStr || *endptr != '\0') {
+        LOG(ERROR) << "Invalid value for int: " << valueStr;
+        return defaultValue;
+      }
+      return static_cast<int>(value);
+    }
+  }
+  return defaultValue;
+}
+
+template <>
+std::string GetConfigValue<std::string>(mlperf_backend_configuration_t *configs,
+                                        const char *key,
+                                        std::string defaultValue) {
+  for (int i = 0; i < configs->count; ++i) {
+    if (strcmp(configs->keys[i], key) == 0) {
+      return std::string(configs->values[i]);
+    }
+  }
+  return defaultValue;
+}
+
 }  // namespace mobile
 }  // namespace mlperf

flutter/cpp/utils.h

@@ -65,8 +65,13 @@ void DeleteBackendConfiguration(mlperf_backend_configuration_t *configs);
 
 mlperf_backend_configuration_t CppToCSettings(const SettingList &settings);
 
-SettingList createSettingList(const BackendSetting &backend_setting,
-                              std::string benchmark_id);
+SettingList CreateSettingList(const BackendSetting &backend_setting,
+                              const std::string &custom_config,
+                              const std::string &benchmark_id);
+
+template <typename T>
+T GetConfigValue(mlperf_backend_configuration_t *configs, const char *key,
+                 T defaultValue);
 
 }  // namespace mobile
 }  // namespace mlperf

flutter/lib/benchmark/benchmark.dart

@@ -92,6 +92,11 @@ class Benchmark {
       setting: commonSettings,
       benchmarkSetting: benchmarkSettings,
     );
+    // Convert TaskConfig.CustomConfig to BenchmarkSetting.CustomSetting
+    final customConfigs = taskConfig.customConfig
+        .map((e) => pb.CustomSetting(id: e.id, value: e.value))
+        .toList();
+    benchmarkSettings.customSetting.addAll(customConfigs);
     final uris = selectedDelegate.modelFile.map((e) => e.modelPath).toList();
     final modelDirName = selectedDelegate.delegateName.replaceAll(' ', '_');
     final backendModelPath =

mobile_back_apple/cpp/backend_coreml/BUILD

@@ -39,6 +39,7 @@ apple_xcframework(
     },
     minimum_os_versions = {
         "ios": "13.1",
+        "macos": "13.1",
     },
     deps = [
         "//mobile_back_apple/cpp/backend_coreml:coreml_c",