Avoid deadlock issues caused by logging in a loop.

Signed-off-by: wangfakang <[email protected]>

src/misc/param.cc

@@ -61,21 +61,29 @@ void initEnv() {
 
 void ncclLoadParam(char const* env, int64_t deftVal, int64_t uninitialized, int64_t* cache) {
   static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+  if (__builtin_expect(__atomic_load_n(cache, __ATOMIC_RELAXED) != uninitialized, true)) {
+    return;
+  }
+
   pthread_mutex_lock(&mutex);
   if (__atomic_load_n(cache, __ATOMIC_RELAXED) == uninitialized) {
     const char* str = ncclGetEnv(env);
     int64_t value = deftVal;
-    if (str && strlen(str) > 0) {
-      errno = 0;
-      value = strtoll(str, nullptr, 0);
-      if (errno) {
-        value = deftVal;
-        INFO(NCCL_ALL,"Invalid value %s for %s, using default %lld.", str, env, (long long)deftVal);
-      } else {
-        INFO(NCCL_ENV,"%s set by environment to %lld.", env, (long long)value);
-      }
+    if (!str || strlen(str) <= 0) {
+      __atomic_store_n(cache, value, __ATOMIC_RELAXED);
+      pthread_mutex_unlock(&mutex);
+      return;
+    }
+    errno = 0;
+    value = strtoll(str, nullptr, 0);
+    // To prevent deadlock issues caused by logging in a loop,
+    // so cache the value before the log operation.
+    __atomic_store_n(cache, errno ? deftVal : value, __ATOMIC_RELAXED);
+    if (errno) {
+      INFO(NCCL_ALL,"Invalid value %s for %s, using default %lld.", str, env, (long long)deftVal);
+    } else {
+      INFO(NCCL_ENV,"%s set by environment to %lld.", env, (long long)value);
     }
-    __atomic_store_n(cache, value, __ATOMIC_RELAXED);
   }
   pthread_mutex_unlock(&mutex);
 }
@@ -84,4 +92,4 @@ const char *ncclGetEnv(const char *name) {
   static pthread_once_t once = PTHREAD_ONCE_INIT;
   pthread_once(&once, initEnv);
   return getenv(name);
-}
+}