Failed to offload sd3.5L to cpu

[moonlightian]

sd3 branch
when I tried to lora sd3.5-Large，lode_safetensors failed where I found that the modes are set to be offloaded to cpu
codes:

def load_target_model(self, args, weight_dtype, accelerator):
        # currently offload to cpu for some models

        # if the file is fp8 and we are using fp8_base, we can load it as is (fp8)
        loading_dtype = None if args.fp8_base else weight_dtype

        # if we load to cpu, flux.to(fp8) takes a long time, so we should load to gpu in future
        state_dict = utils.load_safetensors(
            args.pretrained_model_name_or_path, "cpu", disable_mmap=args.disable_mmap_load_safetensors, dtype=loading_dtype
        )
        mmdit = sd3_utils.load_mmdit(state_dict, loading_dtype, "cpu")

Traceback (most recent call last):
  File "/home/bml/workspace/jinyibo/workspace/llm-factory/src/sd-scripts-sd3/library/utils.py", line 367, in load_safetensors
    state_dict = load_file(path, device=device)
  File "/opt/conda/envs/codelab/lib/python3.10/site-packages/safetensors/torch.py", line 313, in load_file
    with safe_open(filename, framework="pt", device=device) as f:
OSError: No such device (os error 19)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/bml/workspace/jinyibo/workspace/llm-factory/src/sd-scripts-sd3/sd3_train_network.py", line 480, in <module>
    trainer.train(args)
  File "/home/bml/workspace/jinyibo/workspace/llm-factory/src/sd-scripts-sd3/train_network.py", line 413, in train
    model_version, text_encoder, vae, unet = self.load_target_model(args, weight_dtype, accelerator)
  File "/home/bml/workspace/jinyibo/workspace/llm-factory/src/sd-scripts-sd3/sd3_train_network.py", line 70, in load_target_model
    state_dict = utils.load_safetensors(
  File "/home/bml/workspace/jinyibo/workspace/llm-factory/src/sd-scripts-sd3/library/utils.py", line 369, in load_safetensors
    state_dict = load_file(path)  # prevent device invalid Error
  File "/opt/conda/envs/codelab/lib/python3.10/site-packages/safetensors/torch.py", line 313, in load_file
    with safe_open(filename, framework="pt", device=device) as f:
OSError: No such device (os error 19)