Merge pull request #126 from lobehub/fix/i18n

fix: dance playing error

src/features/AgentViewer/ToolBar/index.tsx

@@ -69,7 +69,7 @@ const ToolBar = (props: ToolBarProps) => {
             break;
           }
           case 'power': {
-            viewer.model?.resetToIdle();
+            viewer.resetToIdle();
             break;
           }
           case 'screenShot': {

src/features/DanceList/Item/index.tsx

@@ -33,7 +33,7 @@ const DanceItem = (props: DanceItemProps) => {
     s.setCurrentPlayId,
   ]);
 
-  const [isPlaying, setIsPlaying] = useGlobalStore((s) => [s.isPlaying, s.setIsPlaying]);
+  const [isPlaying, setIsPlaying] = useState(false);
 
   const isCurrentPlay = currentPlayId ? currentPlayId === danceItem.danceId : false;
   const isSelected = currentIdentifier === danceItem.danceId;
@@ -45,20 +45,20 @@ const DanceItem = (props: DanceItemProps) => {
   const { downloading: danceDownloading, percent: dancePercent, fetchDanceUrl } = useLoadDance();
   const viewer = useGlobalStore((s) => s.viewer);
 
-  const handlePlayPause = () => {
-    viewer.model?.disposeAll();
+  const handlePlayPause = async () => {
     if (isPlaying && isCurrentPlay) {
       setIsPlaying(false);
+      viewer?.resetToIdle();
     } else {
       setCurrentPlayId(danceItem.danceId);
       setIsPlaying(true);
       const audioPromise = fetchAudioUrl(danceItem.danceId, danceItem.audio);
       const dancePromise = fetchDanceUrl(danceItem.danceId, danceItem.src);
-      Promise.all([dancePromise, audioPromise]).then((res) => {
-        if (!res) return;
-        const [danceUrl, audioUrl] = res;
-        if (danceUrl && audioUrl) viewer.model?.dance(danceUrl, audioUrl);
-      });
+      const [danceUrl, audioUrl] = await Promise.all([dancePromise, audioPromise]);
+      if (danceUrl && audioUrl)
+        viewer?.dance(danceUrl, audioUrl, () => {
+          setIsPlaying(false);
+        });
     }
   };
 

src/features/audioPlayer/audioPlayer.ts → src/libs/audioPlayer/audioPlayer.ts

@@ -7,11 +7,8 @@ export class AudioPlayer {
     this.bufferSource = undefined;
   }
 
-  public async playFromArrayBuffer(
-    buffer: ArrayBuffer,
-    onEnded?: () => void,
-    onProgress?: () => void,
-  ) {
+  public async playFromArrayBuffer(buffer: ArrayBuffer, onEnded?: () => void) {
+    this.stopPlay();
     const audioBuffer = await this.audio.decodeAudioData(buffer);
 
     this.bufferSource = this.audio.createBufferSource();
@@ -22,9 +19,6 @@ export class AudioPlayer {
     if (onEnded) {
       this.bufferSource.addEventListener('ended', onEnded);
     }
-    if (onProgress) {
-      this.bufferSource.addEventListener('time', onProgress);
-    }
   }
 
   public stopPlay() {

src/libs/vrmViewer/model.ts

@@ -4,7 +4,6 @@ import { AnimationAction, AnimationClip } from 'three';
 import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader';
 import { LoopOnce } from 'three/src/constants';
 
-import { AudioPlayer } from '@/features/audioPlayer/audioPlayer';
 import { loadMixamoAnimation } from '@/libs/FBXAnimation/loadMixamoAnimation';
 import { loadVMDAnimation } from '@/libs/VMDAnimation/loadVMDAnimation';
 import { convert } from '@/libs/VMDAnimation/vmd2vrmanim';
@@ -13,6 +12,7 @@ import IKHandler from '@/libs/VMDAnimation/vrm-ik-handler';
 import { VRMAnimation } from '@/libs/VRMAnimation/VRMAnimation';
 import { loadVRMAnimation } from '@/libs/VRMAnimation/loadVRMAnimation';
 import { VRMLookAtSmootherLoaderPlugin } from '@/libs/VRMLookAtSmootherLoaderPlugin/VRMLookAtSmootherLoaderPlugin';
+import { AudioPlayer } from '@/libs/audioPlayer/audioPlayer';
 import { EmoteController } from '@/libs/emoteController/emoteController';
 import { LipSync } from '@/libs/lipSync/lipSync';
 import { Screenplay } from '@/types/touch';
@@ -28,18 +28,15 @@ export class Model {
 
   private _lookAtTargetParent: THREE.Object3D;
   private _lipSync?: LipSync;
-  private _audioPlayer?: AudioPlayer;
+
   private _action: AnimationAction | undefined;
   private _clip: AnimationClip | undefined;
-  private _audio: string | undefined;
 
   constructor(lookAtTargetParent: THREE.Object3D) {
     this._lookAtTargetParent = lookAtTargetParent;
     this._lipSync = new LipSync(new AudioContext());
-    this._audioPlayer = new AudioPlayer(new AudioContext());
     this._action = undefined;
     this._clip = undefined;
-    this._audio = undefined;
   }
 
   public async loadVRM(url: string): Promise<void> {
@@ -95,11 +92,6 @@ export class Model {
       this._action.stop();
       this._action = undefined;
     }
-
-    if (this._audio) {
-      this._audioPlayer?.stopPlay();
-      this._audio = undefined;
-    }
   }
 
   /**
@@ -140,41 +132,20 @@ export class Model {
     }
   }
 
-  public async loadVMD(animationUrl: string) {
+  public async loadVMD(animationUrl: string, loop: boolean = true) {
     const { vrm, mixer } = this;
 
     if (vrm && mixer) {
       this.disposeAll();
       const clip = await loadVMDAnimation(animationUrl, vrm);
       const action = mixer.clipAction(clip);
+      if (!loop) action.setLoop(LoopOnce, 1);
       action.play();
       this._action = action;
       this._clip = clip;
     }
   }
 
-  /**
-   * 播放舞蹈，以音乐文件的播放作为结束标志。
-   */
-  public async dance(danceUrl: string, audioUrl: string, onEnd?: () => void) {
-    const { vrm, mixer } = this;
-    if (vrm && mixer) {
-      this.disposeAll();
-      const clip = await loadVMDAnimation(danceUrl, vrm);
-      const action = mixer.clipAction(clip);
-      action.setLoop(LoopOnce, 1).play(); // play animation
-      if (audioUrl) {
-        this._audioPlayer?.playFromURL(audioUrl, () => {
-          onEnd?.();
-        });
-        this._audio = audioUrl;
-      }
-
-      this._action = action;
-      this._clip = clip;
-    }
-  }
-
   public async resetToIdle() {
     this.disposeAll();
 

src/libs/vrmViewer/viewer.ts

@@ -1,7 +1,11 @@
 import { Parser } from 'mmd-parser';
 import * as THREE from 'three';
-import { GridHelper, Mesh, MeshLambertMaterial, PlaneGeometry } from 'three';
+import { Audio, GridHelper, Mesh, MeshLambertMaterial, PlaneGeometry } from 'three';
 import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls';
+import { LoopOnce } from 'three/src/constants';
+
+import { loadVMDAnimation } from '@/libs/VMDAnimation/loadVMDAnimation';
+import { AudioPlayer } from '@/libs/audioPlayer/audioPlayer';
 
 import { Model } from './model';
 
@@ -12,6 +16,7 @@ export class Viewer {
   private _renderer?: THREE.WebGLRenderer;
   private _clock: THREE.Clock;
   private _scene: THREE.Scene;
+  private _sound?: Audio;
   private _cameraHelper?: THREE.CameraHelper;
   private _camera?: THREE.PerspectiveCamera;
   private _cameraControls?: OrbitControls;
@@ -45,6 +50,36 @@ export class Viewer {
     this._clock.start();
   }
 
+  /**
+   * 播放舞蹈，以音乐文件的播放作为结束标志。
+   */
+  public async dance(danceUrl: string, audioUrl: string, onEnd?: () => void) {
+    if (!this._sound || !this.model) {
+      console.error('Audio Object or Model Object Not Existed');
+      return null;
+    }
+    this._sound.stop();
+    this.model?.disposeAll();
+    const audioLoader = new THREE.AudioLoader();
+    // 监听音频播放结束事件
+    this._sound.onEnded = () => {
+      onEnd?.();
+      this.model?.loadIdleAnimation();
+    };
+    const buffer = await audioLoader.loadAsync(audioUrl);
+    this._sound.setBuffer(buffer);
+    this._sound.setVolume(0.5);
+    this._sound.play();
+
+    this.model?.loadVMD(danceUrl, false);
+  }
+
+  public resetToIdle() {
+    this._sound?.stop();
+    this.model?.disposeAll();
+    this.model?.loadIdleAnimation();
+  }
+
   /**
    * 加载舞台
    * @param buffer
@@ -111,6 +146,13 @@ export class Viewer {
     this._cameraControls?.target.set(0, 0, 0);
     this._cameraControls.update();
 
+    // Audio 音频播放
+    const listener = new THREE.AudioListener();
+    this._camera.add(listener);
+
+    // 创建一个全局 audio 源
+    this._sound = new THREE.Audio(listener);
+
     const resizeObserver = new ResizeObserver(() => {
       setTimeout(() => this.resize(), 0);
     });

src/panels/RolePanel/RoleEdit/Touch/ActionList/Actions/Play.tsx

@@ -5,7 +5,7 @@ import { memo, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 
 import { DEFAULT_MOTION_ANIMATION } from '@/constants/touch';
-import { speakCharacter } from '@/features/messages/speakCharacter';
+import { speakCharacter } from '@/libs/messages/speakCharacter';
 import { agentSelectors, useAgentStore } from '@/store/agent';
 import { useGlobalStore } from '@/store/global';
 import { TouchAction } from '@/types/touch';

src/services/chat.ts

@@ -1,5 +1,5 @@
 import { OPENAI_API_KEY, OPENAI_END_POINT } from '@/constants/openai';
-import { speakCharacter } from '@/features/messages/speakCharacter';
+import { speakCharacter } from '@/libs/messages/speakCharacter';
 import { useGlobalStore } from '@/store/global';
 import { sessionSelectors, useSessionStore } from '@/store/session';
 import { configSelectors, useSettingStore } from '@/store/setting';