diff --git a/src/features/AgentViewer/ToolBar/index.tsx b/src/features/AgentViewer/ToolBar/index.tsx index 14994193..b3bb5599 100644 --- a/src/features/AgentViewer/ToolBar/index.tsx +++ b/src/features/AgentViewer/ToolBar/index.tsx @@ -69,7 +69,7 @@ const ToolBar = (props: ToolBarProps) => { break; } case 'power': { - viewer.model?.resetToIdle(); + viewer.resetToIdle(); break; } case 'screenShot': { diff --git a/src/features/DanceList/Item/index.tsx b/src/features/DanceList/Item/index.tsx index 58735a2d..8eed198a 100644 --- a/src/features/DanceList/Item/index.tsx +++ b/src/features/DanceList/Item/index.tsx @@ -46,10 +46,9 @@ const DanceItem = (props: DanceItemProps) => { const viewer = useGlobalStore((s) => s.viewer); const handlePlayPause = async () => { - viewer.model?.disposeAll(); if (isPlaying && isCurrentPlay) { setIsPlaying(false); - viewer.model?.loadIdleAnimation(); + viewer?.resetToIdle(); } else { setCurrentPlayId(danceItem.danceId); setIsPlaying(true); @@ -57,7 +56,7 @@ const DanceItem = (props: DanceItemProps) => { const dancePromise = fetchDanceUrl(danceItem.danceId, danceItem.src); const [danceUrl, audioUrl] = await Promise.all([dancePromise, audioPromise]); if (danceUrl && audioUrl) - viewer.model?.dance(danceUrl, audioUrl, () => { + viewer?.dance(danceUrl, audioUrl, () => { setIsPlaying(false); }); } diff --git a/src/features/audioPlayer/audioPlayer.ts b/src/libs/audioPlayer/audioPlayer.ts similarity index 100% rename from src/features/audioPlayer/audioPlayer.ts rename to src/libs/audioPlayer/audioPlayer.ts diff --git a/src/features/messages/speakCharacter.ts b/src/libs/messages/speakCharacter.ts similarity index 100% rename from src/features/messages/speakCharacter.ts rename to src/libs/messages/speakCharacter.ts diff --git a/src/libs/vrmViewer/model.ts b/src/libs/vrmViewer/model.ts index 0b51c951..9ee027e5 100644 --- a/src/libs/vrmViewer/model.ts +++ b/src/libs/vrmViewer/model.ts @@ -4,7 +4,6 @@ import { AnimationAction, AnimationClip } from 'three'; import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader'; import { LoopOnce } from 'three/src/constants'; -import { AudioPlayer } from '@/features/audioPlayer/audioPlayer'; import { loadMixamoAnimation } from '@/libs/FBXAnimation/loadMixamoAnimation'; import { loadVMDAnimation } from '@/libs/VMDAnimation/loadVMDAnimation'; import { convert } from '@/libs/VMDAnimation/vmd2vrmanim'; @@ -13,6 +12,7 @@ import IKHandler from '@/libs/VMDAnimation/vrm-ik-handler'; import { VRMAnimation } from '@/libs/VRMAnimation/VRMAnimation'; import { loadVRMAnimation } from '@/libs/VRMAnimation/loadVRMAnimation'; import { VRMLookAtSmootherLoaderPlugin } from '@/libs/VRMLookAtSmootherLoaderPlugin/VRMLookAtSmootherLoaderPlugin'; +import { AudioPlayer } from '@/libs/audioPlayer/audioPlayer'; import { EmoteController } from '@/libs/emoteController/emoteController'; import { LipSync } from '@/libs/lipSync/lipSync'; import { Screenplay } from '@/types/touch'; @@ -28,14 +28,13 @@ export class Model { private _lookAtTargetParent: THREE.Object3D; private _lipSync?: LipSync; - private _audioPlayer?: AudioPlayer; + private _action: AnimationAction | undefined; private _clip: AnimationClip | undefined; constructor(lookAtTargetParent: THREE.Object3D) { this._lookAtTargetParent = lookAtTargetParent; this._lipSync = new LipSync(new AudioContext()); - this._audioPlayer = new AudioPlayer(new AudioContext()); this._action = undefined; this._clip = undefined; } @@ -93,8 +92,6 @@ export class Model { this._action.stop(); this._action = undefined; } - - this._audioPlayer?.stopPlay(); } /** @@ -135,39 +132,20 @@ export class Model { } } - public async loadVMD(animationUrl: string) { + public async loadVMD(animationUrl: string, loop: boolean = true) { const { vrm, mixer } = this; if (vrm && mixer) { this.disposeAll(); const clip = await loadVMDAnimation(animationUrl, vrm); const action = mixer.clipAction(clip); + if (!loop) action.setLoop(LoopOnce, 1); action.play(); this._action = action; this._clip = clip; } } - /** - * 播放舞蹈，以音乐文件的播放作为结束标志。 - */ - public async dance(danceUrl: string, audioUrl: string, onEnd?: () => void) { - const { vrm, mixer } = this; - if (vrm && mixer) { - this.disposeAll(); - const clip = await loadVMDAnimation(danceUrl, vrm); - const action = mixer.clipAction(clip); - action.setLoop(LoopOnce, 1).play(); // play animation - this._audioPlayer?.playFromURL(audioUrl, () => { - this.disposeAll(); - onEnd?.(); - }); - - this._action = action; - this._clip = clip; - } - } - public async resetToIdle() { this.disposeAll(); diff --git a/src/libs/vrmViewer/viewer.ts b/src/libs/vrmViewer/viewer.ts index 312d2b88..f85df09e 100644 --- a/src/libs/vrmViewer/viewer.ts +++ b/src/libs/vrmViewer/viewer.ts @@ -1,7 +1,11 @@ import { Parser } from 'mmd-parser'; import * as THREE from 'three'; -import { GridHelper, Mesh, MeshLambertMaterial, PlaneGeometry } from 'three'; +import { Audio, GridHelper, Mesh, MeshLambertMaterial, PlaneGeometry } from 'three'; import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls'; +import { LoopOnce } from 'three/src/constants'; + +import { loadVMDAnimation } from '@/libs/VMDAnimation/loadVMDAnimation'; +import { AudioPlayer } from '@/libs/audioPlayer/audioPlayer'; import { Model } from './model'; @@ -12,6 +16,7 @@ export class Viewer { private _renderer?: THREE.WebGLRenderer; private _clock: THREE.Clock; private _scene: THREE.Scene; + private _sound?: Audio; private _cameraHelper?: THREE.CameraHelper; private _camera?: THREE.PerspectiveCamera; private _cameraControls?: OrbitControls; @@ -45,6 +50,36 @@ export class Viewer { this._clock.start(); } + /** + * 播放舞蹈，以音乐文件的播放作为结束标志。 + */ + public async dance(danceUrl: string, audioUrl: string, onEnd?: () => void) { + if (!this._sound || !this.model) { + console.error('Audio Object or Model Object Not Existed'); + return null; + } + this._sound.stop(); + this.model?.disposeAll(); + const audioLoader = new THREE.AudioLoader(); + // 监听音频播放结束事件 + this._sound.onEnded = () => { + onEnd?.(); + this.model?.loadIdleAnimation(); + }; + const buffer = await audioLoader.loadAsync(audioUrl); + this._sound.setBuffer(buffer); + this._sound.setVolume(0.5); + this._sound.play(); + + this.model?.loadVMD(danceUrl, false); + } + + public resetToIdle() { + this._sound?.stop(); + this.model?.disposeAll(); + this.model?.loadIdleAnimation(); + } + /** * 加载舞台 * @param buffer @@ -111,6 +146,13 @@ export class Viewer { this._cameraControls?.target.set(0, 0, 0); this._cameraControls.update(); + // Audio 音频播放 + const listener = new THREE.AudioListener(); + this._camera.add(listener); + + // 创建一个全局 audio 源 + this._sound = new THREE.Audio(listener); + const resizeObserver = new ResizeObserver(() => { setTimeout(() => this.resize(), 0); }); diff --git a/src/panels/RolePanel/RoleEdit/Touch/ActionList/Actions/Play.tsx b/src/panels/RolePanel/RoleEdit/Touch/ActionList/Actions/Play.tsx index d224e3f7..8415aba9 100644 --- a/src/panels/RolePanel/RoleEdit/Touch/ActionList/Actions/Play.tsx +++ b/src/panels/RolePanel/RoleEdit/Touch/ActionList/Actions/Play.tsx @@ -5,7 +5,7 @@ import { memo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { DEFAULT_MOTION_ANIMATION } from '@/constants/touch'; -import { speakCharacter } from '@/features/messages/speakCharacter'; +import { speakCharacter } from '@/libs/messages/speakCharacter'; import { agentSelectors, useAgentStore } from '@/store/agent'; import { useGlobalStore } from '@/store/global'; import { TouchAction } from '@/types/touch'; diff --git a/src/services/chat.ts b/src/services/chat.ts index 04636c6c..ff17b1e9 100644 --- a/src/services/chat.ts +++ b/src/services/chat.ts @@ -1,5 +1,5 @@ import { OPENAI_API_KEY, OPENAI_END_POINT } from '@/constants/openai'; -import { speakCharacter } from '@/features/messages/speakCharacter'; +import { speakCharacter } from '@/libs/messages/speakCharacter'; import { useGlobalStore } from '@/store/global'; import { sessionSelectors, useSessionStore } from '@/store/session'; import { configSelectors, useSettingStore } from '@/store/setting';