Refactoring

Source/FunnyGirl/FunnyGirl.Build.cs

@@ -8,7 +8,7 @@ public class FunnyGirl : ModuleRules
 	{
 		PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs;
 	
-		PublicDependencyModuleNames.AddRange(new string[] { "Core", "CoreUObject", "Engine", "InputCore", "EnhancedInput", "HTTP", "Json", "JsonUtilities", "AssetRegistry", "MediaAssets", "UMG", "IKRig" });
+		PublicDependencyModuleNames.AddRange(new string[] { "Core", "CoreUObject", "Engine", "InputCore", "EnhancedInput", "HTTP", "Json", "JsonUtilities", "AssetRegistry", "MediaAssets", "UMG", "IKRig", "QwenTTSStreaming" });
 
 		PrivateDependencyModuleNames.AddRange(new string[] { "Slate", "SlateCore" });
 

Source/FunnyGirl/Private/Morph/LipSyncComponent.cpp

@@ -6,9 +6,109 @@
 #include "Components/AudioComponent.h"
 #include "Components/SceneComponent.h"
 #include "GameFramework/Actor.h"
+#include "QwenTTSStreamingComponent.h"
 #include "Sound/SoundBase.h"
 #include "Sound/SoundWave.h"
 
+namespace
+{
+void BuildStreamingVisemeWeights(
+	const TArray<float>& Samples,
+	int32 StartSample,
+	int32 WindowSampleCount,
+	float StepSeconds,
+	float Gate,
+	float Gain,
+	float Gamma,
+	float SilenceThreshold,
+	float PlosiveHoldTime,
+	float ShapeTime,
+	float& InOutPlosiveHoldRemaining,
+	float& InOutPreviousRms,
+	TMap<FName, float>& OutWeights)
+{
+	OutWeights.Empty();
+
+	float SumSquares = 0.0f;
+	float SumAbs = 0.0f;
+	float SumDiffAbs = 0.0f;
+	int32 ZeroCrossings = 0;
+	float PreviousSample = Samples[StartSample];
+	for (int32 SampleIndex = 0; SampleIndex < WindowSampleCount; ++SampleIndex)
+	{
+		const float Sample = Samples[StartSample + SampleIndex];
+		SumSquares += Sample * Sample;
+		SumAbs += FMath::Abs(Sample);
+		if (SampleIndex > 0)
+		{
+			SumDiffAbs += FMath::Abs(Sample - PreviousSample);
+			if ((Sample >= 0.0f) != (PreviousSample >= 0.0f))
+			{
+				++ZeroCrossings;
+			}
+		}
+		PreviousSample = Sample;
+	}
+
+	const float Rms = FMath::Sqrt(SumSquares / static_cast<float>(WindowSampleCount));
+	const float MeanAbs = SumAbs / static_cast<float>(WindowSampleCount);
+	const float Brightness = SumAbs > KINDA_SMALL_NUMBER ? FMath::Clamp(SumDiffAbs / (SumAbs * 2.0f), 0.0f, 1.0f) : 0.0f;
+	const float Zcr = static_cast<float>(ZeroCrossings) / static_cast<float>(WindowSampleCount);
+
+	float MouthWeight = 0.0f;
+	if (Rms > Gate)
+	{
+		const float Normalized = FMath::Clamp((Rms - Gate) * Gain, 0.0f, 1.0f);
+		MouthWeight = FMath::Pow(Normalized, FMath::Max(Gamma, 0.1f));
+	}
+
+	const float Onset = Rms - InOutPreviousRms;
+	if (Rms > Gate * 1.6f && Onset > Gate * 1.8f && Brightness < 0.42f)
+	{
+		InOutPlosiveHoldRemaining = PlosiveHoldTime;
+	}
+	InOutPreviousRms = Rms;
+
+	if (MouthWeight <= SilenceThreshold)
+	{
+		InOutPlosiveHoldRemaining = FMath::Max(0.0f, InOutPlosiveHoldRemaining - StepSeconds);
+		return;
+	}
+
+	if (InOutPlosiveHoldRemaining > 0.0f)
+	{
+		InOutPlosiveHoldRemaining = FMath::Max(0.0f, InOutPlosiveHoldRemaining - StepSeconds);
+		OutWeights.Add(TEXT("MBP"), FMath::Clamp(MouthWeight * 1.15f, 0.0f, 1.0f));
+		OutWeights.Add(TEXT("A"), MouthWeight * 0.18f);
+	}
+	else if (Zcr > 0.105f || Brightness > 0.62f)
+	{
+		OutWeights.Add(TEXT("SS"), MouthWeight * FMath::Clamp(0.75f + Brightness * 0.35f, 0.0f, 1.0f));
+		OutWeights.Add(TEXT("I"), MouthWeight * 0.35f);
+	}
+	else if (Brightness > 0.42f)
+	{
+		const float WideBlend = FMath::Clamp((Brightness - 0.42f) / 0.2f, 0.0f, 1.0f);
+		OutWeights.Add(TEXT("I"), MouthWeight * (0.45f + WideBlend * 0.45f));
+		OutWeights.Add(TEXT("E"), MouthWeight * (0.35f + (1.0f - WideBlend) * 0.25f));
+	}
+	else if (Brightness < 0.24f && MeanAbs > Gate * 1.35f)
+	{
+		const bool bUseU = FMath::Fmod(ShapeTime, 0.38f) > 0.19f;
+		OutWeights.Add(bUseU ? TEXT("U") : TEXT("O"), MouthWeight * 0.78f);
+		OutWeights.Add(TEXT("A"), MouthWeight * 0.28f);
+	}
+	else
+	{
+		OutWeights.Add(TEXT("A"), MouthWeight);
+		if (Brightness > 0.30f)
+		{
+			OutWeights.Add(TEXT("E"), MouthWeight * 0.25f);
+		}
+	}
+}
+}
+
 ULipSyncComponent::ULipSyncComponent()
 {
 	PrimaryComponentTick.bCanEverTick = true;
@@ -35,6 +135,28 @@ void ULipSyncComponent::BeginPlay()
 	{
 		MorphController = ResolveMorphController();
 	}
+
+	if (bAutoBindTTSStreamingComponent)
+	{
+		if (!TTSStreamingComponent)
+		{
+			if (AActor* Owner = GetOwner())
+			{
+				TTSStreamingComponent = Owner->FindComponentByClass<UQwenTTSStreamingComponent>();
+			}
+		}
+
+		if (TTSStreamingComponent)
+		{
+			BindToTTSStreamingComponent(TTSStreamingComponent);
+		}
+	}
+}
+
+void ULipSyncComponent::EndPlay(const EEndPlayReason::Type EndPlayReason)
+{
+	UnbindFromTTSStreamingComponent();
+	Super::EndPlay(EndPlayReason);
 }
 
 void ULipSyncComponent::SetVisemeWeight(FName VisemeName, float Weight)
@@ -127,38 +249,139 @@ bool ULipSyncComponent::PlayAudioDrivenLipSyncWithComponent(UAudioComponent* Aud
 	return true;
 }
 
-bool ULipSyncComponent::FollowExternalAudioComponent(UAudioComponent* AudioComponent, const FText& Transcript)
+bool ULipSyncComponent::BindToTTSStreamingComponent(UQwenTTSStreamingComponent* InTTSStreamingComponent)
 {
-	if (!AudioComponent)
+	if (!InTTSStreamingComponent)
 	{
 		return false;
 	}
 
+	if (TTSStreamingComponent != InTTSStreamingComponent || bBoundToTTSStreamingComponent)
+	{
+		UnbindFromTTSStreamingComponent();
+	}
+
+	TTSStreamingComponent = InTTSStreamingComponent;
+	TTSStreamingComponent->OnStreamStarted.AddUniqueDynamic(this, &ULipSyncComponent::HandleTTSStreamStarted);
+	TTSStreamingComponent->OnAudioChunk.AddUniqueDynamic(this, &ULipSyncComponent::HandleTTSAudioChunk);
+	TTSStreamingComponent->OnStreamFinished.AddUniqueDynamic(this, &ULipSyncComponent::HandleTTSStreamFinished);
+	bBoundToTTSStreamingComponent = true;
+
+	AudioDrivenComponent = TTSStreamingComponent;
+	if (TTSStreamingComponent->IsStreaming() || TTSStreamingComponent->IsPlayingVoiceAudio())
+	{
+		StartStreamingAudioAnalyzer(TTSStreamingComponent);
+	}
+
+	return true;
+}
+
+void ULipSyncComponent::UnbindFromTTSStreamingComponent()
+{
+	if (!TTSStreamingComponent || !bBoundToTTSStreamingComponent)
+	{
+		bBoundToTTSStreamingComponent = false;
+		return;
+	}
+
+	TTSStreamingComponent->OnStreamStarted.RemoveDynamic(this, &ULipSyncComponent::HandleTTSStreamStarted);
+	TTSStreamingComponent->OnAudioChunk.RemoveDynamic(this, &ULipSyncComponent::HandleTTSAudioChunk);
+	TTSStreamingComponent->OnStreamFinished.RemoveDynamic(this, &ULipSyncComponent::HandleTTSStreamFinished);
+	bBoundToTTSStreamingComponent = false;
+}
+
+bool ULipSyncComponent::StartStreamingAudioAnalyzer(UAudioComponent* AudioComponent)
+{
 	AudioDrivenComponent = AudioComponent;
-	AudioDrivenTranscript = Transcript;
-	AudioDrivenMode = ELipSyncAudioDrivenMode::TranscriptVisemes;
+	AudioDrivenMode = ELipSyncAudioDrivenMode::StreamingAudioAnalyzer;
 	AudioDrivenPlaybackTime = 0.0f;
 	AudioDrivenDuration = 0.0f;
 	LastEnvelopeValue = 0.0f;
 	LastNonSilentMouthWeight = 0.0f;
 	SilenceHoldRemaining = 0.0f;
+	StreamingAnalyzerPlosiveHoldRemaining = 0.0f;
+	StreamingAnalyzerPreviousRms = 0.0f;
+	StreamingAnalyzerShapePhase = 0.0f;
 	bReceivedEnvelopeValue = false;
 	bAudioDrivenControlsPlayback = false;
-	bWaitingForExternalAudioPlayback = !AudioComponent->IsPlaying();
+	bWaitingForExternalAudioPlayback = AudioComponent && !AudioComponent->IsPlaying();
 	ExternalAudioPlaybackWaitTime = 0.0f;
-
-	AudioComponent->EnvelopeFollowerAttackTime = 8;
-	AudioComponent->EnvelopeFollowerReleaseTime = 90;
-	AudioComponent->OnAudioFinished.AddUniqueDynamic(this, &ULipSyncComponent::HandleAudioDrivenFinished);
-	AudioComponent->OnAudioPlaybackPercent.AddUniqueDynamic(this, &ULipSyncComponent::HandleAudioDrivenPlaybackPercent);
-	AudioComponent->OnAudioSingleEnvelopeValue.AddUniqueDynamic(this, &ULipSyncComponent::HandleAudioDrivenEnvelopeValue);
-
-	RebuildGeneratedTranscriptVisemes();
 	ClearVisemes();
 	bAudioDrivenLipSyncActive = true;
 	return true;
 }
 
+void ULipSyncComponent::PushAudioSamplesPCM16Bytes(const TArray<uint8>& PCMBytes, int32 SampleRate, int32 NumChannels)
+{
+	if (PCMBytes.Num() < 2 || SampleRate <= 0 || NumChannels <= 0)
+	{
+		return;
+	}
+
+	if (StreamingAnalyzerSampleRate != SampleRate)
+	{
+		ResetStreamingAnalyzerTimeline();
+		StreamingAnalyzerSampleRate = SampleRate;
+	}
+
+	const int32 BytesPerFrame = NumChannels * static_cast<int32>(sizeof(int16));
+	const int32 FrameCount = PCMBytes.Num() / BytesPerFrame;
+	TArray<float> MonoSamples;
+	MonoSamples.Reserve(FrameCount);
+
+	for (int32 FrameIndex = 0; FrameIndex < FrameCount; ++FrameIndex)
+	{
+		float MixedSample = 0.0f;
+		for (int32 ChannelIndex = 0; ChannelIndex < NumChannels; ++ChannelIndex)
+		{
+			const int32 ByteIndex = FrameIndex * BytesPerFrame + ChannelIndex * 2;
+			const int16 Sample = static_cast<int16>(static_cast<uint16>(PCMBytes[ByteIndex]) | (static_cast<uint16>(PCMBytes[ByteIndex + 1]) << 8));
+			MixedSample += static_cast<float>(Sample) / 32768.0f;
+		}
+		MonoSamples.Add(FMath::Clamp(MixedSample / static_cast<float>(NumChannels), -1.0f, 1.0f));
+	}
+
+	AppendStreamingAnalyzerMarkers(MonoSamples, SampleRate);
+	if (!bAudioDrivenLipSyncActive && AudioDrivenComponent && AudioDrivenComponent->IsPlaying())
+	{
+		StartStreamingAudioAnalyzer(AudioDrivenComponent.Get());
+	}
+}
+
+void ULipSyncComponent::PushAudioSamplesFloat(const TArray<float>& Samples, int32 SampleRate, int32 NumChannels)
+{
+	if (Samples.Num() == 0 || SampleRate <= 0 || NumChannels <= 0)
+	{
+		return;
+	}
+
+	if (StreamingAnalyzerSampleRate != SampleRate)
+	{
+		ResetStreamingAnalyzerTimeline();
+		StreamingAnalyzerSampleRate = SampleRate;
+	}
+
+	const int32 FrameCount = Samples.Num() / NumChannels;
+	TArray<float> MonoSamples;
+	MonoSamples.Reserve(FrameCount);
+
+	for (int32 FrameIndex = 0; FrameIndex < FrameCount; ++FrameIndex)
+	{
+		float MixedSample = 0.0f;
+		for (int32 ChannelIndex = 0; ChannelIndex < NumChannels; ++ChannelIndex)
+		{
+			MixedSample += Samples[FrameIndex * NumChannels + ChannelIndex];
+		}
+		MonoSamples.Add(FMath::Clamp(MixedSample / static_cast<float>(NumChannels), -1.0f, 1.0f));
+	}
+
+	AppendStreamingAnalyzerMarkers(MonoSamples, SampleRate);
+	if (!bAudioDrivenLipSyncActive && AudioDrivenComponent && AudioDrivenComponent->IsPlaying())
+	{
+		StartStreamingAudioAnalyzer(AudioDrivenComponent.Get());
+	}
+}
+
 void ULipSyncComponent::StopAudioDrivenLipSync()
 {
 	bAudioDrivenLipSyncActive = false;
@@ -170,6 +393,7 @@ void ULipSyncComponent::StopAudioDrivenLipSync()
 
 	bWaitingForExternalAudioPlayback = false;
 	ExternalAudioPlaybackWaitTime = 0.0f;
+	ResetStreamingAnalyzerTimeline();
 
 	ClearVisemes();
 }
@@ -283,6 +507,12 @@ void ULipSyncComponent::UpdateAudioDrivenLipSync(float DeltaTime)
 		return;
 	}
 
+	if (AudioDrivenMode == ELipSyncAudioDrivenMode::StreamingAudioAnalyzer)
+	{
+		UpdateStreamingAudioAnalyzer(DeltaTime);
+		return;
+	}
+
 	if (!AudioDrivenComponent || !AudioDrivenComponent->IsPlaying())
 	{
 		if (AudioDrivenComponent && !bAudioDrivenControlsPlayback && bWaitingForExternalAudioPlayback)
@@ -317,6 +547,135 @@ void ULipSyncComponent::UpdateAudioDrivenLipSync(float DeltaTime)
 	SetAudioDrivenTargetVisemes(GetAudioDrivenMouthWeight(DeltaTime));
 }
 
+void ULipSyncComponent::UpdateStreamingAudioAnalyzer(float DeltaTime)
+{
+	if (AudioDrivenComponent && !AudioDrivenComponent->IsPlaying())
+	{
+		const bool bUsingTTSStreaming = TTSStreamingComponent && AudioDrivenComponent == TTSStreamingComponent;
+		if (bUsingTTSStreaming && (TTSStreamingComponent->IsStreaming() || StreamingAnalyzerMarkerIndex < StreamingAnalyzerMarkers.Num()))
+		{
+			bWaitingForExternalAudioPlayback = true;
+			ExternalAudioPlaybackWaitTime += DeltaTime;
+			if (TTSStreamingComponent->IsStreaming() || ExternalAudioPlaybackWaitTime < 1.0f)
+			{
+				TargetVisemeWeights.Empty();
+				return;
+			}
+
+			bWaitingForExternalAudioPlayback = false;
+			ExternalAudioPlaybackWaitTime = 0.0f;
+		}
+
+		if (bWaitingForExternalAudioPlayback)
+		{
+			ExternalAudioPlaybackWaitTime += DeltaTime;
+			if (ExternalAudioPlaybackWaitTime < 10.0f)
+			{
+				return;
+			}
+		}
+
+		HandleAudioDrivenFinished();
+		return;
+	}
+
+	bWaitingForExternalAudioPlayback = false;
+	ExternalAudioPlaybackWaitTime = 0.0f;
+	if (!bStreamingAnalyzerPlaybackActive)
+	{
+		AudioDrivenPlaybackTime = 0.0f;
+		StreamingAnalyzerMarkerIndex = 0;
+		bStreamingAnalyzerPlaybackActive = true;
+	}
+
+	AudioDrivenPlaybackTime += DeltaTime;
+	ApplyStreamingAnalyzerMarkerAtTime(AudioDrivenPlaybackTime);
+}
+
+void ULipSyncComponent::AppendStreamingAnalyzerMarkers(const TArray<float>& MonoSamples, int32 SampleRate)
+{
+	if (MonoSamples.Num() == 0 || SampleRate <= 0)
+	{
+		return;
+	}
+
+	const int32 WindowSampleCount = FMath::Clamp(
+		FMath::RoundToInt(StreamingAnalyzerWindowSeconds * static_cast<float>(SampleRate)),
+		128,
+		FMath::Max(128, MonoSamples.Num()));
+
+	if (MonoSamples.Num() < WindowSampleCount)
+	{
+		StreamingAnalyzerQueuedDuration += static_cast<float>(MonoSamples.Num()) / static_cast<float>(SampleRate);
+		return;
+	}
+
+	const int32 StepSampleCount = FMath::Max(1, FMath::RoundToInt(static_cast<float>(SampleRate) * StreamingAnalyzerWindowSeconds * 0.5f));
+	const float StepSeconds = static_cast<float>(StepSampleCount) / static_cast<float>(SampleRate);
+	const float Gate = FMath::Max(StreamingAnalyzerNoiseGate, 0.0f);
+	for (int32 StartSample = 0; StartSample + WindowSampleCount <= MonoSamples.Num(); StartSample += StepSampleCount)
+	{
+		FLipSyncStreamingMarker Marker;
+		Marker.Time = StreamingAnalyzerQueuedDuration + (static_cast<float>(StartSample) / static_cast<float>(SampleRate));
+
+		BuildStreamingVisemeWeights(
+			MonoSamples,
+			StartSample,
+			WindowSampleCount,
+			StepSeconds,
+			Gate,
+			StreamingAnalyzerGain,
+			StreamingAnalyzerMouthGamma,
+			SilenceThreshold,
+			StreamingAnalyzerPlosiveHoldTime,
+			Marker.Time,
+			StreamingAnalyzerPlosiveHoldRemaining,
+			StreamingAnalyzerPreviousRms,
+			Marker.VisemeWeights);
+
+		StreamingAnalyzerMarkers.Add(MoveTemp(Marker));
+	}
+
+	StreamingAnalyzerQueuedDuration += static_cast<float>(MonoSamples.Num()) / static_cast<float>(SampleRate);
+}
+
+void ULipSyncComponent::ResetStreamingAnalyzerTimeline()
+{
+	StreamingAnalyzerMarkers.Reset();
+	StreamingAnalyzerQueuedDuration = 0.0f;
+	StreamingAnalyzerMarkerIndex = 0;
+	StreamingAnalyzerPlosiveHoldRemaining = 0.0f;
+	StreamingAnalyzerPreviousRms = 0.0f;
+	StreamingAnalyzerShapePhase = 0.0f;
+	bStreamingAnalyzerPlaybackActive = false;
+}
+
+void ULipSyncComponent::ApplyStreamingAnalyzerMarkerAtTime(float PlaybackTime)
+{
+	TargetVisemeWeights.Empty();
+
+	if (StreamingAnalyzerMarkers.Num() == 0)
+	{
+		return;
+	}
+
+	while (StreamingAnalyzerMarkerIndex + 1 < StreamingAnalyzerMarkers.Num()
+		&& StreamingAnalyzerMarkers[StreamingAnalyzerMarkerIndex + 1].Time <= PlaybackTime)
+	{
+		++StreamingAnalyzerMarkerIndex;
+	}
+
+	TargetVisemeWeights = StreamingAnalyzerMarkers[StreamingAnalyzerMarkerIndex].VisemeWeights;
+
+	const float OldestAllowedTime = PlaybackTime - StreamingAnalyzerBufferSeconds;
+	if (StreamingAnalyzerMarkerIndex > 32 && StreamingAnalyzerMarkers[StreamingAnalyzerMarkerIndex].Time > OldestAllowedTime)
+	{
+		const int32 RemoveCount = FMath::Max(0, StreamingAnalyzerMarkerIndex - 16);
+		StreamingAnalyzerMarkers.RemoveAt(0, RemoveCount, EAllowShrinking::No);
+		StreamingAnalyzerMarkerIndex -= RemoveCount;
+	}
+}
+
 float ULipSyncComponent::GetAudioDrivenMouthWeight(float DeltaTime)
 {
 	float EnvelopeValue = 0.0f;
@@ -648,3 +1007,40 @@ void ULipSyncComponent::HandleAudioDrivenEnvelopeValue(const USoundWave* Playing
 	LastEnvelopeValue = FMath::Max(EnvelopeValue, 0.0f);
 	bReceivedEnvelopeValue = true;
 }
+
+void ULipSyncComponent::HandleTTSStreamStarted()
+{
+	AudioDrivenComponent = TTSStreamingComponent;
+	AudioDrivenMode = ELipSyncAudioDrivenMode::StreamingAudioAnalyzer;
+	bAudioDrivenLipSyncActive = false;
+	bWaitingForExternalAudioPlayback = true;
+	ExternalAudioPlaybackWaitTime = 0.0f;
+	AudioDrivenPlaybackTime = 0.0f;
+	ResetStreamingAnalyzerTimeline();
+	ClearVisemes();
+}
+
+void ULipSyncComponent::HandleTTSAudioChunk(const FQwenTTSStreamChunk& Chunk)
+{
+	if (!TTSStreamingComponent || Chunk.PCMData.Num() == 0)
+	{
+		return;
+	}
+
+	AudioDrivenComponent = TTSStreamingComponent;
+	PushAudioSamplesPCM16Bytes(Chunk.PCMData, Chunk.SampleRate, Chunk.NumChannels);
+	if (TTSStreamingComponent->IsPlayingVoiceAudio() && (!bAudioDrivenLipSyncActive || AudioDrivenMode != ELipSyncAudioDrivenMode::StreamingAudioAnalyzer))
+	{
+		StartStreamingAudioAnalyzer(TTSStreamingComponent);
+	}
+}
+
+void ULipSyncComponent::HandleTTSStreamFinished()
+{
+	if (!TTSStreamingComponent || TTSStreamingComponent->IsPlayingVoiceAudio())
+	{
+		return;
+	}
+
+	HandleAudioDrivenFinished();
+}

Source/FunnyGirl/Public/Morph/LipSyncComponent.h

@@ -4,11 +4,13 @@
 
 #include "CoreMinimal.h"
 #include "Components/ActorComponent.h"
+#include "QwenTTSStreamingTypes.h"
 #include "LipSyncComponent.generated.h"
 
 class ULipSyncMappingDataAsset;
 class UMorphControllerComponent;
 class UAudioComponent;
+class UQwenTTSStreamingComponent;
 class USoundBase;
 class USoundWave;
 
@@ -17,7 +19,14 @@ enum class ELipSyncAudioDrivenMode : uint8
 {
 	EnvelopeMouthOpen UMETA(DisplayName = "Envelope Mouth Open"),
 	TranscriptVisemes UMETA(DisplayName = "Transcript Visemes"),
-	VisemeSequence UMETA(DisplayName = "Viseme Sequence")
+	VisemeSequence UMETA(DisplayName = "Viseme Sequence"),
+	StreamingAudioAnalyzer UMETA(DisplayName = "Streaming Audio Analyzer")
+};
+
+struct FLipSyncStreamingMarker
+{
+	float Time = 0.0f;
+	TMap<FName, float> VisemeWeights;
 };
 
 /**
@@ -41,9 +50,6 @@ public:
 	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync")
 	bool bAutoResolveMorphController = true;
 
-	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Smoothing", meta = (ClampMin = "0.1", DeprecatedProperty, DeprecationMessage = "Use VisemeAttackSpeed and VisemeReleaseSpeed instead."))
-	float InterpolationSpeed = 18.0f;
-
 	/** How quickly viseme weights rise toward the target. */
 	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Smoothing", meta = (ClampMin = "0.1"))
 	float VisemeAttackSpeed = 24.0f;
@@ -56,50 +62,40 @@ public:
 	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Smoothing", meta = (ClampMin = "0"))
 	float SilenceHoldTime = 0.045f;
 
-	/** Crossfades to the next audio-driven viseme instead of hard switching. 0 disables crossfade. */
-	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Smoothing", meta = (ClampMin = "0", ClampMax = "0.45"))
-	float AudioDrivenVisemeCrossfade = 0.12f;
-
 	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync", meta = (ClampMin = "0"))
 	float SilenceThreshold = 0.001f;
 
-	// --- Audio-driven lip sync ---
+	/** How much of the generated streaming marker timeline to keep behind current playback. */
+	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Streaming Analyzer", meta = (ClampMin = "0.1", ClampMax = "3.0"))
+	float StreamingAnalyzerBufferSeconds = 1.0f;
 
-	/** Default audio asset used by Play Audio Driven Lip Sync when no sound is passed explicitly. */
-	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Audio Driven")
-	TObjectPtr<USoundBase> AudioDrivenSound;
+	/** Analysis window size. 25-40 ms is a good range for speech. */
+	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Streaming Analyzer", meta = (ClampMin = "0.01", ClampMax = "0.08"))
+	float StreamingAnalyzerWindowSeconds = 0.032f;
 
-	/** Audio-only mode cannot know phonemes. Envelope mode opens/closes the mouth naturally; sequence mode cycles fake visemes for debugging. */
-	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Audio Driven")
-	ELipSyncAudioDrivenMode AudioDrivenMode = ELipSyncAudioDrivenMode::EnvelopeMouthOpen;
+	/** Extra gain applied to PCM RMS before mouth weights are generated. */
+	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Streaming Analyzer", meta = (ClampMin = "0"))
+	float StreamingAnalyzerGain = 10.0f;
 
-	/** Main viseme used by Envelope Mouth Open mode. */
-	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Audio Driven")
-	FName AudioDrivenEnvelopeViseme = TEXT("A");
+	/** Shapes mouth intensity. Lower values make quiet speech open the mouth more. */
+	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Streaming Analyzer", meta = (ClampMin = "0.1"))
+	float StreamingAnalyzerMouthGamma = 0.65f;
 
-	/** Optional transcript for Transcript Visemes mode. The component builds an approximate viseme sequence from this text. */
-	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Audio Driven", meta = (MultiLine = "true"))
-	FText AudioDrivenTranscript;
+	/** Noise gate for PCM RMS. Raise this if the mouth moves during silence. */
+	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Streaming Analyzer", meta = (ClampMin = "0", ClampMax = "0.2"))
+	float StreamingAnalyzerNoiseGate = 0.012f;
 
-	/** Viseme sequence used while audio-driven lip sync is playing. The audio envelope controls intensity. */
-	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Audio Driven")
-	TArray<FName> AudioDrivenVisemeSequence;
+	/** How long a detected plosive/closed-lip onset keeps the mouth close viseme. */
+	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Streaming Analyzer", meta = (ClampMin = "0", ClampMax = "0.2"))
+	float StreamingAnalyzerPlosiveHoldTime = 0.055f;
 
-	/** How many viseme steps per second to advance while test audio is playing. */
-	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Audio Driven", meta = (ClampMin = "0.1"))
-	float AudioDrivenVisemeRate = 9.0f;
+	/** Assign your Qwen TTS component here. The lip sync component subscribes to streamed PCM automatically. */
+	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Streaming Analyzer", meta = (DisplayName = "TTS Streaming Component"))
+	TObjectPtr<UQwenTTSStreamingComponent> TTSStreamingComponent;
 
-	/** Multiplies the audio envelope before it becomes mouth intensity. */
-	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Audio Driven", meta = (ClampMin = "0"))
-	float AudioDrivenEnvelopeGain = 5.0f;
-
-	/** Smooths low audio values into a more visible mouth movement. */
-	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Audio Driven", meta = (ClampMin = "0.1"))
-	float AudioDrivenEnvelopeGamma = 0.65f;
-
-	/** Used when neither cooked envelope data nor realtime envelope callbacks are available. Disable for production if you prefer silence over synthetic movement. */
-	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Audio Driven")
-	bool bUseFallbackPulseWithoutEnvelope = false;
+	/** If no TTS component is assigned, try to find one on the same actor during BeginPlay. */
+	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Lip Sync|Streaming Analyzer")
+	bool bAutoBindTTSStreamingComponent = true;
 
 	UFUNCTION(BlueprintCallable, Category = "FunnyGirl|Lip Sync")
 	void SetVisemeWeight(FName VisemeName, float Weight);
@@ -115,10 +111,6 @@ public:
 
 	bool PlayAudioDrivenLipSyncWithComponent(UAudioComponent* AudioComponent, USoundBase* Sound, float StartTime = 0.0f);
 
-	/** Follow an audio component that is controlled elsewhere, such as a streaming TTS component. Does not stop, replace, or start audio playback. */
-	UFUNCTION(BlueprintCallable, Category = "FunnyGirl|Lip Sync|Audio Driven", meta = (DisplayName = "Follow External Audio Component"))
-	bool FollowExternalAudioComponent(UAudioComponent* AudioComponent, const FText& Transcript);
-
 	UFUNCTION(BlueprintCallable, Category = "FunnyGirl|Lip Sync|Audio Driven", meta = (DisplayName = "Stop Audio Driven Lip Sync"))
 	void StopAudioDrivenLipSync();
 
@@ -136,8 +128,23 @@ public:
 
 	virtual void TickComponent(float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) override;
 
+	// Legacy/test audio fields are intentionally not editable or Blueprint-visible.
+	UPROPERTY()
+	TObjectPtr<USoundBase> AudioDrivenSound;
+
+	ELipSyncAudioDrivenMode AudioDrivenMode = ELipSyncAudioDrivenMode::EnvelopeMouthOpen;
+	FName AudioDrivenEnvelopeViseme = TEXT("A");
+	FText AudioDrivenTranscript;
+	TArray<FName> AudioDrivenVisemeSequence;
+	float AudioDrivenVisemeRate = 9.0f;
+	float AudioDrivenEnvelopeGain = 5.0f;
+	float AudioDrivenEnvelopeGamma = 0.65f;
+	float AudioDrivenVisemeCrossfade = 0.12f;
+	bool bUseFallbackPulseWithoutEnvelope = false;
+
 protected:
 	virtual void BeginPlay() override;
+	virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
 
 private:
 	UPROPERTY()
@@ -162,10 +169,28 @@ private:
 
 	TArray<FName> GeneratedTranscriptVisemes;
 	TArray<float> GeneratedTranscriptVisemeUnits;
+	TArray<FLipSyncStreamingMarker> StreamingAnalyzerMarkers;
+	int32 StreamingAnalyzerSampleRate = 0;
+	float StreamingAnalyzerPlosiveHoldRemaining = 0.0f;
+	float StreamingAnalyzerPreviousRms = 0.0f;
+	float StreamingAnalyzerShapePhase = 0.0f;
+	float StreamingAnalyzerQueuedDuration = 0.0f;
+	int32 StreamingAnalyzerMarkerIndex = 0;
+	bool bStreamingAnalyzerPlaybackActive = false;
+	bool bBoundToTTSStreamingComponent = false;
 
 	UMorphControllerComponent* ResolveMorphController() const;
 	UAudioComponent* ResolveAudioDrivenComponent();
+	bool BindToTTSStreamingComponent(UQwenTTSStreamingComponent* InTTSStreamingComponent);
+	void UnbindFromTTSStreamingComponent();
+	bool StartStreamingAudioAnalyzer(UAudioComponent* AudioComponent = nullptr);
+	void PushAudioSamplesPCM16Bytes(const TArray<uint8>& PCMBytes, int32 SampleRate, int32 NumChannels = 1);
+	void PushAudioSamplesFloat(const TArray<float>& Samples, int32 SampleRate, int32 NumChannels = 1);
 	void UpdateAudioDrivenLipSync(float DeltaTime);
+	void UpdateStreamingAudioAnalyzer(float DeltaTime);
+	void AppendStreamingAnalyzerMarkers(const TArray<float>& MonoSamples, int32 SampleRate);
+	void ResetStreamingAnalyzerTimeline();
+	void ApplyStreamingAnalyzerMarkerAtTime(float PlaybackTime);
 	float GetAudioDrivenMouthWeight(float DeltaTime);
 	void RebuildGeneratedTranscriptVisemes();
 	const TArray<FName>& ResolveActiveAudioDrivenVisemeSequence() const;
@@ -181,4 +206,13 @@ private:
 
 	UFUNCTION()
 	void HandleAudioDrivenEnvelopeValue(const USoundWave* PlayingSoundWave, const float EnvelopeValue);
+
+	UFUNCTION()
+	void HandleTTSStreamStarted();
+
+	UFUNCTION()
+	void HandleTTSAudioChunk(const FQwenTTSStreamChunk& Chunk);
+
+	UFUNCTION()
+	void HandleTTSStreamFinished();
 };