// SimpleRecorder.cpp — Full implementation // Video: back-buffer → raw BGRA → FFmpeg pipe → NVENC → video_only.mp4 // Audio: Submix listener → float PCM → WAV file → audio_only.wav // Mux: FFmpeg CLI → final_video_with_audio.mp4 #include "SimpleRecorder.h" #include "AudioVideoRecord.h" // For LogAudioVideoRecord #include "AudioDevice.h" #include "AudioMixerDevice.h" #include "Engine/GameViewportClient.h" #include "Framework/Application/SlateApplication.h" #include "Misc/FileHelper.h" #include "Misc/Paths.h" #include "HAL/PlatformFileManager.h" #include "HAL/PlatformProcess.h" #include "RenderingThread.h" #include "RHI.h" #include "RHICommandList.h" #include "RHIResources.h" #include "Serialization/Archive.h" // ===================================================================== // FSubmixBridge — wraps a callback in a TSharedRef for UE 5.6 API // UObjects can't use AsShared(), so we need this bridge. // ===================================================================== class FSimpleRecorderSubmixBridge : public ISubmixBufferListener { public: TWeakObjectPtr Owner; FSimpleRecorderSubmixBridge(USimpleRecorder* InOwner) : Owner(InOwner) {} virtual void OnNewSubmixBuffer( const USoundSubmix* OwningSubmix, float* AudioData, int32 NumSamples, int32 NumChannels, const int32 SampleRate, double AudioClock) override { if (USimpleRecorder* Rec = Owner.Get()) { Rec->OnNewSubmixBuffer(OwningSubmix, AudioData, NumSamples, NumChannels, SampleRate, AudioClock); } } }; // ===================================================================== // Constructor // ===================================================================== USimpleRecorder::USimpleRecorder() { // Default output directory: /Saved/Recordings OutputDirectory = FPaths::Combine(FPaths::ProjectSavedDir(), TEXT("Recordings")); } // ===================================================================== // BeginDestroy — safety net: stop recording if object is garbage-collected // ===================================================================== void USimpleRecorder::BeginDestroy() { if (bIsRecording) { StopRecording(); } Super::BeginDestroy(); } // ===================================================================== // Helpers // ===================================================================== /** Resolve the ffmpeg executable path. */ FString USimpleRecorder::GetFFmpegExecutable() const { if (!FFmpegPath.IsEmpty()) { return FFmpegPath; } // Assume ffmpeg is on the system PATH return TEXT("ffmpeg"); } /** Set up output file paths and make sure the directory exists. */ void USimpleRecorder::InitOutputPaths() { // Ensure absolute path FString Dir = OutputDirectory; if (FPaths::IsRelative(Dir)) { Dir = FPaths::ConvertRelativePathToFull(Dir); } // Create directory if it doesn't exist IPlatformFile& PlatformFile = FPlatformFileManager::Get().GetPlatformFile(); if (!PlatformFile.DirectoryExists(*Dir)) { PlatformFile.CreateDirectoryTree(*Dir); } VideoFilePath = FPaths::Combine(Dir, TEXT("video_only.mp4")); AudioFilePath = FPaths::Combine(Dir, TEXT("audio_only.wav")); FinalFilePath = FPaths::Combine(Dir, TEXT("final_video_with_audio.mp4")); } // ===================================================================== // StartRecording // ===================================================================== void USimpleRecorder::StartRecording() { if (bIsRecording) { UE_LOG(LogAudioVideoRecord, Warning, TEXT("SimpleRecorder: Already recording!")); return; } InitOutputPaths(); bIsRecording = true; // ─── 0. Auto-detect viewport size if not explicitly set ───────── if (bAutoDetectResolution) { if (GEngine && GEngine->GameViewport) { FVector2D ViewportSize; GEngine->GameViewport->GetViewportSize(ViewportSize); if (ViewportSize.X > 0 && ViewportSize.Y > 0) { CaptureWidth = FMath::RoundToInt32(ViewportSize.X); CaptureHeight = FMath::RoundToInt32(ViewportSize.Y); // Ensure even dimensions (required by most video encoders) CaptureWidth = CaptureWidth & ~1; CaptureHeight = CaptureHeight & ~1; } } } UE_LOG(LogAudioVideoRecord, Log, TEXT("SimpleRecorder: Starting recording...")); UE_LOG(LogAudioVideoRecord, Log, TEXT(" Resolution: %dx%d @ %d FPS"), CaptureWidth, CaptureHeight, CaptureFPS); UE_LOG(LogAudioVideoRecord, Log, TEXT(" Video → %s"), *VideoFilePath); UE_LOG(LogAudioVideoRecord, Log, TEXT(" Audio → %s"), *AudioFilePath); UE_LOG(LogAudioVideoRecord, Log, TEXT(" Final → %s"), *FinalFilePath); // ─── 1. Open FFmpeg video pipe ────────────────────────────────── { FString FFmpeg = GetFFmpegExecutable(); // Build command: // ffmpeg -y -f rawvideo -pix_fmt bgra // -video_size WxH -framerate FPS // -i - (stdin) // -c:v ... -b:v BITRATE // output.mp4 // // Encoder selection per platform: // Mac: h264_videotoolbox (Apple HW encoder) // Windows: h264_nvenc / h264_amf / h264_qsv (detected at runtime) // Linux: libx264 (software fallback) #if PLATFORM_MAC FString EncoderFlags = FString::Printf(TEXT("-c:v h264_videotoolbox -b:v %s"), *VideoBitrate); #elif PLATFORM_WINDOWS FString EncoderFlags = FString::Printf(TEXT("-c:v h264_nvenc -preset p5 -b:v %s"), *VideoBitrate); #else FString EncoderFlags = FString::Printf(TEXT("-c:v libx264 -preset medium -b:v %s"), *VideoBitrate); #endif FString Cmd = FString::Printf( TEXT("\"%s\" -y -f rawvideo -pix_fmt bgra -video_size %dx%d -framerate %d ") TEXT("-i - %s \"%s\""), *FFmpeg, CaptureWidth, CaptureHeight, CaptureFPS, *EncoderFlags, *VideoFilePath ); UE_LOG(LogAudioVideoRecord, Log, TEXT(" FFmpeg cmd: %s"), *Cmd); // popen opens a pipe — we write raw BGRA bytes into FFmpeg's stdin #if PLATFORM_WINDOWS FFmpegVideoPipe = _popen(TCHAR_TO_ANSI(*Cmd), "wb"); #else FFmpegVideoPipe = popen(TCHAR_TO_ANSI(*Cmd), "w"); #endif if (!FFmpegVideoPipe) { UE_LOG(LogAudioVideoRecord, Error, TEXT("SimpleRecorder: Failed to open FFmpeg pipe! Make sure ffmpeg.exe is on your PATH.")); bIsRecording = false; return; } } // ─── 2. Register back-buffer delegate (video frames) ──────────── if (FSlateApplication::IsInitialized()) { BackBufferDelegateHandle = FSlateApplication::Get().GetRenderer()->OnBackBufferReadyToPresent() .AddUObject(this, &USimpleRecorder::OnBackBufferReady); } // ─── 3. Start audio submix recording ──────────────────────────── { // Clear any leftover audio data FScopeLock Lock(&AudioBufferCritSection); AudioBuffer.Empty(); } if (GEngine && GEngine->GetMainAudioDevice()) { FAudioDevice* AudioDevice = GEngine->GetMainAudioDevice().GetAudioDevice(); if (AudioDevice) { // If user didn't set a submix, use the master submix USoundSubmix* Submix = TargetSubmix; if (!Submix) { // Get the main submix from the audio device Submix = &AudioDevice->GetMainSubmixObject(); } if (Submix) { SubmixBridge = MakeShared(this); AudioDevice->RegisterSubmixBufferListener(SubmixBridge.ToSharedRef(), *Submix); UE_LOG(LogAudioVideoRecord, Log, TEXT(" Audio submix listener registered.")); } else { UE_LOG(LogAudioVideoRecord, Warning, TEXT("SimpleRecorder: Could not find a submix to record audio from.")); } } } UE_LOG(LogAudioVideoRecord, Log, TEXT("SimpleRecorder: Recording started.")); } // ===================================================================== // StopRecording // ===================================================================== void USimpleRecorder::StopRecording() { if (!bIsRecording) { UE_LOG(LogAudioVideoRecord, Warning, TEXT("SimpleRecorder: Not recording.")); return; } bIsRecording = false; UE_LOG(LogAudioVideoRecord, Log, TEXT("SimpleRecorder: Stopping recording...")); // ─── 1. Unregister back-buffer delegate ───────────────────────── if (FSlateApplication::IsInitialized() && BackBufferDelegateHandle.IsValid()) { FSlateApplication::Get().GetRenderer()->OnBackBufferReadyToPresent() .Remove(BackBufferDelegateHandle); BackBufferDelegateHandle.Reset(); } // ─── 2. Close the FFmpeg video pipe ───────────────────────────── if (FFmpegVideoPipe) { #if PLATFORM_WINDOWS _pclose(FFmpegVideoPipe); #else pclose(FFmpegVideoPipe); #endif FFmpegVideoPipe = nullptr; UE_LOG(LogAudioVideoRecord, Log, TEXT(" FFmpeg video pipe closed → video_only.mp4 written.")); } // ─── 3. Unregister audio submix listener ──────────────────────── if (GEngine && GEngine->GetMainAudioDevice()) { FAudioDevice* AudioDevice = GEngine->GetMainAudioDevice().GetAudioDevice(); if (AudioDevice) { USoundSubmix* Submix = TargetSubmix; if (!Submix) { Submix = &AudioDevice->GetMainSubmixObject(); } if (Submix) { if (SubmixBridge.IsValid()) { AudioDevice->UnregisterSubmixBufferListener(SubmixBridge.ToSharedRef(), *Submix); SubmixBridge.Reset(); } } } } // ─── 4. Save audio to .wav ────────────────────────────────────── SaveAudioToWav(); // ─── 5. Mux video + audio → final mp4 ────────────────────────── MuxAudioVideo(); UE_LOG(LogAudioVideoRecord, Log, TEXT("SimpleRecorder: Recording stopped. All files saved.")); } // ===================================================================== // OnBackBufferReady — called every presented frame on the RENDER thread // ===================================================================== void USimpleRecorder::OnBackBufferReady(SWindow& SlateWindow, const FTextureRHIRef& BackBuffer) { // Safety: if we already stopped, do nothing if (!bIsRecording || !FFmpegVideoPipe) { return; } // We're on the render thread here. // Read the back-buffer pixels into a CPU-side array. FRHICommandListImmediate& RHICmdList = FRHICommandListImmediate::Get(); // Determine actual back-buffer size (may differ from our target) const FIntPoint BBSize = BackBuffer->GetSizeXY(); // We need a rect matching our capture resolution, clamped to back-buffer const int32 ReadW = FMath::Min(CaptureWidth, (int32)BBSize.X); const int32 ReadH = FMath::Min(CaptureHeight, (int32)BBSize.Y); FIntRect ReadRect(0, 0, ReadW, ReadH); // ReadSurfaceData puts pixels into a TArray (BGRA, 8-bit per channel) TArray Pixels; Pixels.SetNumUninitialized(ReadW * ReadH); RHICmdList.ReadSurfaceData( BackBuffer.GetReference(), ReadRect, Pixels, FReadSurfaceDataFlags(RCM_UNorm) ); // If the back-buffer is smaller than our target, we still write the full // frame size so FFmpeg doesn't choke. Pad with black if needed. if (ReadW == CaptureWidth && ReadH == CaptureHeight) { // Fast path — dimensions match exactly fwrite(Pixels.GetData(), sizeof(FColor), Pixels.Num(), FFmpegVideoPipe); } else { // Slow path — need to pad to CaptureWidth x CaptureHeight // Allocate a black frame TArray PaddedFrame; PaddedFrame.SetNumZeroed(CaptureWidth * CaptureHeight); for (int32 Row = 0; Row < ReadH; ++Row) { FMemory::Memcpy( &PaddedFrame[Row * CaptureWidth], &Pixels[Row * ReadW], ReadW * sizeof(FColor)); } fwrite(PaddedFrame.GetData(), sizeof(FColor), PaddedFrame.Num(), FFmpegVideoPipe); } } // ===================================================================== // ISubmixBufferListener — audio callback (called from the audio thread) // ===================================================================== void USimpleRecorder::OnNewSubmixBuffer( const USoundSubmix* OwningSubmix, float* AudioData, int32 NumSamples, int32 NumChannels, const int32 SampleRate, double AudioClock) { if (!bIsRecording) { return; } FScopeLock Lock(&AudioBufferCritSection); // Store the format (may already be set, but harmless to update) AudioSampleRate = SampleRate; AudioNumChannels = NumChannels; // Append the incoming samples AudioBuffer.Append(AudioData, NumSamples); } // ===================================================================== // SaveAudioToWav — writes accumulated PCM float data as a 16-bit WAV // ===================================================================== void USimpleRecorder::SaveAudioToWav() { FScopeLock Lock(&AudioBufferCritSection); if (AudioBuffer.Num() == 0) { UE_LOG(LogAudioVideoRecord, Warning, TEXT("SimpleRecorder: No audio data captured.")); return; } UE_LOG(LogAudioVideoRecord, Log, TEXT(" Saving audio: %d samples, %d channels, %d Hz"), AudioBuffer.Num(), AudioNumChannels, AudioSampleRate); // Convert float [-1,1] → int16 TArray PCM16; PCM16.SetNumUninitialized(AudioBuffer.Num()); for (int32 i = 0; i < AudioBuffer.Num(); ++i) { float Clamped = FMath::Clamp(AudioBuffer[i], -1.0f, 1.0f); PCM16[i] = static_cast(Clamped * 32767.0f); } // ── WAV header ────────────────────────────────────────────────── const int32 BitsPerSample = 16; const int32 BytesPerSample = BitsPerSample / 8; const int32 DataSize = PCM16.Num() * BytesPerSample; const int32 ByteRate = AudioSampleRate * AudioNumChannels * BytesPerSample; const int16 BlockAlign = static_cast(AudioNumChannels * BytesPerSample); TArray WavFile; // Reserve enough space: 44-byte header + data WavFile.Reserve(44 + DataSize); // Helper lambdas to append little-endian integers auto Write4CC = [&](const char* FourCC) { WavFile.Append(reinterpret_cast(FourCC), 4); }; auto WriteInt32 = [&](int32 Val) { WavFile.Append(reinterpret_cast(&Val), 4); }; auto WriteInt16 = [&](int16 Val) { WavFile.Append(reinterpret_cast(&Val), 2); }; // RIFF header Write4CC("RIFF"); WriteInt32(36 + DataSize); // File size - 8 Write4CC("WAVE"); // fmt sub-chunk Write4CC("fmt "); WriteInt32(16); // Sub-chunk size (PCM) WriteInt16(1); // Audio format = PCM WriteInt16(static_cast(AudioNumChannels)); WriteInt32(AudioSampleRate); WriteInt32(ByteRate); WriteInt16(BlockAlign); WriteInt16(static_cast(BitsPerSample)); // data sub-chunk Write4CC("data"); WriteInt32(DataSize); WavFile.Append(reinterpret_cast(PCM16.GetData()), DataSize); // Write to disk if (FFileHelper::SaveArrayToFile(WavFile, *AudioFilePath)) { UE_LOG(LogAudioVideoRecord, Log, TEXT(" audio_only.wav saved (%d bytes)."), WavFile.Num()); } else { UE_LOG(LogAudioVideoRecord, Error, TEXT(" Failed to save audio_only.wav!")); } // Free memory AudioBuffer.Empty(); } // ===================================================================== // MuxAudioVideo — runs FFmpeg to combine video + audio into final mp4 // ===================================================================== void USimpleRecorder::MuxAudioVideo() { FString FFmpeg = GetFFmpegExecutable(); // ffmpeg -y -i video_only.mp4 -i audio_only.wav -c:v copy -c:a aac -shortest final.mp4 FString Cmd = FString::Printf( TEXT("\"%s\" -y -i \"%s\" -i \"%s\" -c:v copy -c:a aac -shortest \"%s\""), *FFmpeg, *VideoFilePath, *AudioFilePath, *FinalFilePath ); UE_LOG(LogAudioVideoRecord, Log, TEXT(" Mux cmd: %s"), *Cmd); // Run synchronously (blocks until done) int32 ReturnCode = -1; FString StdOut; FString StdErr; FPlatformProcess::ExecProcess( *FFmpeg, *FString::Printf( TEXT("-y -i \"%s\" -i \"%s\" -c:v copy -c:a aac -shortest \"%s\""), *VideoFilePath, *AudioFilePath, *FinalFilePath), &ReturnCode, &StdOut, &StdErr ); if (ReturnCode == 0) { UE_LOG(LogAudioVideoRecord, Log, TEXT(" Mux complete → final_video_with_audio.mp4 saved.")); } else { UE_LOG(LogAudioVideoRecord, Error, TEXT(" Mux failed (code %d). stderr: %s"), ReturnCode, *StdErr); } }