Linux video pipeline: GPU-rendered capture, tearing and audio fixes

SerialPrograms/CMakeLists.txt

@@ -562,7 +562,7 @@ else() # macOS and Linux
     endif() # end Linux
 
     # Find OpenCV
-    pkg_search_module(OpenCV REQUIRED opencv4 opencv)
+    pkg_search_module(OpenCV REQUIRED opencv5 opencv4 opencv)
     target_include_directories(SerialProgramsLib SYSTEM PRIVATE ${OpenCV_INCLUDE_DIRS}) # "SYSTEM" to suppress warnings
     target_link_directories(SerialProgramsLib PUBLIC ${OpenCV_LIBRARY_DIRS})
     target_link_libraries(SerialProgramsLib PUBLIC ${OpenCV_LINK_LIBRARIES})

SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.cpp

@@ -47,6 +47,7 @@ AudioSession::AudioSession(Logger& logger, AudioOption& option)
      : m_logger(logger)
      , m_option(option)
      , m_devices(new AudioPassthroughPairQtThread(logger))
+     , m_consecutive_timeouts(0)
 {
     AudioSession::reset();
     m_devices->add_listener(*this);
@@ -74,6 +75,7 @@ void AudioSession::save(AudioOption& option) const{
 void AudioSession::load(const AudioOption& option){
     {
         std::lock_guard<Mutex> lg(m_lock);
+        m_consecutive_timeouts.store(0, std::memory_order_relaxed);
         signal_pre_input_change();
 
         m_option.m_input_file       = option.m_input_file;
@@ -124,6 +126,7 @@ AudioOption::AudioDisplayType AudioSession::display_type() const{
 
 void AudioSession::clear_audio_input(){
     std::lock_guard<Mutex> lg(m_lock);
+    m_consecutive_timeouts.store(0, std::memory_order_relaxed);
     m_logger.log("Clearing audio input...");
     signal_pre_input_change();
     m_devices->clear_audio_source();
@@ -141,6 +144,7 @@ void AudioSession::clear_audio_input(){
 }
 void AudioSession::set_audio_input(std::string file){
     std::lock_guard<Mutex> lg(m_lock);
+    m_consecutive_timeouts.store(0, std::memory_order_relaxed);
     signal_pre_input_change();
     m_option.m_input_file = std::move(file);
     sanitize_format();
@@ -149,6 +153,7 @@ void AudioSession::set_audio_input(std::string file){
 }
 void AudioSession::set_audio_input(AudioDeviceInfo info){
     std::lock_guard<Mutex> lg(m_lock);
+    m_consecutive_timeouts.store(0, std::memory_order_relaxed);
     m_logger.log("Setting audio input to: " + info.display_name());
     signal_pre_input_change();
     m_option.m_input_file.clear();
@@ -163,6 +168,7 @@ void AudioSession::set_audio_input(AudioDeviceInfo info){
 }
 void AudioSession::set_format(AudioChannelFormat format){
     std::lock_guard<Mutex> lg(m_lock);
+    m_consecutive_timeouts.store(0, std::memory_order_relaxed);
     signal_pre_input_change();
     m_option.m_input_format = format;
     if (sanitize_format()){
@@ -289,20 +295,54 @@ void AudioSession::add_overlay(uint64_t starting_seqnum, size_t end_seqnum, Colo
 
 void AudioSession::on_fft(size_t sample_rate, std::shared_ptr<const AlignedVector<float>> fft_output){
     m_spectrum_holder.push_spectrum(sample_rate, std::move(fft_output));
+    m_consecutive_timeouts.store(0, std::memory_order_relaxed);
     global_watchdog().delay(*this);
 }
+
+
+const size_t AUDIO_RESET_FAST_ATTEMPTS = 3;
+const size_t AUDIO_RESET_SLOW_STRIDE = 12;
+
+
 void AudioSession::on_watchdog_timeout(){
 //    m_logger.log("AudioSession::on_watchdog_timeout()", COLOR_RED);
     if (m_option.m_input_file.empty() && !m_option.m_input_device){
         return;
     }
 
     uint8_t watchdog_timeout = GlobalSettings::instance().AUDIO_PIPELINE->AUTO_RESET_SECONDS;
-    m_logger.log("No audio detected for " + std::to_string(watchdog_timeout) + " seconds...", COLOR_RED);
+    size_t failures = m_consecutive_timeouts.fetch_add(1, std::memory_order_relaxed) + 1;
 
     if (watchdog_timeout == 0){
+        //  Log once so that a deliberately silent setup doesn't fill the log.
+        if (failures == 1){
+            m_logger.log("No audio detected for " + std::to_string(watchdog_timeout) + " seconds...", COLOR_RED);
+        }
         return;
     }
+
+    if (failures > AUDIO_RESET_FAST_ATTEMPTS){
+        if (failures == AUDIO_RESET_FAST_ATTEMPTS + 1){
+            m_logger.log(
+                "Audio has not recovered after " + std::to_string(AUDIO_RESET_FAST_ATTEMPTS) +
+                " resets. Backing off to one attempt per " +
+                std::to_string(watchdog_timeout * AUDIO_RESET_SLOW_STRIDE) + " seconds.",
+                COLOR_RED
+            );
+            m_logger.log(
+                "If the audio and the video come from the same capture card, some cards "
+                "cannot start their audio stream while their video stream is already "
+                "running. Select the audio input device before starting the video, or "
+                "unplug and replug the card.",
+                COLOR_RED
+            );
+        }
+        if ((failures - AUDIO_RESET_FAST_ATTEMPTS) % AUDIO_RESET_SLOW_STRIDE != 0){
+            return;
+        }
+    }
+
+    m_logger.log("No audio detected for " + std::to_string(watchdog_timeout) + " seconds...", COLOR_RED);
     m_logger.log("Resetting the audio...", COLOR_GREEN);
     reset();
 }

SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.h

@@ -16,6 +16,7 @@
 #ifndef PokemonAutomation_AudioPipeline_AudioSession_H
 #define PokemonAutomation_AudioPipeline_AudioSession_H
 
+#include <atomic>
 #include "Common/Cpp/ListenerSet.h"
 #include "Common/Cpp/Concurrency/Watchdog.h"
 #include "AudioFeed.h"
@@ -101,6 +102,8 @@ class AudioSession final : public AudioFeed, private FFTListener, private Watchd
 
     mutable Mutex m_lock;
 
+    std::atomic<size_t> m_consecutive_timeouts;
+
     ListenerSet<StateListener> m_listeners;
 
 };

SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioSink.cpp

@@ -45,8 +45,11 @@ const char* audio_error_to_str(QAudio::Error error){
         return "OpenError (device cannot be opened with the requested format)";
     case QAudio::IOError:
         return "IOError";
+#if QT_VERSION < QT_VERSION_CHECK(6, 11, 0)
+    //  Deprecated in Qt 6.11, which no longer emits it.
     case QAudio::UnderrunError:
         return "UnderrunError";
+#endif
     case QAudio::FatalError:
         return "FatalError";
     default:

SerialPrograms/Source/CommonFramework/Main.cpp

@@ -1,9 +1,11 @@
 
 #include <QDir>
 #include <QApplication>
+#include <QSurfaceFormat>
 #include <QFileInfo>
 //#include <QTextStream>
 #include <QMessageBox>
+#include <fstream>
 #include "Common/Cpp/Logging/LastLogTracker.h"
 #include "Common/Cpp/Logging/FileLogger.h"
 #include "Common/Cpp/Logging/GlobalLogger.h"
@@ -87,6 +89,35 @@ FileLogger& global_file_logger(){
 
 
 
+#if defined(__linux__)
+//  With nodrop=1 (the kernel default), uvcvideo delivers frames that lost USB
+//  packets instead of dropping them. The missing rows are stale, which shows up
+//  as a horizontal tear that detection reads as a normal frame.
+void warn_if_uvcvideo_delivers_torn_frames(Logger& logger){
+    std::ifstream file("/sys/module/uvcvideo/parameters/nodrop");
+    if (!file){
+        return;
+    }
+    std::string value;
+    std::getline(file, value);
+    while (!value.empty() && (value.back() == '\r' || value.back() == ' ')){
+        value.pop_back();
+    }
+    if (value != "1"){
+        return;
+    }
+    logger.log(
+        "uvcvideo is set to nodrop=1. Frames that lose USB packets will be delivered "
+        "partially written instead of dropped: part of the image will be stale. This "
+        "is visible as a horizontal tear and is read as-is by detection. To fix it:  "
+        "echo 0 | sudo tee /sys/module/uvcvideo/parameters/nodrop  (to persist:  "
+        "echo \"options uvcvideo nodrop=0\" | sudo tee /etc/modprobe.d/uvcvideo.conf)",
+        COLOR_RED
+    );
+}
+#endif
+
+
 int run_program(int argc, char *argv[]){
 //    qputenv("QT_RHI_BACKEND", "opengl");
 
@@ -121,6 +152,10 @@ int run_program(int argc, char *argv[]){
     logger.log("Program setting folder: " + SETTINGS_PATH());
     logger.log("Program resources folder: " + RESOURCE_PATH());
 
+#if defined(__linux__)
+    warn_if_uvcvideo_delivers_torn_frames(logger);
+#endif
+
     qRegisterMetaType<size_t>("size_t");
     qRegisterMetaType<uint8_t>("uint8_t");
     qRegisterMetaType<std::string>("std::string");
@@ -247,6 +282,23 @@ int main(int argc, char *argv[]){
     if (qEnvironmentVariableIsEmpty("QT_MEDIA_BACKEND")){
         qputenv("QT_MEDIA_BACKEND", "gstreamer");
     }
+
+    //  Use mmap instead of dmabuf. With dmabuf, Qt imports capture buffers as
+    //  EGLImages, and drivers that reject the fourcc (NVIDIA with YUYV) render
+    //  a flat color without reporting an error.
+    if (qEnvironmentVariableIsEmpty("QT_GSTREAMER_V4L2SRC_IOMODE")){
+        qputenv("QT_GSTREAMER_V4L2SRC_IOMODE", "2");
+    }
+#endif
+
+#if defined(__linux__)
+    //  Force vsync, or video tears under Wayland. Must be set before the
+    //  QApplication is constructed.
+    {
+        QSurfaceFormat format = QSurfaceFormat::defaultFormat();
+        format.setSwapInterval(1);
+        QSurfaceFormat::setDefaultFormat(format);
+    }
 #endif
 
     //  So far, this is only needed on Mac where static initialization is fucked up.

SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.cpp

@@ -19,6 +19,12 @@
 #include "CameraWidgetQt6.h"
 #include "CameraWidgetQt6.5.h"
 #include "CameraWidgetQt6_QQuickWidget.h"
+#if defined(__linux__)
+#include "CameraWidgetQt6_QML.h"
+#include "CameraWidgetQt6_QVideoWidget.h"
+#include "CameraWidgetOpenCV_QOpenGLWidget.h"
+#include "CameraWidgetOpenCV.h"
+#endif
 
 
 namespace PokemonAutomation{
@@ -56,7 +62,30 @@ struct CameraBackends{
             "qt6.5-QGraphicsScene", "Qt6.5: QGraphicsScene",
             std::make_unique<CameraQt65QMediaCaptureSession::CameraBackend>()
         );
-#ifdef _WIN32
+#if defined(__linux__)
+        //  The other Qt backends tear under Wayland.
+        m_backends.emplace_back(
+            "qt6-QQuickWidget", "Qt6: QQuickWidget",
+            std::make_unique<CameraQt6_QQuickWidget::CameraBackend>()
+        );
+
+        m_backends.emplace_back(
+            "qt6-QML", "Qt6: QML VideoOutput",
+            std::make_unique<CameraQt6QML::CameraBackend>()
+        );
+        m_backends.emplace_back(
+            "qt6-QVideoWidget", "Qt6: QOpenGLWidget",
+            std::make_unique<CameraQt6QVideoWidget::CameraBackend>()
+        );
+        m_backends.emplace_back(
+            "opencv-QOpenGLWidget", "OpenCV: QOpenGLWidget",
+            std::make_unique<CameraOpenCV_QOpenGLWidget::CameraBackend>()
+        );
+        m_backends.emplace_back(
+            "opencv-QWidget", "OpenCV: QWidget (CPU)",
+            std::make_unique<CameraOpenCV::CameraBackend>()
+        );
+#elif defined(_WIN32)
         m_backends.emplace_back(
             "qt6-QQuickWidget", "Qt6: QQuickWidget",
             std::make_unique<CameraQt6_QQuickWidget::CameraBackend>()