diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index 774d2ae41c..385a6d8adb 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -562,7 +562,7 @@ else() # macOS and Linux endif() # end Linux # Find OpenCV - pkg_search_module(OpenCV REQUIRED opencv4 opencv) + pkg_search_module(OpenCV REQUIRED opencv5 opencv4 opencv) target_include_directories(SerialProgramsLib SYSTEM PRIVATE ${OpenCV_INCLUDE_DIRS}) # "SYSTEM" to suppress warnings target_link_directories(SerialProgramsLib PUBLIC ${OpenCV_LIBRARY_DIRS}) target_link_libraries(SerialProgramsLib PUBLIC ${OpenCV_LINK_LIBRARIES}) diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.cpp b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.cpp index 5483a94c9d..6c5544ee71 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.cpp +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.cpp @@ -47,6 +47,7 @@ AudioSession::AudioSession(Logger& logger, AudioOption& option) : m_logger(logger) , m_option(option) , m_devices(new AudioPassthroughPairQtThread(logger)) + , m_consecutive_timeouts(0) { AudioSession::reset(); m_devices->add_listener(*this); @@ -74,6 +75,7 @@ void AudioSession::save(AudioOption& option) const{ void AudioSession::load(const AudioOption& option){ { std::lock_guard lg(m_lock); + m_consecutive_timeouts.store(0, std::memory_order_relaxed); signal_pre_input_change(); m_option.m_input_file = option.m_input_file; @@ -124,6 +126,7 @@ AudioOption::AudioDisplayType AudioSession::display_type() const{ void AudioSession::clear_audio_input(){ std::lock_guard lg(m_lock); + m_consecutive_timeouts.store(0, std::memory_order_relaxed); m_logger.log("Clearing audio input..."); signal_pre_input_change(); m_devices->clear_audio_source(); @@ -141,6 +144,7 @@ void AudioSession::clear_audio_input(){ } void AudioSession::set_audio_input(std::string file){ std::lock_guard lg(m_lock); + m_consecutive_timeouts.store(0, std::memory_order_relaxed); signal_pre_input_change(); m_option.m_input_file = std::move(file); sanitize_format(); @@ -149,6 +153,7 @@ void AudioSession::set_audio_input(std::string file){ } void AudioSession::set_audio_input(AudioDeviceInfo info){ std::lock_guard lg(m_lock); + m_consecutive_timeouts.store(0, std::memory_order_relaxed); m_logger.log("Setting audio input to: " + info.display_name()); signal_pre_input_change(); m_option.m_input_file.clear(); @@ -163,6 +168,7 @@ void AudioSession::set_audio_input(AudioDeviceInfo info){ } void AudioSession::set_format(AudioChannelFormat format){ std::lock_guard lg(m_lock); + m_consecutive_timeouts.store(0, std::memory_order_relaxed); signal_pre_input_change(); m_option.m_input_format = format; if (sanitize_format()){ @@ -289,8 +295,15 @@ void AudioSession::add_overlay(uint64_t starting_seqnum, size_t end_seqnum, Colo void AudioSession::on_fft(size_t sample_rate, std::shared_ptr> fft_output){ m_spectrum_holder.push_spectrum(sample_rate, std::move(fft_output)); + m_consecutive_timeouts.store(0, std::memory_order_relaxed); global_watchdog().delay(*this); } + + +const size_t AUDIO_RESET_FAST_ATTEMPTS = 3; +const size_t AUDIO_RESET_SLOW_STRIDE = 12; + + void AudioSession::on_watchdog_timeout(){ // m_logger.log("AudioSession::on_watchdog_timeout()", COLOR_RED); if (m_option.m_input_file.empty() && !m_option.m_input_device){ @@ -298,11 +311,38 @@ void AudioSession::on_watchdog_timeout(){ } uint8_t watchdog_timeout = GlobalSettings::instance().AUDIO_PIPELINE->AUTO_RESET_SECONDS; - m_logger.log("No audio detected for " + std::to_string(watchdog_timeout) + " seconds...", COLOR_RED); + size_t failures = m_consecutive_timeouts.fetch_add(1, std::memory_order_relaxed) + 1; if (watchdog_timeout == 0){ + // Log once so that a deliberately silent setup doesn't fill the log. + if (failures == 1){ + m_logger.log("No audio detected for " + std::to_string(watchdog_timeout) + " seconds...", COLOR_RED); + } return; } + + if (failures > AUDIO_RESET_FAST_ATTEMPTS){ + if (failures == AUDIO_RESET_FAST_ATTEMPTS + 1){ + m_logger.log( + "Audio has not recovered after " + std::to_string(AUDIO_RESET_FAST_ATTEMPTS) + + " resets. Backing off to one attempt per " + + std::to_string(watchdog_timeout * AUDIO_RESET_SLOW_STRIDE) + " seconds.", + COLOR_RED + ); + m_logger.log( + "If the audio and the video come from the same capture card, some cards " + "cannot start their audio stream while their video stream is already " + "running. Select the audio input device before starting the video, or " + "unplug and replug the card.", + COLOR_RED + ); + } + if ((failures - AUDIO_RESET_FAST_ATTEMPTS) % AUDIO_RESET_SLOW_STRIDE != 0){ + return; + } + } + + m_logger.log("No audio detected for " + std::to_string(watchdog_timeout) + " seconds...", COLOR_RED); m_logger.log("Resetting the audio...", COLOR_GREEN); reset(); } diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.h b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.h index 52ec946df1..8c93aa1661 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.h +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.h @@ -16,6 +16,7 @@ #ifndef PokemonAutomation_AudioPipeline_AudioSession_H #define PokemonAutomation_AudioPipeline_AudioSession_H +#include #include "Common/Cpp/ListenerSet.h" #include "Common/Cpp/Concurrency/Watchdog.h" #include "AudioFeed.h" @@ -101,6 +102,8 @@ class AudioSession final : public AudioFeed, private FFTListener, private Watchd mutable Mutex m_lock; + std::atomic m_consecutive_timeouts; + ListenerSet m_listeners; }; diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioSink.cpp b/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioSink.cpp index 87abefaef4..26f7445c97 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioSink.cpp +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioSink.cpp @@ -45,8 +45,11 @@ const char* audio_error_to_str(QAudio::Error error){ return "OpenError (device cannot be opened with the requested format)"; case QAudio::IOError: return "IOError"; +#if QT_VERSION < QT_VERSION_CHECK(6, 11, 0) + // Deprecated in Qt 6.11, which no longer emits it. case QAudio::UnderrunError: return "UnderrunError"; +#endif case QAudio::FatalError: return "FatalError"; default: diff --git a/SerialPrograms/Source/CommonFramework/Main.cpp b/SerialPrograms/Source/CommonFramework/Main.cpp index 1c70b1eea5..c56e9b7bc0 100644 --- a/SerialPrograms/Source/CommonFramework/Main.cpp +++ b/SerialPrograms/Source/CommonFramework/Main.cpp @@ -1,9 +1,11 @@ #include #include +#include #include //#include #include +#include #include "Common/Cpp/Logging/LastLogTracker.h" #include "Common/Cpp/Logging/FileLogger.h" #include "Common/Cpp/Logging/GlobalLogger.h" @@ -87,6 +89,35 @@ FileLogger& global_file_logger(){ +#if defined(__linux__) +// With nodrop=1 (the kernel default), uvcvideo delivers frames that lost USB +// packets instead of dropping them. The missing rows are stale, which shows up +// as a horizontal tear that detection reads as a normal frame. +void warn_if_uvcvideo_delivers_torn_frames(Logger& logger){ + std::ifstream file("/sys/module/uvcvideo/parameters/nodrop"); + if (!file){ + return; + } + std::string value; + std::getline(file, value); + while (!value.empty() && (value.back() == '\r' || value.back() == ' ')){ + value.pop_back(); + } + if (value != "1"){ + return; + } + logger.log( + "uvcvideo is set to nodrop=1. Frames that lose USB packets will be delivered " + "partially written instead of dropped: part of the image will be stale. This " + "is visible as a horizontal tear and is read as-is by detection. To fix it: " + "echo 0 | sudo tee /sys/module/uvcvideo/parameters/nodrop (to persist: " + "echo \"options uvcvideo nodrop=0\" | sudo tee /etc/modprobe.d/uvcvideo.conf)", + COLOR_RED + ); +} +#endif + + int run_program(int argc, char *argv[]){ // qputenv("QT_RHI_BACKEND", "opengl"); @@ -121,6 +152,10 @@ int run_program(int argc, char *argv[]){ logger.log("Program setting folder: " + SETTINGS_PATH()); logger.log("Program resources folder: " + RESOURCE_PATH()); +#if defined(__linux__) + warn_if_uvcvideo_delivers_torn_frames(logger); +#endif + qRegisterMetaType("size_t"); qRegisterMetaType("uint8_t"); qRegisterMetaType("std::string"); @@ -247,6 +282,23 @@ int main(int argc, char *argv[]){ if (qEnvironmentVariableIsEmpty("QT_MEDIA_BACKEND")){ qputenv("QT_MEDIA_BACKEND", "gstreamer"); } + + // Use mmap instead of dmabuf. With dmabuf, Qt imports capture buffers as + // EGLImages, and drivers that reject the fourcc (NVIDIA with YUYV) render + // a flat color without reporting an error. + if (qEnvironmentVariableIsEmpty("QT_GSTREAMER_V4L2SRC_IOMODE")){ + qputenv("QT_GSTREAMER_V4L2SRC_IOMODE", "2"); + } +#endif + +#if defined(__linux__) + // Force vsync, or video tears under Wayland. Must be set before the + // QApplication is constructed. + { + QSurfaceFormat format = QSurfaceFormat::defaultFormat(); + format.setSwapInterval(1); + QSurfaceFormat::setDefaultFormat(format); + } #endif // So far, this is only needed on Mac where static initialization is fucked up. diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.cpp index 05f6ad6544..7b8ff4832b 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.cpp @@ -19,6 +19,12 @@ #include "CameraWidgetQt6.h" #include "CameraWidgetQt6.5.h" #include "CameraWidgetQt6_QQuickWidget.h" +#if defined(__linux__) +#include "CameraWidgetQt6_QML.h" +#include "CameraWidgetQt6_QVideoWidget.h" +#include "CameraWidgetOpenCV_QOpenGLWidget.h" +#include "CameraWidgetOpenCV.h" +#endif namespace PokemonAutomation{ @@ -56,7 +62,30 @@ struct CameraBackends{ "qt6.5-QGraphicsScene", "Qt6.5: QGraphicsScene", std::make_unique() ); -#ifdef _WIN32 +#if defined(__linux__) + // The other Qt backends tear under Wayland. + m_backends.emplace_back( + "qt6-QQuickWidget", "Qt6: QQuickWidget", + std::make_unique() + ); + + m_backends.emplace_back( + "qt6-QML", "Qt6: QML VideoOutput", + std::make_unique() + ); + m_backends.emplace_back( + "qt6-QVideoWidget", "Qt6: QOpenGLWidget", + std::make_unique() + ); + m_backends.emplace_back( + "opencv-QOpenGLWidget", "OpenCV: QOpenGLWidget", + std::make_unique() + ); + m_backends.emplace_back( + "opencv-QWidget", "OpenCV: QWidget (CPU)", + std::make_unique() + ); +#elif defined(_WIN32) m_backends.emplace_back( "qt6-QQuickWidget", "Qt6: QQuickWidget", std::make_unique() diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.cpp new file mode 100644 index 0000000000..e7e74d2816 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.cpp @@ -0,0 +1,311 @@ +/* Camera Widget (OpenCV V4L2) + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#if defined(__linux__) + +#include +#include +#include +#include +#if defined(__linux__) +#include +#include +#include +#include +#endif +#include "Common/Qt/Redispatch.h" +#include "CommonFramework/Logging/Logger.h" +#include "VideoFrameQt.h" +#include "MediaServicesQt6.h" +#include "CameraWidgetOpenCV.h" + +namespace PokemonAutomation{ +namespace CameraOpenCV{ + +std::vector CameraBackend::get_all_cameras() const{ + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); + std::vector ret; + for (const auto& info : cameras){ + ret.emplace_back(info.id().toStdString()); + } + return ret; +} + +std::string CameraBackend::get_camera_name(const CameraInfo& info) const{ + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); + for (const auto& camera : cameras){ + if (camera.id().toStdString() == info.device_name()){ + return camera.description().toStdString(); + } + } + return ""; +} + +std::unique_ptr CameraBackend::make_video_source( + Logger& logger, + const CameraInfo& info, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps +) const{ + return std::make_unique(logger, info, resolution, format, fps); +} + + +CameraVideoSource::~CameraVideoSource(){ + m_stop.store(true, std::memory_order_release); + if (m_thread.joinable()){ + m_thread.join(); + } +} + +CameraVideoSource::CameraVideoSource( + Logger& logger, + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps +) + : VideoSource(logger, true) + , m_logger(logger) + , m_stop(false) + , m_last_frame(logger) + , m_snapshot_manager(logger, m_last_frame) +{ + if (!info){ + return; + } + m_logger.log("Starting Camera: Backend = OpenCV V4L2"); + +#if defined(__linux__) + int fd = open(info.device_name().c_str(), O_RDWR); + if (fd >= 0) { + struct v4l2_fmtdesc fmtdesc = {}; + fmtdesc.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + while (ioctl(fd, VIDIOC_ENUM_FMT, &fmtdesc) == 0) { + VideoFormat fmt = VideoFormat::OTHER; + bool valid_fmt = false; + if (fmtdesc.pixelformat == V4L2_PIX_FMT_MJPEG || fmtdesc.pixelformat == V4L2_PIX_FMT_JPEG) { fmt = VideoFormat::MJPEG; valid_fmt = true; } + else if (fmtdesc.pixelformat == V4L2_PIX_FMT_YUYV) { fmt = VideoFormat::YUYV; valid_fmt = true; } + + if (valid_fmt) { + struct v4l2_frmsizeenum frmsize = {}; + frmsize.pixel_format = fmtdesc.pixelformat; + while (ioctl(fd, VIDIOC_ENUM_FRAMESIZES, &frmsize) == 0) { + if (frmsize.type == V4L2_FRMSIZE_TYPE_DISCRETE) { + uint32_t w = frmsize.discrete.width; + uint32_t h = frmsize.discrete.height; + + int ival_index = 0; + while (true) { + struct v4l2_frmivalenum frmival = {}; + frmival.index = ival_index; + frmival.pixel_format = fmtdesc.pixelformat; + frmival.width = w; + frmival.height = h; + + if (ioctl(fd, VIDIOC_ENUM_FRAMEINTERVALS, &frmival) != 0) { + break; + } + + if (frmival.type == V4L2_FRMIVAL_TYPE_DISCRETE) { + double fps = (double)frmival.discrete.denominator / frmival.discrete.numerator; + m_formats[{w, h}][fmt].insert((size_t)std::round(fps)); + } + ival_index++; + } + } + frmsize.index++; + } + } + fmtdesc.index++; + } + close(fd); + } +#endif + + // Ensure the desired format is at least present in case probing failed + if (m_formats[{desired_resolution.width, desired_resolution.height}][desired_format].empty()) { + m_formats[{desired_resolution.width, desired_resolution.height}][desired_format].insert(desired_fps); + } + + m_resolution = desired_resolution; + m_format = desired_format; + m_fps = desired_fps; + + std::string device_path = info.device_name(); + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); + for (const auto& camera : cameras){ + if (camera.id().toStdString() == device_path){ + m_camera_description = camera.description().toStdString(); + break; + } + } + +#if defined(__linux__) + int device_index = 0; + if (device_path.find("/dev/video") == 0) { + try { + device_index = std::stoi(device_path.substr(10)); + } catch (...) {} + } + m_device_index = device_index; + m_cap = std::make_unique(m_device_index, cv::CAP_V4L2); +#else + m_cap = std::make_unique(device_path); +#endif + + if (!m_cap->isOpened()){ + m_logger.log("Camera failed to open via OpenCV.", COLOR_RED); + return; + } + + if (m_format == VideoFormat::MJPEG){ + m_cap->set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G')); + m_cap->set(cv::CAP_PROP_CONVERT_RGB, 1); + } else if (m_format == VideoFormat::YUYV){ + m_cap->set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('Y', 'U', 'Y', 'V')); + m_cap->set(cv::CAP_PROP_CONVERT_RGB, 0); + } + + m_cap->set(cv::CAP_PROP_FRAME_WIDTH, desired_resolution.width); + m_cap->set(cv::CAP_PROP_FRAME_HEIGHT, desired_resolution.height); + if (desired_fps > 0) { + m_cap->set(cv::CAP_PROP_FPS, desired_fps); + } + + m_logger.log("Resolution: " + m_resolution.to_string() + ", FPS: " + std::to_string(m_fps)); + + m_thread = std::thread(&CameraVideoSource::thread_body, this); +} + +void CameraVideoSource::thread_body(){ + cv::Mat raw_mat, bgra, safe_raw; + int consecutive_failures = 0; + while (!m_stop.load(std::memory_order_acquire)){ + if (!m_cap->read(raw_mat) || raw_mat.empty()){ + consecutive_failures++; + if (consecutive_failures % 30 == 1) { + m_logger.log("Failed to read frame from OpenCV. (Failures: " + std::to_string(consecutive_failures) + ")", COLOR_RED); + } + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + if (consecutive_failures > 40) { + m_logger.log("Too many consecutive read failures. Attempting to restart capture device...", COLOR_RED); + m_cap->release(); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); +#if defined(__linux__) + m_cap->open(m_device_index, cv::CAP_V4L2); + if (!m_cap->isOpened() && !m_camera_description.empty()) { + m_logger.log("Failed to reopen device index " + std::to_string(m_device_index) + ". Searching for it by name...", COLOR_RED); + const auto new_cameras = GlobalMediaServices::instance().get_all_cameras(); + for (const auto& camera : new_cameras) { + if (camera.description().toStdString() == m_camera_description) { + std::string new_path = camera.id().toStdString(); + if (new_path.find("/dev/video") == 0) { + try { + m_device_index = std::stoi(new_path.substr(10)); + m_logger.log("Found device at new index " + std::to_string(m_device_index), COLOR_RED); + m_cap->open(m_device_index, cv::CAP_V4L2); + if (m_cap->isOpened()) break; + } catch (...) {} + } + } + } + } +#endif + if (m_cap->isOpened()) { + if (m_format == VideoFormat::MJPEG){ + m_cap->set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G')); + m_cap->set(cv::CAP_PROP_CONVERT_RGB, 1); + } else if (m_format == VideoFormat::YUYV){ + m_cap->set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('Y', 'U', 'Y', 'V')); + m_cap->set(cv::CAP_PROP_CONVERT_RGB, 0); + } + m_cap->set(cv::CAP_PROP_FRAME_WIDTH, m_resolution.width); + m_cap->set(cv::CAP_PROP_FRAME_HEIGHT, m_resolution.height); + if (m_fps > 0) { + m_cap->set(cv::CAP_PROP_FPS, m_fps); + } + consecutive_failures = 0; + } + } + continue; + } + consecutive_failures = 0; + + WallClock now = current_time(); + WallClock convert_start = now; + + if (m_format == VideoFormat::YUYV) { + cv::cvtColor(raw_mat, bgra, cv::COLOR_YUV2BGRA_YUYV); + } else { + cv::cvtColor(raw_mat, bgra, cv::COLOR_BGR2BGRA); + } + + QVideoFrameFormat format(QSize(bgra.cols, bgra.rows), QVideoFrameFormat::Format_BGRA8888); + QVideoFrame frame(format); + + if (frame.map(QVideoFrame::WriteOnly)){ + memcpy(frame.bits(0), bgra.data, bgra.total() * bgra.elemSize()); + frame.unmap(); + + report_video_path_stats( + m_logger, "OpenCV-QWidget", frame, true, convert_start, current_time() + ); + + if (!m_last_frame.push_frame(frame, now)){ + continue; + } + report_source_frame(std::make_shared(now, frame)); + } + } +} + +QWidget* CameraVideoSource::make_display_QtWidget(QWidget* parent){ + return new CameraVideoDisplay(parent, *this); +} + +CameraVideoDisplay::~CameraVideoDisplay(){ + m_source.remove_source_frame_listener(*this); +} +CameraVideoDisplay::CameraVideoDisplay(QWidget* parent, CameraVideoSource& source) + : QWidget(parent) + , m_source(source) +{ + source.add_source_frame_listener(*this); +} +void CameraVideoDisplay::on_frame(std::shared_ptr frame){ + m_last_frame = std::move(frame); + queue_on_main_thread([this]{ + this->update(); + }); +} +void CameraVideoDisplay::paintEvent(QPaintEvent* event){ + QWidget::paintEvent(event); + + if (!m_last_frame){ + return; + } + + QVideoFrame frame = m_last_frame->frame; + if (!frame.isValid()){ + return; + } + + QRect rect(0, 0, this->width(), this->height()); + QVideoFrame::PaintOptions options; + QPainter painter(this); + + frame.paint(&painter, rect, options); + m_source.report_rendered_frame(current_time()); +} + + +} +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.h new file mode 100644 index 0000000000..b614c4714a --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.h @@ -0,0 +1,120 @@ +/* Camera Widget (OpenCV V4L2) + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_VideoPipeline_OpenCV_H +#define PokemonAutomation_VideoPipeline_OpenCV_H + +#include +#if defined(__linux__) + +#include +#include +#include +#include "Common/Cpp/Concurrency/Mutex.h" +#include "CommonFramework/VideoPipeline/VideoSource.h" +#include "CommonFramework/VideoPipeline/CameraInfo.h" +#include "CameraImplementations.h" +#include "QVideoFrameCache.h" +#include "SnapshotManager.h" + +namespace cv { + class VideoCapture; + class Mat; +} + +namespace PokemonAutomation{ +namespace CameraOpenCV{ + +class CameraBackend : public PokemonAutomation::CameraBackend{ +public: + virtual std::vector get_all_cameras() const override; + virtual std::string get_camera_name(const CameraInfo& info) const override; + + virtual std::unique_ptr make_video_source( + Logger& logger, + const CameraInfo& info, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps + ) const override; +}; + +class CameraVideoSource : public VideoSource{ +public: + virtual ~CameraVideoSource(); + CameraVideoSource( + Logger& logger, + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps + ); + + virtual Resolution current_resolution() const override{ + return m_resolution; + } + virtual VideoFormat current_format() const override{ + return m_format; + } + virtual FramesPerSecond current_fps() const override{ + return m_fps; + } + virtual const VideoFormatSet& supported_formats() const override{ + return m_formats; + } + + virtual VideoSnapshot snapshot_latest_blocking() override{ + return m_snapshot_manager.snapshot_latest_blocking(); + } + virtual VideoSnapshot snapshot_recent_nonblocking(WallClock min_time) override{ + return m_snapshot_manager.snapshot_recent_nonblocking(min_time); + } + + virtual QWidget* make_display_QtWidget(QWidget* parent) override; + +private: + void thread_body(); + +private: + friend class CameraVideoDisplay; + + Logger& m_logger; + Resolution m_resolution; + VideoFormat m_format; + FramesPerSecond m_fps; + + VideoFormatSet m_formats; + + int m_device_index = 0; + std::string m_camera_description; + std::unique_ptr m_cap; + std::thread m_thread; + std::atomic m_stop; + + QVideoFrameCache m_last_frame; + SnapshotManager m_snapshot_manager; +}; + +class CameraVideoDisplay : public QWidget, private VideoFrameListener{ +public: + ~CameraVideoDisplay(); + CameraVideoDisplay(QWidget* parent, CameraVideoSource& source); + +private: + virtual void on_frame(std::shared_ptr frame) override; + virtual void paintEvent(QPaintEvent* event) override; + +private: + CameraVideoSource& m_source; + std::shared_ptr m_last_frame; + + LifetimeSanitizer m_sanitizer; +}; + +} +} +#endif +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.cpp new file mode 100644 index 0000000000..454f9d249f --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.cpp @@ -0,0 +1,341 @@ +/* Camera Widget (OpenCV V4L2) + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#if defined(__linux__) + +#include +#include +#include +#include +#include +#include +#if defined(__linux__) +#include +#include +#include +#include +#endif +#include "Common/Qt/Redispatch.h" +#include "CommonFramework/Logging/Logger.h" +#include "VideoFrameQt.h" +#include "MediaServicesQt6.h" +#include "CameraWidgetOpenCV_QOpenGLWidget.h" + +namespace PokemonAutomation{ +namespace CameraOpenCV_QOpenGLWidget{ + +std::vector CameraBackend::get_all_cameras() const{ + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); + std::vector ret; + for (const auto& info : cameras){ + ret.emplace_back(info.id().toStdString()); + } + return ret; +} + +std::string CameraBackend::get_camera_name(const CameraInfo& info) const{ + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); + for (const auto& camera : cameras){ + if (camera.id().toStdString() == info.device_name()){ + return camera.description().toStdString(); + } + } + return ""; +} + +std::unique_ptr CameraBackend::make_video_source( + Logger& logger, + const CameraInfo& info, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps +) const{ + return std::make_unique(logger, info, resolution, format, fps); +} + + +CameraVideoSource::~CameraVideoSource(){ + m_stop.store(true, std::memory_order_release); + if (m_thread.joinable()){ + m_thread.join(); + } +} + +CameraVideoSource::CameraVideoSource( + Logger& logger, + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps +) + : VideoSource(logger, true) + , m_logger(logger) + , m_stop(false) + , m_last_frame(logger) + , m_snapshot_manager(logger, m_last_frame) +{ + if (!info){ + return; + } + m_logger.log("Starting Camera: Backend = OpenCV V4L2"); + +#if defined(__linux__) + int fd = open(info.device_name().c_str(), O_RDWR); + if (fd >= 0) { + struct v4l2_fmtdesc fmtdesc = {}; + fmtdesc.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + while (ioctl(fd, VIDIOC_ENUM_FMT, &fmtdesc) == 0) { + VideoFormat fmt = VideoFormat::OTHER; + bool valid_fmt = false; + if (fmtdesc.pixelformat == V4L2_PIX_FMT_MJPEG || fmtdesc.pixelformat == V4L2_PIX_FMT_JPEG) { fmt = VideoFormat::MJPEG; valid_fmt = true; } + else if (fmtdesc.pixelformat == V4L2_PIX_FMT_YUYV) { fmt = VideoFormat::YUYV; valid_fmt = true; } + + if (valid_fmt) { + struct v4l2_frmsizeenum frmsize = {}; + frmsize.pixel_format = fmtdesc.pixelformat; + while (ioctl(fd, VIDIOC_ENUM_FRAMESIZES, &frmsize) == 0) { + if (frmsize.type == V4L2_FRMSIZE_TYPE_DISCRETE) { + uint32_t w = frmsize.discrete.width; + uint32_t h = frmsize.discrete.height; + + int ival_index = 0; + while (true) { + struct v4l2_frmivalenum frmival = {}; + frmival.index = ival_index; + frmival.pixel_format = fmtdesc.pixelformat; + frmival.width = w; + frmival.height = h; + + if (ioctl(fd, VIDIOC_ENUM_FRAMEINTERVALS, &frmival) != 0) { + break; + } + + if (frmival.type == V4L2_FRMIVAL_TYPE_DISCRETE) { + double fps = (double)frmival.discrete.denominator / frmival.discrete.numerator; + m_formats[{w, h}][fmt].insert((size_t)std::round(fps)); + } + ival_index++; + } + } + frmsize.index++; + } + } + fmtdesc.index++; + } + close(fd); + } +#endif + + // Ensure the desired format is at least present in case probing failed + if (m_formats[{desired_resolution.width, desired_resolution.height}][desired_format].empty()) { + m_formats[{desired_resolution.width, desired_resolution.height}][desired_format].insert(desired_fps); + } + + m_resolution = desired_resolution; + m_format = desired_format; + m_fps = desired_fps; + + std::string device_path = info.device_name(); + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); + for (const auto& camera : cameras){ + if (camera.id().toStdString() == device_path){ + m_camera_description = camera.description().toStdString(); + break; + } + } + +#if defined(__linux__) + int device_index = 0; + if (device_path.find("/dev/video") == 0) { + try { + device_index = std::stoi(device_path.substr(10)); + } catch (...) {} + } + m_device_index = device_index; + m_cap = std::make_unique(m_device_index, cv::CAP_V4L2); +#else + m_cap = std::make_unique(device_path); +#endif + + if (!m_cap->isOpened()){ + m_logger.log("Camera failed to open via OpenCV.", COLOR_RED); + return; + } + + if (m_format == VideoFormat::MJPEG){ + m_cap->set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G')); + m_cap->set(cv::CAP_PROP_CONVERT_RGB, 1); + } else if (m_format == VideoFormat::YUYV){ + m_cap->set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('Y', 'U', 'Y', 'V')); + m_cap->set(cv::CAP_PROP_CONVERT_RGB, 0); + } + + m_cap->set(cv::CAP_PROP_FRAME_WIDTH, desired_resolution.width); + m_cap->set(cv::CAP_PROP_FRAME_HEIGHT, desired_resolution.height); + if (desired_fps > 0) { + m_cap->set(cv::CAP_PROP_FPS, desired_fps); + } + + m_logger.log("Resolution: " + m_resolution.to_string() + ", FPS: " + std::to_string(m_fps)); + + m_thread = std::thread(&CameraVideoSource::thread_body, this); +} + +void CameraVideoSource::thread_body(){ + cv::Mat raw_mat, bgra, safe_raw; + int consecutive_failures = 0; + while (!m_stop.load(std::memory_order_acquire)){ + if (!m_cap->read(raw_mat) || raw_mat.empty()){ + consecutive_failures++; + if (consecutive_failures % 30 == 1) { + m_logger.log("Failed to read frame from OpenCV. (Failures: " + std::to_string(consecutive_failures) + ")", COLOR_RED); + } + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + if (consecutive_failures > 40) { + m_logger.log("Too many consecutive read failures. Attempting to restart capture device...", COLOR_RED); + m_cap->release(); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); +#if defined(__linux__) + m_cap->open(m_device_index, cv::CAP_V4L2); + if (!m_cap->isOpened() && !m_camera_description.empty()) { + m_logger.log("Failed to reopen device index " + std::to_string(m_device_index) + ". Searching for it by name...", COLOR_RED); + const auto new_cameras = GlobalMediaServices::instance().get_all_cameras(); + for (const auto& camera : new_cameras) { + if (camera.description().toStdString() == m_camera_description) { + std::string new_path = camera.id().toStdString(); + if (new_path.find("/dev/video") == 0) { + try { + m_device_index = std::stoi(new_path.substr(10)); + m_logger.log("Found device at new index " + std::to_string(m_device_index), COLOR_RED); + m_cap->open(m_device_index, cv::CAP_V4L2); + if (m_cap->isOpened()) break; + } catch (...) {} + } + } + } + } +#endif + if (m_cap->isOpened()) { + if (m_format == VideoFormat::MJPEG){ + m_cap->set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G')); + m_cap->set(cv::CAP_PROP_CONVERT_RGB, 1); + } else if (m_format == VideoFormat::YUYV){ + m_cap->set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('Y', 'U', 'Y', 'V')); + m_cap->set(cv::CAP_PROP_CONVERT_RGB, 0); + } + m_cap->set(cv::CAP_PROP_FRAME_WIDTH, m_resolution.width); + m_cap->set(cv::CAP_PROP_FRAME_HEIGHT, m_resolution.height); + if (m_fps > 0) { + m_cap->set(cv::CAP_PROP_FPS, m_fps); + } + consecutive_failures = 0; + } + } + continue; + } + consecutive_failures = 0; + + WallClock now = current_time(); + WallClock convert_start = now; + + if (m_format == VideoFormat::YUYV) { + cv::cvtColor(raw_mat, bgra, cv::COLOR_YUV2BGRA_YUYV); + } else { + cv::cvtColor(raw_mat, bgra, cv::COLOR_BGR2BGRA); + } + + QVideoFrameFormat format(QSize(bgra.cols, bgra.rows), QVideoFrameFormat::Format_BGRA8888); + QVideoFrame frame(format); + + if (frame.map(QVideoFrame::WriteOnly)){ + memcpy(frame.bits(0), bgra.data, bgra.total() * bgra.elemSize()); + frame.unmap(); + + report_video_path_stats( + m_logger, "OpenCV-QOpenGLWidget", frame, true, convert_start, current_time() + ); + + if (!m_last_frame.push_frame(frame, now)){ + continue; + } + report_source_frame(std::make_shared(now, frame)); + } + } +} + +QWidget* CameraVideoSource::make_display_QtWidget(QWidget* parent){ + return new CameraVideoDisplay(parent, *this); +} + +CameraVideoDisplay::~CameraVideoDisplay(){ + m_source.remove_source_frame_listener(*this); +} +CameraVideoDisplay::CameraVideoDisplay(QWidget* parent, CameraVideoSource& source) + : QOpenGLWidget(parent) + , m_source(source) + , m_width(80) + , m_height(45) + , m_sanitizer("CameraVideoDisplay") +{ + this->setMinimumSize(80, 45); + + + source.add_source_frame_listener(*this); +} +void CameraVideoDisplay::resizeEvent(QResizeEvent* event){ + QOpenGLWidget::resizeEvent(event); + m_width.store(event->size().width(), std::memory_order_relaxed); + m_height.store(event->size().height(), std::memory_order_relaxed); +} + +void CameraVideoDisplay::on_frame(std::shared_ptr frame){ + int w = m_width.load(std::memory_order_relaxed); + int h = m_height.load(std::memory_order_relaxed); + + std::shared_ptr scaled_frame = frame; + + if (w > 0 && h > 0 && frame && frame->frame.isValid()){ + QSize target_size(w, h); + if (frame->frame.size() != target_size){ + QImage img = frame->frame.toImage(); + QImage scaled = img.scaled(target_size, Qt::IgnoreAspectRatio, Qt::FastTransformation); + QVideoFrame out(scaled); + scaled_frame = std::make_shared(frame->timestamp, out); + } + } + + QMetaObject::invokeMethod(this, [this, frame = std::move(scaled_frame)]() mutable { + m_last_frame = std::move(frame); + this->update(); + }, Qt::QueuedConnection); +} + +void CameraVideoDisplay::paintEvent(QPaintEvent* event){ + QOpenGLWidget::paintEvent(event); + + if (!m_last_frame){ + return; + } + + QVideoFrame frame = m_last_frame->frame; + if (!frame.isValid()){ + return; + } + + QRect rect(0, 0, this->width(), this->height()); + QVideoFrame::PaintOptions options; + QPainter painter(this); + + frame.paint(&painter, rect, options); + m_source.report_rendered_frame(current_time()); +} + + +} +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.h new file mode 100644 index 0000000000..2d68e9f962 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.h @@ -0,0 +1,127 @@ +/* Camera Widget (OpenCV V4L2) + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_CameraWidgetOpenCV_QOpenGLWidget_H +#define PokemonAutomation_CameraWidgetOpenCV_QOpenGLWidget_H + +#include +#if defined(__linux__) + +#include +#include +#include +#include +#include "Common/Cpp/Concurrency/Mutex.h" +#include "CommonFramework/VideoPipeline/VideoSource.h" +#include "CommonFramework/VideoPipeline/CameraInfo.h" +#include "CameraImplementations.h" +#include "QVideoFrameCache.h" +#include "SnapshotManager.h" + +namespace cv { + class VideoCapture; + class Mat; +} + +namespace PokemonAutomation{ +namespace CameraOpenCV_QOpenGLWidget{ + +class CameraBackend : public PokemonAutomation::CameraBackend{ +public: + virtual std::vector get_all_cameras() const override; + virtual std::string get_camera_name(const CameraInfo& info) const override; + + virtual std::unique_ptr make_video_source( + Logger& logger, + const CameraInfo& info, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps + ) const override; +}; + +class CameraVideoSource : public VideoSource{ +public: + virtual ~CameraVideoSource(); + CameraVideoSource( + Logger& logger, + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps + ); + + virtual Resolution current_resolution() const override{ + return m_resolution; + } + virtual VideoFormat current_format() const override{ + return m_format; + } + virtual FramesPerSecond current_fps() const override{ + return m_fps; + } + virtual const VideoFormatSet& supported_formats() const override{ + return m_formats; + } + + virtual VideoSnapshot snapshot_latest_blocking() override{ + return m_snapshot_manager.snapshot_latest_blocking(); + } + virtual VideoSnapshot snapshot_recent_nonblocking(WallClock min_time) override{ + return m_snapshot_manager.snapshot_recent_nonblocking(min_time); + } + + virtual QWidget* make_display_QtWidget(QWidget* parent) override; + +private: + void thread_body(); + +private: + friend class CameraVideoDisplay; + + Logger& m_logger; + Resolution m_resolution; + VideoFormat m_format; + FramesPerSecond m_fps; + + VideoFormatSet m_formats; + + int m_device_index = 0; + std::string m_camera_description; + std::unique_ptr m_cap; + std::thread m_thread; + std::atomic m_stop; + + QVideoFrameCache m_last_frame; + SnapshotManager m_snapshot_manager; +}; + +class CameraVideoDisplay : public QOpenGLWidget, private VideoFrameListener{ +public: + ~CameraVideoDisplay(); + CameraVideoDisplay(QWidget* parent, CameraVideoSource& source); + +private: + virtual void on_frame(std::shared_ptr frame) override; + +protected: + virtual void resizeEvent(QResizeEvent* event) override; + virtual void paintEvent(QPaintEvent* event) override; + +private: + CameraVideoSource& m_source; + std::shared_ptr m_last_frame; + + std::atomic m_width; + std::atomic m_height; + + LifetimeSanitizer m_sanitizer; +}; + +} +} +#endif +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp index 71a0977507..bf8cecfb3f 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp @@ -174,11 +174,21 @@ void CameraVideoSource::set_video_output(QGraphicsVideoItem& item){ &m_camera->camera(), [&](const QVideoFrame& frame){ // This runs on the QCamera's thread. So it is off the critical path. + WallClock copy_start = current_time(); + QVideoFrame owned = deep_copy_frame(frame); + WallClock copy_end = current_time(); + report_video_path_stats( + m_logger, "Qt6.5-QGraphicsScene", frame, owned.isValid(), copy_start, copy_end + ); + if (!owned.isValid()){ + return; + } + WallClock now = current_time(); - if (!m_last_frame.push_frame(frame, now)){ + if (!m_last_frame.push_frame(owned, now)){ return; } - report_source_frame(std::make_shared(now, frame)); + report_source_frame(std::make_shared(now, owned)); } ); } diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.h index 163f3d7be0..a88d50bf1e 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.h @@ -15,6 +15,7 @@ #include #include #include +#include "VideoFrameQt.h" #include #include #include diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp index 13b6dff18d..40e87704ff 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp @@ -155,11 +155,21 @@ void CameraVideoSource::init( &m_camera->camera(), [&](const QVideoFrame& frame){ // This runs on the QCamera's thread. So it is off the critical path. + WallClock copy_start = current_time(); + QVideoFrame owned = deep_copy_frame(frame); + WallClock copy_end = current_time(); + report_video_path_stats( + m_logger, "Qt6-QVideoSink", frame, owned.isValid(), copy_start, copy_end + ); + if (!owned.isValid()){ + return; + } + WallClock now = current_time(); - if (!m_last_frame.push_frame(frame, now)){ + if (!m_last_frame.push_frame(owned, now)){ return; } - report_source_frame(std::make_shared(now, frame)); + report_source_frame(std::make_shared(now, owned)); } ); } diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.h index 64fc9966ed..12a5215a14 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.h @@ -14,6 +14,7 @@ #include #include #include +#include "VideoFrameQt.h" #include "CommonFramework/Tools/StatAccumulator.h" #include "CommonFramework/VideoPipeline/VideoSource.h" #include "CommonFramework/VideoPipeline/CameraInfo.h" diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.cpp new file mode 100644 index 0000000000..6bb3da1a1d --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.cpp @@ -0,0 +1,255 @@ +/* Camera Widget (Qt6 QML VideoOutput) + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#if QT_VERSION_MAJOR == 6 + +#include +#include +#include +#include +#include +#include "Common/Qt/Redispatch.h" +#include "CommonFramework/Logging/Logger.h" +#include "QFormatAggregator.h" +#include "CameraWidgetQt6_QML.h" +#include "MediaServicesQt6.h" +#include "VideoFrameQt.h" + +namespace PokemonAutomation{ +namespace CameraQt6QML{ + +std::vector CameraBackend::get_all_cameras() const{ +#if 1 + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); +#else + const auto cameras = QMediaDevices::videoInputs(); +#endif + std::vector ret; + for (const auto& info : cameras){ + ret.emplace_back(info.id().toStdString()); + } + return ret; +} +std::string CameraBackend::get_camera_name(const CameraInfo& info) const{ +#if 1 + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); +#else + const auto cameras = QMediaDevices::videoInputs(); +#endif + for (const auto& camera : cameras){ + if (camera.id().toStdString() == info.device_name()){ + return camera.description().toStdString(); + } + } + global_logger_tagged().log("Error: No such camera for CameraInfo: " + info.device_name(), COLOR_RED); + return ""; +} + +std::unique_ptr CameraBackend::make_video_source( + Logger& logger, + const CameraInfo& info, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps +) const{ + return std::make_unique(logger, info, resolution, format, fps); +} + + +CameraVideoSource::~CameraVideoSource(){ + if (!m_capture){ + return; + } + try{ + m_logger.log("Stopping Camera..."); + }catch (...){} + + run_on_main_thread_and_wait([&]{ + m_metaobject.reset(); + m_capture.reset(); + m_video_sink.reset(); + m_camera.reset(); + }); +} +CameraVideoSource::CameraVideoSource( + Logger& logger, + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps +) + : VideoSource(logger, true) + , m_logger(logger) + , m_last_frame(logger) + , m_snapshot_manager(logger, m_last_frame) +{ + if (!info){ + return; + } + m_logger.log("Starting Camera: Backend = CameraQt6QML"); + + run_on_main_thread_and_wait([&]{ + init(info, desired_resolution, desired_format, desired_fps); + }); +} +void CameraVideoSource::init( + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps +){ + m_metaobject.reset(new QObject()); + + auto cameras = QMediaDevices::videoInputs(); + const QCameraDevice* device = nullptr; + for (const auto& camera : cameras){ + if (camera.id().toStdString() == info.device_name()){ + device = &camera; + break; + } + } + if (device == nullptr){ + m_logger.log("Camera not found: " + info.device_name(), COLOR_RED); + return; + } + m_logger.log("Camera: " + device->description().toStdString()); + + QCameraFormat format = build_format_set( + m_logger, + m_formats, + *device, + desired_resolution, + desired_format, + desired_fps + ); + if (format.isNull()){ + return; + } + + get_format(format, m_resolution, m_format, m_fps); + m_logger.log( + "Resolution: " + m_resolution.to_string() + + ", Format: " + VideoFormat_database().find(m_format)->display + + ", FPS: " + std::to_string(m_fps) + ); + + m_camera.reset(new QCameraThread(m_logger, *device, format)); + m_video_sink.reset(new QVideoSink()); + m_capture.reset(new QMediaCaptureSession()); + m_capture->setCamera(&m_camera->camera()); + m_capture->setVideoSink(m_video_sink.get()); + + // Restart the camera if the capture pipeline breaks. + m_metaobject->connect( + &m_camera->camera(), &QCamera::errorOccurred, + &m_camera->camera(), [this](QCamera::Error, const QString& errorString){ + m_logger.log("Camera error detected: " + errorString.toStdString() + ". Attempting auto-restart...", COLOR_RED); + QMetaObject::invokeMethod(&m_camera->camera(), "start", Qt::QueuedConnection); + } + ); + + // Context is the QCamera: runs on the camera thread and outlives every display. + m_metaobject->connect( + m_video_sink.get(), &QVideoSink::videoFrameChanged, + &m_camera->camera(), [this](const QVideoFrame& frame){ + WallClock copy_start = current_time(); + QVideoFrame owned = deep_copy_frame(frame); + WallClock copy_end = current_time(); + report_video_path_stats( + m_logger, "QML", frame, owned.isValid(), copy_start, copy_end + ); + if (!owned.isValid()){ + return; + } + + WallClock now = current_time(); + if (!m_last_frame.push_frame(owned, now)){ + return; + } + report_source_frame(std::make_shared(now, owned)); + } + ); +} + +QWidget* CameraVideoSource::make_display_QtWidget(QWidget* parent){ + CameraVideoDisplay* display = new CameraVideoDisplay(parent); + + // Null if init() found no camera or usable format. + if (m_camera == nullptr || m_video_sink == nullptr){ + m_logger.log("make_display_QtWidget(): camera was not initialized", COLOR_RED); + return display; + } + + QObject* root = display->rootObject(); + if (root == nullptr){ + m_logger.log("make_display_QtWidget(): display->rootObject() == nullptr", COLOR_RED); + return display; + } + + QObject* vo = root->findChild("videoOutput"); + if (vo == nullptr){ + m_logger.log("make_display_QtWidget(): videoOutput == nullptr", COLOR_RED); + return display; + } + + QVideoSink* qml_sink = vo->property("videoSink").value(); + if (qml_sink == nullptr){ + m_logger.log("make_display_QtWidget(): videoSink == nullptr", COLOR_RED); + return display; + } + + QObject::disconnect(m_display_connection); + + // qml_sink must be the context: it is destroyed whenever the display is + // rebuilt, which drops this connection, and it queues the call onto the + // GUI thread. + m_display_connection = QObject::connect( + m_video_sink.get(), &QVideoSink::videoFrameChanged, + qml_sink, [this, qml_sink](const QVideoFrame& frame){ + qml_sink->setVideoFrame(frame); + report_rendered_frame(current_time()); + } + ); + + return display; +} + +CameraVideoDisplay::~CameraVideoDisplay(){ +} +CameraVideoDisplay::CameraVideoDisplay(QWidget* parent) + : QQuickWidget(parent) + , m_sanitizer("CameraVideoDisplay") +{ + this->setMinimumSize(80, 45); + this->setResizeMode(QQuickWidget::SizeRootObjectToView); + this->setFocusPolicy(Qt::NoFocus); + + QString qml = QStringLiteral( + "import QtQuick\n" + "import QtMultimedia\n" + "Rectangle {\n" + " color: \"black\"\n" + " VideoOutput {\n" + " id: videoOutput\n" + " anchors.fill: parent\n" + " objectName: \"videoOutput\"\n" + " fillMode: VideoOutput.PreserveAspectFit\n" + " }\n" + "}\n" + ); + + QTemporaryFile file; + if (file.open()){ + file.write(qml.toUtf8()); + file.close(); + this->setSource(QUrl::fromLocalFile(file.fileName())); + } +} + +} +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.h new file mode 100644 index 0000000000..43845ced82 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.h @@ -0,0 +1,119 @@ +/* Camera Widget (Qt6 QML VideoOutput) + * + * From: https://github.com/PokemonAutomation/ + * + * Same rendering path as CameraWidgetQt6_QQuickWidget, plus an automatic + * restart when the QCamera reports an error. Kept separate so the restart + * behavior can be evaluated against a capture card that glitches. + * + */ + +#ifndef PokemonAutomation_VideoPipeline_Qt6QML_H +#define PokemonAutomation_VideoPipeline_Qt6QML_H + +#include +#if QT_VERSION_MAJOR == 6 + +#include +#include +#include +#include +#include "CommonFramework/Tools/StatAccumulator.h" +#include "CommonFramework/VideoPipeline/VideoSource.h" +#include "CommonFramework/VideoPipeline/CameraInfo.h" +#include "QCameraThread.h" +#include "QVideoFrameCache.h" +#include "SnapshotManager.h" +#include "CameraImplementations.h" + +class QCamera; +class QVideoSink; + +namespace PokemonAutomation{ +namespace CameraQt6QML{ + +class CameraBackend : public PokemonAutomation::CameraBackend{ +public: + virtual std::vector get_all_cameras() const override; + virtual std::string get_camera_name(const CameraInfo& info) const override; + + virtual std::unique_ptr make_video_source( + Logger& logger, + const CameraInfo& info, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps + ) const override; +}; + +class CameraVideoSource : public VideoSource{ +public: + virtual ~CameraVideoSource(); + CameraVideoSource( + Logger& logger, + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps + ); + + virtual Resolution current_resolution() const override{ return m_resolution; } + virtual VideoFormat current_format() const override{ return m_format; } + virtual FramesPerSecond current_fps() const override{ return m_fps; } + virtual const VideoFormatSet& supported_formats() const override{ return m_formats; } + + virtual VideoSnapshot snapshot_latest_blocking() override{ + return m_snapshot_manager.snapshot_latest_blocking(); + } + virtual VideoSnapshot snapshot_recent_nonblocking(WallClock min_time) override{ + return m_snapshot_manager.snapshot_recent_nonblocking(min_time); + } + + virtual QWidget* make_display_QtWidget(QWidget* parent) override; + +private: + void init( + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps + ); + +private: + friend class CameraVideoDisplay; + + std::unique_ptr m_metaobject; + + Logger& m_logger; + Resolution m_resolution; + VideoFormat m_format; + FramesPerSecond m_fps; + + std::unique_ptr m_camera; + std::unique_ptr m_video_sink; + std::unique_ptr m_capture; + + // Torn down by handle: m_metaobject is neither sender nor receiver, so + // m_metaobject->disconnect() would be a no-op. + QMetaObject::Connection m_display_connection; + + VideoFormatSet m_formats; + +private: + QVideoFrameCache m_last_frame; + SnapshotManager m_snapshot_manager; +}; + +class CameraVideoDisplay : public QQuickWidget{ +public: + virtual ~CameraVideoDisplay(); + CameraVideoDisplay(QWidget* parent); + +private: + LifetimeSanitizer m_sanitizer; +}; + +} +} +#endif +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.cpp index 730166dfec..03f55ad638 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "Common/Qt/Redispatch.h" #include "VideoFrameQt.h" @@ -91,9 +92,10 @@ CameraVideoSource::CameraVideoSource( , m_snapshot_manager(logger, m_last_frame) { if (!info){ + m_logger.log("Camera has no device info. Not starting.", COLOR_RED); return; } - m_logger.log("Starting Camera: Backend = CameraQt6QVideoSink"); + m_logger.log("Starting Camera: Backend = CameraQt6_QQuickWidget, device = \"" + info.device_name() + "\""); run_on_main_thread_and_wait([&]{ init(info, desired_resolution, desired_format, desired_fps); @@ -152,16 +154,27 @@ void CameraVideoSource::init( } }); #endif + // Context is the QCamera: runs on the camera thread and outlives every display. m_metaobject->connect( m_video_sink.get(), &QVideoSink::videoFrameChanged, - &m_camera->camera(), [&](const QVideoFrame& frame){ + &m_camera->camera(), [this](const QVideoFrame& frame){ // This runs on the QCamera's thread. So it is off the critical path. + WallClock copy_start = current_time(); + QVideoFrame owned = deep_copy_frame(frame); + WallClock copy_end = current_time(); + report_video_path_stats( + m_logger, "QQuickWidget", frame, owned.isValid(), copy_start, copy_end + ); + if (!owned.isValid()){ + return; + } + WallClock now = current_time(); - if (!m_last_frame.push_frame(frame, now)){ + if (!m_last_frame.push_frame(owned, now)){ return; } - report_source_frame(std::make_shared(now, frame)); + report_source_frame(std::make_shared(now, owned)); } ); } @@ -170,6 +183,12 @@ void CameraVideoSource::init( QWidget* CameraVideoSource::make_display_QtWidget(QWidget* parent){ CameraVideoDisplay* display = new CameraVideoDisplay(parent); + // Null if init() found no camera or usable format. + if (m_camera == nullptr || m_video_sink == nullptr){ + m_logger.log("make_display_QtWidget(): camera was not initialized", COLOR_RED); + return display; + } + QObject* root = display->rootObject(); if (root == nullptr){ m_logger.log("make_display_QtWidget(): display->rootObject() == nullptr", COLOR_RED); @@ -188,17 +207,16 @@ QWidget* CameraVideoSource::make_display_QtWidget(QWidget* parent){ return display; } - m_metaobject->disconnect(); - m_metaobject->connect( + QObject::disconnect(m_display_connection); + + // qml_sink must be the context: it is destroyed whenever the display is + // rebuilt, which drops this connection, and it queues the call onto the + // GUI thread. + m_display_connection = QObject::connect( m_video_sink.get(), &QVideoSink::videoFrameChanged, qml_sink, [this, qml_sink](const QVideoFrame& frame){ - WallClock now = current_time(); - - QMetaObject::invokeMethod(qml_sink, [qml_sink, frame](){ - qml_sink->setVideoFrame(frame); - }, Qt::QueuedConnection); - - report_rendered_frame(now); + qml_sink->setVideoFrame(frame); + report_rendered_frame(current_time()); } ); @@ -211,6 +229,7 @@ QWidget* CameraVideoSource::make_display_QtWidget(QWidget* parent){ CameraVideoDisplay::~CameraVideoDisplay(){ } + CameraVideoDisplay::CameraVideoDisplay(QWidget* parent) : QQuickWidget(parent) , m_sanitizer("CameraVideoDisplay") @@ -239,7 +258,6 @@ CameraVideoDisplay::CameraVideoDisplay(QWidget* parent) file.close(); this->setSource(QUrl::fromLocalFile(file.fileName())); } - } diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.h index ca4bce5588..99a17ee5d5 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.h @@ -15,7 +15,9 @@ #include #include #include +#include "VideoFrameQt.h" #include +#include #include "CommonFramework/Tools/StatAccumulator.h" #include "CommonFramework/VideoPipeline/VideoSource.h" #include "CommonFramework/VideoPipeline/CameraInfo.h" @@ -113,6 +115,10 @@ class CameraVideoSource : public VideoSource{ std::unique_ptr m_video_sink; std::unique_ptr m_capture; + // Torn down by handle: m_metaobject is neither sender nor receiver, so + // m_metaobject->disconnect() would be a no-op. + QMetaObject::Connection m_display_connection; + VideoFormatSet m_formats; diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.cpp new file mode 100644 index 0000000000..3d6170710e --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.cpp @@ -0,0 +1,213 @@ +/* Camera Widget (Qt6 QVideoWidget) + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#if QT_VERSION_MAJOR == 6 + +#include +#include +#include +#include "Common/Qt/Redispatch.h" +#include "CommonFramework/Logging/Logger.h" +#include "QFormatAggregator.h" +#include "CameraWidgetQt6_QVideoWidget.h" +#include "MediaServicesQt6.h" +#include "VideoFrameQt.h" + +namespace PokemonAutomation{ +namespace CameraQt6QVideoWidget{ + +std::vector CameraBackend::get_all_cameras() const{ +#if 1 + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); +#else + const auto cameras = QMediaDevices::videoInputs(); +#endif + std::vector ret; + for (const auto& info : cameras){ + ret.emplace_back(info.id().toStdString()); + } + return ret; +} +std::string CameraBackend::get_camera_name(const CameraInfo& info) const{ +#if 1 + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); +#else + const auto cameras = QMediaDevices::videoInputs(); +#endif + for (const auto& camera : cameras){ + if (camera.id().toStdString() == info.device_name()){ + return camera.description().toStdString(); + } + } + global_logger_tagged().log("Error: No such camera for CameraInfo: " + info.device_name(), COLOR_RED); + return ""; +} + +std::unique_ptr CameraBackend::make_video_source( + Logger& logger, + const CameraInfo& info, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps +) const{ + return std::make_unique(logger, info, resolution, format, fps); +} + + +CameraVideoSource::~CameraVideoSource(){ + if (!m_capture){ + return; + } + try{ + m_logger.log("Stopping Camera..."); + }catch (...){} + + run_on_main_thread_and_wait([&]{ + m_metaobject.reset(); + m_capture.reset(); + m_video_sink.reset(); + m_camera.reset(); + }); +} +CameraVideoSource::CameraVideoSource( + Logger& logger, + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps +) + : VideoSource(logger, true) + , m_logger(logger) + , m_last_frame(logger) + , m_snapshot_manager(logger, m_last_frame) +{ + if (!info){ + return; + } + m_logger.log("Starting Camera: Backend = CameraQt6QVideoWidget"); + + run_on_main_thread_and_wait([&]{ + init(info, desired_resolution, desired_format, desired_fps); + }); +} +void CameraVideoSource::init( + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps +){ + m_metaobject.reset(new QObject()); + + auto cameras = QMediaDevices::videoInputs(); + const QCameraDevice* device = nullptr; + for (const auto& camera : cameras){ + if (camera.id().toStdString() == info.device_name()){ + device = &camera; + break; + } + } + if (device == nullptr){ + m_logger.log("Camera not found: " + info.device_name(), COLOR_RED); + return; + } + m_logger.log("Camera: " + device->description().toStdString()); + + QCameraFormat format = build_format_set( + m_logger, + m_formats, + *device, + desired_resolution, + desired_format, + desired_fps + ); + if (format.isNull()){ + return; + } + + get_format(format, m_resolution, m_format, m_fps); + m_logger.log( + "Resolution: " + m_resolution.to_string() + + ", Format: " + VideoFormat_database().find(m_format)->display + + ", FPS: " + std::to_string(m_fps) + ); + + m_camera.reset(new QCameraThread(m_logger, *device, format)); + m_video_sink.reset(new QVideoSink()); + m_capture.reset(new QMediaCaptureSession()); + m_capture->setCamera(&m_camera->camera()); + m_capture->setVideoSink(m_video_sink.get()); + + m_metaobject->connect( + m_video_sink.get(), &QVideoSink::videoFrameChanged, + &m_camera->camera(), [this](const QVideoFrame& frame){ + WallClock copy_start = current_time(); + QVideoFrame owned = deep_copy_frame(frame); + WallClock copy_end = current_time(); + report_video_path_stats( + m_logger, "QOpenGLWidget", frame, owned.isValid(), copy_start, copy_end + ); + if (!owned.isValid()){ + return; + } + + WallClock now = current_time(); + if (!m_last_frame.push_frame(owned, now)){ + return; + } + report_source_frame(std::make_shared(now, owned)); + } + ); +} + +QWidget* CameraVideoSource::make_display_QtWidget(QWidget* parent){ + return new CameraVideoDisplay(parent, *this); +} + +CameraVideoDisplay::~CameraVideoDisplay(){ + m_source.remove_source_frame_listener(*this); +} +CameraVideoDisplay::CameraVideoDisplay(QWidget* parent, CameraVideoSource& source) + : QOpenGLWidget(parent) + , m_source(source) + , m_sanitizer("CameraVideoDisplay") +{ + this->setMinimumSize(80, 45); + + + source.add_source_frame_listener(*this); +} + +void CameraVideoDisplay::on_frame(std::shared_ptr frame){ + QMetaObject::invokeMethod(this, [this, frame = std::move(frame)]() mutable { + m_last_frame = std::move(frame); + this->update(); + }, Qt::QueuedConnection); +} + +void CameraVideoDisplay::paintEvent(QPaintEvent* event){ + QOpenGLWidget::paintEvent(event); + + if (!m_last_frame){ + return; + } + + QVideoFrame frame = m_last_frame->frame; + if (!frame.isValid()){ + return; + } + + QRect rect(0, 0, this->width(), this->height()); + QVideoFrame::PaintOptions options; + QPainter painter(this); + + frame.paint(&painter, rect, options); + m_source.report_rendered_frame(current_time()); +} + +} +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.h new file mode 100644 index 0000000000..4e3adf2cf4 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.h @@ -0,0 +1,118 @@ +/* Camera Widget (Qt6 QVideoWidget) + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_VideoPipeline_Qt6QVideoWidget_H +#define PokemonAutomation_VideoPipeline_Qt6QVideoWidget_H + +#include +#if QT_VERSION_MAJOR == 6 + +#include +#include +#include +#include "VideoFrameQt.h" +#include +#include "CommonFramework/Tools/StatAccumulator.h" +#include "CommonFramework/VideoPipeline/VideoSource.h" +#include "CommonFramework/VideoPipeline/CameraInfo.h" +#include "QCameraThread.h" +#include "QVideoFrameCache.h" +#include "SnapshotManager.h" +#include "CameraImplementations.h" + +class QCamera; +class QVideoSink; + +namespace PokemonAutomation{ +namespace CameraQt6QVideoWidget{ + +class CameraBackend : public PokemonAutomation::CameraBackend{ +public: + virtual std::vector get_all_cameras() const override; + virtual std::string get_camera_name(const CameraInfo& info) const override; + + virtual std::unique_ptr make_video_source( + Logger& logger, + const CameraInfo& info, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps + ) const override; +}; + +class CameraVideoSource : public VideoSource{ +public: + virtual ~CameraVideoSource(); + CameraVideoSource( + Logger& logger, + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps + ); + + virtual Resolution current_resolution() const override{ return m_resolution; } + virtual VideoFormat current_format() const override{ return m_format; } + virtual FramesPerSecond current_fps() const override{ return m_fps; } + virtual const VideoFormatSet& supported_formats() const override{ return m_formats; } + + virtual VideoSnapshot snapshot_latest_blocking() override{ + return m_snapshot_manager.snapshot_latest_blocking(); + } + virtual VideoSnapshot snapshot_recent_nonblocking(WallClock min_time) override{ + return m_snapshot_manager.snapshot_recent_nonblocking(min_time); + } + + virtual QWidget* make_display_QtWidget(QWidget* parent) override; + +private: + void init( + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps + ); + +private: + friend class CameraVideoDisplay; + + std::unique_ptr m_metaobject; + + Logger& m_logger; + Resolution m_resolution; + VideoFormat m_format; + FramesPerSecond m_fps; + + std::unique_ptr m_camera; + std::unique_ptr m_video_sink; + std::unique_ptr m_capture; + + VideoFormatSet m_formats; + +private: + QVideoFrameCache m_last_frame; + SnapshotManager m_snapshot_manager; +}; + +class CameraVideoDisplay : public QOpenGLWidget, private VideoFrameListener{ +public: + ~CameraVideoDisplay(); + CameraVideoDisplay(QWidget* parent, CameraVideoSource& source); + +private: + virtual void on_frame(std::shared_ptr frame) override; + virtual void paintEvent(QPaintEvent* event) override; + +private: + CameraVideoSource& m_source; + std::shared_ptr m_last_frame; + LifetimeSanitizer m_sanitizer; +}; + +} +} +#endif +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/VideoFrameQt.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/VideoFrameQt.cpp index d663439275..ee2b937f65 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/VideoFrameQt.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/VideoFrameQt.cpp @@ -5,6 +5,11 @@ */ #include +#include "Common/Cpp/Color.h" +#include +#include +#include "CommonFramework/GlobalSettingsPanel.h" +#include "CommonFramework/VideoPipeline/VideoPipelineOptions.h" #include "VideoFrameQt.h" namespace PokemonAutomation{ @@ -46,4 +51,127 @@ QVideoFrameFormat::PixelFormat VideoFormat_to_QVideoFrameFormat(VideoFormat form +// Capture backends (gstreamer in particular) hand out buffer pool frames. +// Retaining one keeps its buffer checked out and the pool allocates another, +// so memory grows without bound. Copy any frame that will be retained. +QVideoFrame deep_copy_frame(const QVideoFrame& src){ + if (!src.isValid()){ + return {}; + } + + QVideoFrame source = src; + if (!source.map(QVideoFrame::ReadOnly)){ + return {}; + } + + QVideoFrame dst(src.surfaceFormat()); + if (!dst.map(QVideoFrame::WriteOnly)){ + source.unmap(); + return {}; + } + + for (int plane = 0; plane < source.planeCount(); plane++){ + const uchar* src_bits = source.bits(plane); + uchar* dst_bits = dst.bits(plane); + if (src_bits == nullptr || dst_bits == nullptr){ + continue; + } + + int src_stride = source.bytesPerLine(plane); + int dst_stride = dst.bytesPerLine(plane); + int src_bytes = source.mappedBytes(plane); + int dst_bytes = dst.mappedBytes(plane); + + if (src_stride == dst_stride){ + std::memcpy(dst_bits, src_bits, std::min(src_bytes, dst_bytes)); + continue; + } + + // Both strides are at least the visible row length. + if (src_stride <= 0 || dst_stride <= 0){ + continue; + } + int rows = std::min(src_bytes / src_stride, dst_bytes / dst_stride); + int copy_bytes = std::min(src_stride, dst_stride); + for (int r = 0; r < rows; r++){ + std::memcpy(dst_bits + r * dst_stride, src_bits + r * src_stride, copy_bytes); + } + } + + source.unmap(); + dst.unmap(); + + // QVideoFrameCache::push_frame() dedups on startTime. + dst.setStartTime(src.startTime()); + dst.setEndTime(src.endTime()); + + return dst; +} + + + +static const char* color_space_name(QVideoFrameFormat::ColorSpace cs){ + switch (cs){ + case QVideoFrameFormat::ColorSpace_BT601: return "BT601"; + case QVideoFrameFormat::ColorSpace_BT709: return "BT709"; + case QVideoFrameFormat::ColorSpace_AdobeRgb: return "AdobeRgb"; + case QVideoFrameFormat::ColorSpace_BT2020: return "BT2020"; + default: return "Undefined"; + } +} +static const char* color_range_name(QVideoFrameFormat::ColorRange cr){ + switch (cr){ + case QVideoFrameFormat::ColorRange_Video: return "Limited"; + case QVideoFrameFormat::ColorRange_Full: return "Full"; + default: return "Unknown"; + } +} + + +void report_video_path_stats( + Logger& logger, const char* backend, + const QVideoFrame& src, bool copy_ok, + WallClock copy_start, WallClock copy_end +){ + if (!GlobalSettings::instance().VIDEO_PIPELINE->LOG_VIDEO_PATH_STATS){ + return; + } + + static WallClock last_report = current_time(); + static uint64_t frame_count = 0; + static uint64_t copy_failures = 0; + static uint64_t total_micros = 0; + + frame_count++; + if (!copy_ok){ + copy_failures++; + } + total_micros += (uint64_t)std::chrono::duration_cast( + copy_end - copy_start + ).count(); + + if (copy_end - last_report < std::chrono::seconds(1)){ + return; + } + + bool gpu = src.handleType() == QVideoFrame::RhiTextureHandle; + logger.log( + std::string("VIDEO PATH [") + backend + "]: handle = " + + (gpu ? "RhiTextureHandle (GPU)" : "NoHandle (CPU)") + + ", format = " + QVideoFrameFormat::pixelFormatToString(src.pixelFormat()).toStdString() + + ", size = " + std::to_string(src.width()) + "x" + std::to_string(src.height()) + + ", planes = " + std::to_string(src.planeCount()) + + ", colorspace = " + color_space_name(src.surfaceFormat().colorSpace()) + + ", range = " + color_range_name(src.surfaceFormat().colorRange()) + + ", fps = " + std::to_string(frame_count) + + ", copy_failures = " + std::to_string(copy_failures) + + ", deep_copy avg = " + std::to_string(total_micros / frame_count) + " us", + copy_failures > 0 ? COLOR_RED : COLOR_PURPLE + ); + last_report = copy_end; + frame_count = 0; + copy_failures = 0; + total_micros = 0; +} + } diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/VideoFrameQt.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/VideoFrameQt.h index 90e9e6d4ef..25d545da94 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/VideoFrameQt.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/VideoFrameQt.h @@ -9,6 +9,7 @@ #include #include "Common/Cpp/Time.h" +#include "Common/Cpp/Logging/AbstractLogger.h" #include "CommonFramework/VideoPipeline/VideoFormats.h" namespace PokemonAutomation{ @@ -46,5 +47,13 @@ class VideoFrame{ }; +QVideoFrame deep_copy_frame(const QVideoFrame& frame); + +void report_video_path_stats( + Logger& logger, const char* backend, + const QVideoFrame& src, bool copy_ok, + WallClock copy_start, WallClock copy_end +); + } #endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoPipelineOptions.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoPipelineOptions.h index f9969a1141..3358e742c9 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoPipelineOptions.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoPipelineOptions.h @@ -42,11 +42,20 @@ class VideoPipelineOptions : public GroupOption{ 5 #endif ) + , LOG_VIDEO_PATH_STATS( + "Log Video Path Diagnostics:
" + "Once a second, log what the capture backend is actually delivering: pixel format, " + "resolution, whether frames arrive on the CPU or as a GPU texture, and what it costs " + "to copy one. Useful when diagnosing a capture card that renders wrong or burns CPU.", + LockMode::UNLOCK_WHILE_RUNNING, + false + ) { PA_ADD_OPTION(SHOW_ALL_FPS); PA_ADD_OPTION(VIDEO_BACKEND0); PA_ADD_OPTION(AUTO_RESET_SECONDS); + PA_ADD_OPTION(LOG_VIDEO_PATH_STATS); } public: @@ -54,6 +63,7 @@ class VideoPipelineOptions : public GroupOption{ VideoBackendOption VIDEO_BACKEND0; SimpleIntegerOption AUTO_RESET_SECONDS; + BooleanCheckBoxOption LOG_VIDEO_PATH_STATS; }; diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSession.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSession.cpp index e9b7c12758..c824af88e3 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSession.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSession.cpp @@ -297,7 +297,11 @@ void VideoSession::internal_set_source( Resolution desired_resolution = resolution ? resolution : m_option.m_resolution; // cout << "desired_resolution = " << desired_resolution.to_string() << endl; + m_logger.log("Changing video to: " + device->display_name()); source = device->make_VideoSource(m_logger, desired_resolution, format, fps); + if (!source){ + m_logger.log("No video source was created.", COLOR_RED); + } if (source){ resolution = source->current_resolution(); format = source->current_format(); diff --git a/SerialPrograms/Source/CommonTools/Images/WaterfillUtilities.h b/SerialPrograms/Source/CommonTools/Images/WaterfillUtilities.h index 5aaaeee06a..61f5d3bb5b 100644 --- a/SerialPrograms/Source/CommonTools/Images/WaterfillUtilities.h +++ b/SerialPrograms/Source/CommonTools/Images/WaterfillUtilities.h @@ -10,6 +10,7 @@ #include #include +#include #include "Common/Cpp/ImageResolution.h" #include "CommonFramework/ImageTypes/BinaryImage.h" diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp index 9bed1fa736..0954ec683d 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp @@ -59,8 +59,8 @@ ConsoleSystemSession::ConsoleSystemSession( , m_console_number(console_number) , m_logger(global_logger_raw(), "Console " + std::to_string(console_number)) , m_option(option) - , m_video(m_logger, option.m_video) , m_audio(m_logger, option.m_audio) + , m_video(m_logger, option.m_video) , m_overlay(m_logger, option.m_overlay) , m_controllers(option.m_controllers.size()) , m_history(m_logger) @@ -114,8 +114,9 @@ void ConsoleSystemSession::load_json(const JsonValue& json){ std::lock_guard lg(m_lock); m_option.load_json(json); - m_video.load(m_option.m_video); + // Audio before video. See the member declarations. m_audio.load(m_option.m_audio); + m_video.load(m_option.m_video); m_overlay.load(m_option.m_overlay); size_t c = 0; diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.h b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.h index db00c25105..3579a54386 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.h +++ b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.h @@ -123,8 +123,10 @@ class ConsoleSystemSession final TaggedLogger m_logger; ConsoleSystemOption& m_option; - VideoSession m_video; + // Audio must be constructed before video. Some USB capture cards cannot + // start their audio stream once their video stream is running. AudioSession m_audio; + VideoSession m_video; VideoOverlaySession m_overlay; struct ControllerEntry{ diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.cpp b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.cpp index 3d83bad3c3..6cb4afb6b9 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.cpp @@ -10,6 +10,10 @@ #include #include #include +#if CV_VERSION_MAJOR >= 5 +// OpenCV 5 moved minAreaRect()/getPerspectiveTransform() out of imgproc into their own header. +#include +#endif #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Rectangle.tpp" #include "CommonFramework/Logging/Logger.h" diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp index 94ff02788e..d7735acd26 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp @@ -8,6 +8,11 @@ #include #include #include +#include +#if CV_VERSION_MAJOR >= 5 +// OpenCV 5 moved minAreaRect()/boxPoints() out of imgproc into their own header. +#include +#endif #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Filesystem/Filesystem.h" #include "Common/Cpp/Logging/GlobalLogger.h" diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/RngManipulation/PokemonBDSP_StateReidentifier.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/RngManipulation/PokemonBDSP_StateReidentifier.cpp index 098ec2dfe8..40df3c4159 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/RngManipulation/PokemonBDSP_StateReidentifier.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/RngManipulation/PokemonBDSP_StateReidentifier.cpp @@ -144,8 +144,7 @@ static ReidentifyResult reidentify_by_types( for (uint64_t phase = 0; phase < stride; phase++){ std::vector positions; std::vector observed_types; - uint64_t scanned = 0; - for (uint64_t index = phase; index < nibbles.size(); index += stride, scanned++){ + for (uint64_t index = phase; index < nibbles.size(); index += stride){ if (nibble_blinked(nibbles[(size_t)index])){ positions.emplace_back(index); observed_types.emplace_back(nibble_type(nibbles[(size_t)index])); diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/Map/PokemonLZA_DirectionArrowDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/Map/PokemonLZA_DirectionArrowDetector.cpp index f2aa1ffa2c..93bfca1f32 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/Map/PokemonLZA_DirectionArrowDetector.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/Map/PokemonLZA_DirectionArrowDetector.cpp @@ -11,6 +11,10 @@ #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" //#include "Tests/TestUtils.h" #include +#if CV_VERSION_MAJOR >= 5 +// OpenCV 5 moved minAreaRect()/boxPoints() out of imgproc into their own header. +#include +#endif #include #include diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 715214a137..e4dd0f2b34 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -690,8 +690,16 @@ file(GLOB LIBRARY_SOURCES Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.h Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.h + Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.cpp + Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.h + Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.cpp + Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.h Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickView.cpp Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickView.h + Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.cpp + Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.h + Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.cpp + Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.h Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.cpp Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.h Source/CommonFramework/VideoPipeline/Backends/MediaServicesQt6.cpp