From 5ae9b260ffd0ab25b56c0ef1fbe3bd088e91c7df Mon Sep 17 00:00:00 2001 From: Jan Grulich Date: Tue, 15 Feb 2022 21:36:46 +0100 Subject: [PATCH] Implement MouseCursorMonitorPipeWire to track cursor changes separately Current implementation has mouse cursor as part of the screen itself which means that everytime a cursor changes location, we have to update whole screen content, which brings unnecessary load overhead. Using our own mouse cursor monitor implementation allows us to track only mouse cursor changes and update them separately for much better performance. Bug: webrtc:13429 Change-Id: I224e9145f0bc7e45eafe4490de160f2ad4c8b545 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/244507 Reviewed-by: Tomas Gunnarsson Reviewed-by: Mark Foltz Commit-Queue: Mark Foltz Cr-Commit-Position: refs/heads/main@{#36011} --- modules/desktop_capture/BUILD.gn | 2 + .../wayland/mouse_cursor_monitor_pipewire.cc | 59 +++++++++++++ .../wayland/mouse_cursor_monitor_pipewire.h | 43 ++++++++++ .../linux/wayland/screencast_portal.h | 2 +- .../linux/wayland/shared_screencast_stream.cc | 83 ++++++++++++++++++- .../linux/wayland/shared_screencast_stream.h | 23 +++++ .../mouse_cursor_monitor_linux.cc | 11 +++ 7 files changed, 218 insertions(+), 5 deletions(-) create mode 100644 modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.cc create mode 100644 modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.h diff --git a/modules/desktop_capture/BUILD.gn b/modules/desktop_capture/BUILD.gn index 63a0bb3412..25ea1cb05a 100644 --- a/modules/desktop_capture/BUILD.gn +++ b/modules/desktop_capture/BUILD.gn @@ -561,6 +561,8 @@ rtc_library("desktop_capture_generic") { "linux/wayland/base_capturer_pipewire.h", "linux/wayland/egl_dmabuf.cc", "linux/wayland/egl_dmabuf.h", + "linux/wayland/mouse_cursor_monitor_pipewire.cc", + "linux/wayland/mouse_cursor_monitor_pipewire.h", "linux/wayland/scoped_glib.cc", "linux/wayland/scoped_glib.h", "linux/wayland/screencast_portal.cc", diff --git a/modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.cc b/modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.cc new file mode 100644 index 0000000000..3d33b0fbb8 --- /dev/null +++ b/modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.cc @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.h" + +#include + +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_capturer.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +MouseCursorMonitorPipeWire::MouseCursorMonitorPipeWire( + const DesktopCaptureOptions& options) + : options_(options) { + sequence_checker_.Detach(); +} + +MouseCursorMonitorPipeWire::~MouseCursorMonitorPipeWire() {} + +void MouseCursorMonitorPipeWire::Init(Callback* callback, Mode mode) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(!callback_); + RTC_DCHECK(callback); + + callback_ = callback; + mode_ = mode; +} + +void MouseCursorMonitorPipeWire::Capture() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(callback_); + + std::unique_ptr mouse_cursor = + options_.screencast_stream()->CaptureCursor(); + + if (mouse_cursor && mouse_cursor->image()->data()) { + callback_->OnMouseCursor(mouse_cursor.release()); + } + + if (mode_ == SHAPE_AND_POSITION) { + absl::optional mouse_cursor_position = + options_.screencast_stream()->CaptureCursorPosition(); + if (mouse_cursor_position) { + callback_->OnMouseCursorPosition(mouse_cursor_position.value()); + } + } +} + +} // namespace webrtc diff --git a/modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.h b/modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.h new file mode 100644 index 0000000000..8af4f879f8 --- /dev/null +++ b/modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.h @@ -0,0 +1,43 @@ +/* + * Copyright 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_LINUX_WAYLAND_MOUSE_CURSOR_MONITOR_PIPEWIRE_H_ +#define MODULES_DESKTOP_CAPTURE_LINUX_WAYLAND_MOUSE_CURSOR_MONITOR_PIPEWIRE_H_ + +#include + +#include "api/scoped_refptr.h" +#include "api/sequence_checker.h" +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_capture_types.h" +#include "modules/desktop_capture/linux/wayland/shared_screencast_stream.h" +#include "modules/desktop_capture/mouse_cursor.h" +#include "modules/desktop_capture/mouse_cursor_monitor.h" + +namespace webrtc { + +class MouseCursorMonitorPipeWire : public MouseCursorMonitor { + public: + explicit MouseCursorMonitorPipeWire(const DesktopCaptureOptions& options); + ~MouseCursorMonitorPipeWire() override; + + // MouseCursorMonitor: + void Init(Callback* callback, Mode mode) override; + void Capture() override; + + DesktopCaptureOptions options_ RTC_GUARDED_BY(sequence_checker_); + Callback* callback_ RTC_GUARDED_BY(sequence_checker_) = nullptr; + Mode mode_ RTC_GUARDED_BY(sequence_checker_) = SHAPE_AND_POSITION; + RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; +}; + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_LINUX_WAYLAND_MOUSE_CURSOR_MONITOR_PIPEWIRE_H_ diff --git a/modules/desktop_capture/linux/wayland/screencast_portal.h b/modules/desktop_capture/linux/wayland/screencast_portal.h index 00f78af578..fd6bb1d468 100644 --- a/modules/desktop_capture/linux/wayland/screencast_portal.h +++ b/modules/desktop_capture/linux/wayland/screencast_portal.h @@ -90,7 +90,7 @@ class ScreenCastPortal { CaptureSourceType capture_source_type_ = ScreenCastPortal::CaptureSourceType::kScreen; - CursorMode cursor_mode_ = ScreenCastPortal::CursorMode::kEmbedded; + CursorMode cursor_mode_ = ScreenCastPortal::CursorMode::kMetadata; GDBusConnection* connection_ = nullptr; GDBusProxy* proxy_ = nullptr; diff --git a/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc b/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc index f49bf1e325..a8c86e267b 100644 --- a/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc +++ b/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc @@ -57,6 +57,12 @@ const char kDrmLib[] = "libdrm.so.2"; #define SPA_POD_PROP_FLAG_DONT_FIXATE (1u << 4) #endif +constexpr int kCursorBpp = 4; +constexpr int CursorMetaSize(int w, int h) { + return (sizeof(struct spa_meta_cursor) + sizeof(struct spa_meta_bitmap) + + w * h * kCursorBpp); +} + struct PipeWireVersion { int major = 0; int minor = 0; @@ -183,6 +189,8 @@ class SharedScreenCastStreamPrivate { bool StartScreenCastStream(uint32_t stream_node_id, int fd); void StopScreenCastStream(); std::unique_ptr CaptureFrame(); + std::unique_ptr CaptureCursor(); + DesktopVector CaptureCursorPosition(); private: uint32_t pw_stream_node_id_ = 0; @@ -194,6 +202,8 @@ class SharedScreenCastStreamPrivate { webrtc::Mutex queue_lock_; ScreenCaptureFrameQueue queue_ RTC_GUARDED_BY(&queue_lock_); + std::unique_ptr mouse_cursor_; + DesktopVector mouse_cursor_position_ = DesktopVector(-1, -1); int64_t modifier_; std::unique_ptr egl_dmabuf_; @@ -387,6 +397,18 @@ void SharedScreenCastStreamPrivate::OnStreamParamChanged( &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, SPA_POD_Id(SPA_META_VideoCrop), SPA_PARAM_META_size, SPA_POD_Int(sizeof(struct spa_meta_region))))); + params.push_back(reinterpret_cast(spa_pod_builder_add_object( + &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, + SPA_POD_Id(SPA_META_Cursor), SPA_PARAM_META_size, + SPA_POD_CHOICE_RANGE_Int(CursorMetaSize(64, 64), CursorMetaSize(1, 1), + CursorMetaSize(384, 384))))); + params.push_back(reinterpret_cast(spa_pod_builder_add_object( + &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, + SPA_POD_Id(SPA_META_VideoDamage), SPA_PARAM_META_size, + SPA_POD_CHOICE_RANGE_Int(sizeof(struct spa_meta_region) * 16, + sizeof(struct spa_meta_region) * 1, + sizeof(struct spa_meta_region) * 16)))); + pw_stream_update_params(that->pw_stream_, params.data(), params.size()); } @@ -604,14 +626,53 @@ std::unique_ptr SharedScreenCastStreamPrivate::CaptureFrame() { return std::move(frame); } +std::unique_ptr SharedScreenCastStreamPrivate::CaptureCursor() { + if (!mouse_cursor_) { + return nullptr; + } + + return std::move(mouse_cursor_); +} + +DesktopVector SharedScreenCastStreamPrivate::CaptureCursorPosition() { + return mouse_cursor_position_; +} + void SharedScreenCastStreamPrivate::ProcessBuffer(pw_buffer* buffer) { spa_buffer* spa_buffer = buffer->buffer; ScopedBuf map; std::unique_ptr src_unique_ptr; uint8_t* src = nullptr; + // Try to update the mouse cursor first, because it can be the only + // information carried by the buffer + { + const struct spa_meta_cursor* cursor = + static_cast(spa_buffer_find_meta_data( + spa_buffer, SPA_META_Cursor, sizeof(*cursor))); + if (spa_meta_cursor_is_valid(cursor)) { + struct spa_meta_bitmap* bitmap = nullptr; + + if (cursor->bitmap_offset) + bitmap = + SPA_MEMBER(cursor, cursor->bitmap_offset, struct spa_meta_bitmap); + + if (bitmap && bitmap->size.width > 0 && bitmap->size.height > 0) { + const uint8_t* bitmap_data = + SPA_MEMBER(bitmap, bitmap->offset, uint8_t); + BasicDesktopFrame* mouse_frame = new BasicDesktopFrame( + DesktopSize(bitmap->size.width, bitmap->size.height)); + mouse_frame->CopyPixelsFrom( + bitmap_data, bitmap->stride, + DesktopRect::MakeWH(bitmap->size.width, bitmap->size.height)); + mouse_cursor_ = std::make_unique( + mouse_frame, DesktopVector(cursor->hotspot.x, cursor->hotspot.y)); + } + mouse_cursor_position_.set(cursor->position.x, cursor->position.y); + } + } + if (spa_buffer->datas[0].chunk->size == 0) { - RTC_LOG(LS_ERROR) << "Failed to get video stream: Zero size."; return; } @@ -674,7 +735,6 @@ void SharedScreenCastStreamPrivate::ProcessBuffer(pw_buffer* buffer) { if (!src) { return; } - struct spa_meta_region* video_metadata = static_cast(spa_buffer_find_meta_data( spa_buffer, SPA_META_VideoCrop, sizeof(*video_metadata))); @@ -753,8 +813,8 @@ void SharedScreenCastStreamPrivate::ProcessBuffer(pw_buffer* buffer) { spa_video_format_.format == SPA_VIDEO_FORMAT_RGBA) { uint8_t* tmp_src = queue_.current_frame()->data(); for (int i = 0; i < video_size_.height(); ++i) { - // If both sides decided to go with the RGBx format we need to convert it - // to BGRx to match color format expected by WebRTC. + // If both sides decided to go with the RGBx format we need to convert + // it to BGRx to match color format expected by WebRTC. ConvertRGBxToBGRx(tmp_src, queue_.current_frame()->stride()); tmp_src += queue_.current_frame()->stride(); } @@ -795,4 +855,19 @@ std::unique_ptr SharedScreenCastStream::CaptureFrame() { return private_->CaptureFrame(); } +std::unique_ptr SharedScreenCastStream::CaptureCursor() { + return private_->CaptureCursor(); +} + +absl::optional SharedScreenCastStream::CaptureCursorPosition() { + DesktopVector position = private_->CaptureCursorPosition(); + + // Consider only (x >= 0 and y >= 0) a valid position + if (position.x() < 0 || position.y() < 0) { + return absl::nullopt; + } + + return position; +} + } // namespace webrtc diff --git a/modules/desktop_capture/linux/wayland/shared_screencast_stream.h b/modules/desktop_capture/linux/wayland/shared_screencast_stream.h index d7105ba7f4..443ec745d5 100644 --- a/modules/desktop_capture/linux/wayland/shared_screencast_stream.h +++ b/modules/desktop_capture/linux/wayland/shared_screencast_stream.h @@ -13,9 +13,11 @@ #include +#include "absl/types/optional.h" #include "api/ref_counted_base.h" #include "api/scoped_refptr.h" #include "modules/desktop_capture/desktop_frame.h" +#include "modules/desktop_capture/mouse_cursor.h" #include "rtc_base/system/rtc_export.h" namespace webrtc { @@ -29,8 +31,29 @@ class RTC_EXPORT SharedScreenCastStream bool StartScreenCastStream(uint32_t stream_node_id, int fd); void StopScreenCastStream(); + + // Below functions return the most recent information we get from a + // PipeWire buffer on each Process() callback. This assumes that we + // managed to successfuly connect to a PipeWire stream provided by the + // compositor (based on stream parameters). The cursor data are obtained + // from spa_meta_cursor stream metadata and therefore the cursor is not + // part of actual screen/window frame. + + // Returns the most recent screen/window frame we obtained from PipeWire + // buffer. Will return an empty frame in case we didn't manage to get a frame + // from PipeWire buffer. std::unique_ptr CaptureFrame(); + // Returns the most recent mouse cursor image. Will return an nullptr cursor + // in case we didn't manage to get a cursor from PipeWire buffer. NOTE: the + // cursor image might not be updated on every cursor location change, but + // actually only when its shape changes. + std::unique_ptr CaptureCursor(); + + // Returns the most recent mouse cursor position. Will not return a value in + // case we didn't manage to get it from PipeWire buffer. + absl::optional CaptureCursorPosition(); + ~SharedScreenCastStream(); protected: diff --git a/modules/desktop_capture/mouse_cursor_monitor_linux.cc b/modules/desktop_capture/mouse_cursor_monitor_linux.cc index 87edb40373..b44500d4e8 100644 --- a/modules/desktop_capture/mouse_cursor_monitor_linux.cc +++ b/modules/desktop_capture/mouse_cursor_monitor_linux.cc @@ -17,6 +17,10 @@ #include "modules/desktop_capture/linux/x11/mouse_cursor_monitor_x11.h" #endif // defined(WEBRTC_USE_X11) +#if defined(WEBRTC_USE_PIPEWIRE) +#include "modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.h" +#endif // defined(WEBRTC_USE_PIPEWIRE) + namespace webrtc { // static @@ -44,6 +48,13 @@ MouseCursorMonitor* MouseCursorMonitor::CreateForScreen( // static std::unique_ptr MouseCursorMonitor::Create( const DesktopCaptureOptions& options) { +#if defined(WEBRTC_USE_PIPEWIRE) + if (options.allow_pipewire() && DesktopCapturer::IsRunningUnderWayland() && + options.screencast_stream()) { + return std::make_unique(options); + } +#endif // defined(WEBRTC_USE_PIPEWIRE) + #if defined(WEBRTC_USE_X11) return MouseCursorMonitorX11::Create(options); #else