mirror
/
electron
mirror of https://github.com/electron/electron
1
0
Fork 0
Code Issues Releases Activity
electron/patches/chromium/cherry-pick-7196a42b42ce.patch

252 lines
12 KiB
Diff
Raw Permalink Blame History

From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Johannes Kron <kron@chromium.org>
Date: Thu, 17 Nov 2022 19:54:33 +0000
Subject: Reconfigure stream on window resize when using ScreenCaptureKit
Bug: chromium:1352405
Change-Id: I315104eb4ee985a2d04ea90f01341129c6307501
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4027202
Reviewed-by: ccameron chromium <ccameron@chromium.org>
Reviewed-by: Robert Sesek <rsesek@chromium.org>
Commit-Queue: Johannes Kron <kron@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1072984}
diff --git a/content/browser/media/capture/screen_capture_kit_device_mac.mm b/content/browser/media/capture/screen_capture_kit_device_mac.mm
index c17f6f076fc852df85d5deb919434458b886db37..5d640a372f1649e0e2cb7e1828dbb3c66eb1e4d7 100644
--- a/content/browser/media/capture/screen_capture_kit_device_mac.mm
+++ b/content/browser/media/capture/screen_capture_kit_device_mac.mm
@@ -6,15 +6,18 @@
#import <ScreenCaptureKit/ScreenCaptureKit.h>
+#include "base/mac/foundation_util.h"
#include "base/mac/scoped_nsobject.h"
#include "base/task/bind_post_task.h"
#include "base/threading/thread_checker.h"
#include "base/timer/timer.h"
#include "content/browser/media/capture/io_surface_capture_device_base_mac.h"
+#include "third_party/abseil-cpp/absl/types/optional.h"
#include "third_party/webrtc/modules/desktop_capture/desktop_capture_types.h"
#include "ui/gfx/native_widget_types.h"
-using SampleCallback = base::RepeatingCallback<void(gfx::ScopedInUseIOSurface)>;
+using SampleCallback = base::RepeatingCallback<void(gfx::ScopedInUseIOSurface,
+ absl::optional<gfx::Size>)>;
using ErrorCallback = base::RepeatingClosure;
API_AVAILABLE(macos(12.3))
@@ -45,17 +48,64 @@ - (void)stream:(SCStream*)stream
CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
if (!pixelBuffer)
return;
+
+ // Read out width, height and scaling from metadata to determine the captured
+ // content size.
+ absl::optional<gfx::Size> contentSize;
+ CFArrayRef attachmentsArray =
+ CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, false);
+ if (attachmentsArray && CFArrayGetCount(attachmentsArray) > 0) {
+ CFDictionaryRef attachment = base::mac::CFCast<CFDictionaryRef>(
+ CFArrayGetValueAtIndex(attachmentsArray, 0));
+ if (attachment) {
+ CFDictionaryRef contentRectValue = base::mac::CFCast<CFDictionaryRef>(
+ CFDictionaryGetValue(attachment, SCStreamFrameInfoContentRect));
+ CFNumberRef contentScaleValue = base::mac::CFCast<CFNumberRef>(
+ CFDictionaryGetValue(attachment, SCStreamFrameInfoContentScale));
+ if (contentRectValue && contentScaleValue) {
+ CGRect contentRect = {};
+ bool succeed = CGRectMakeWithDictionaryRepresentation(contentRectValue,
+ &contentRect);
+ float contentScale = 1.0f;
+ succeed &= CFNumberGetValue(contentScaleValue, kCFNumberFloatType,
+ &contentScale);
+ if (succeed) {
+ contentSize.emplace(round(contentRect.size.width / contentScale),
+ round(contentRect.size.height / contentScale));
+ }
+ }
+ }
+ }
IOSurfaceRef ioSurface = CVPixelBufferGetIOSurface(pixelBuffer);
if (!ioSurface)
return;
_sampleCallback.Run(
- gfx::ScopedInUseIOSurface(ioSurface, base::scoped_policy::RETAIN));
+ gfx::ScopedInUseIOSurface(ioSurface, base::scoped_policy::RETAIN),
+ contentSize);
}
- (void)stream:(SCStream*)stream didStopWithError:(NSError*)error {
_errorCallback.Run();
}
++ (base::scoped_nsobject<SCStreamConfiguration>)
+ createStreamConfigurationWithFrameSize:(gfx::Size)frameSize
+ destRectInFrame:(gfx::RectF)destRectInFrame
+ frameRate:(float)frameRate {
+ base::scoped_nsobject<SCStreamConfiguration> config(
+ [[SCStreamConfiguration alloc] init]);
+ [config setWidth:frameSize.width()];
+ [config setHeight:frameSize.height()];
+ [config setPixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange];
+ [config setDestinationRect:destRectInFrame.ToCGRect()];
+ [config setBackgroundColor:CGColorGetConstantColor(kCGColorBlack)];
+ [config setScalesToFit:YES];
+ [config setShowsCursor:YES];
+ [config setColorSpaceName:kCGColorSpaceSRGB];
+ [config setMinimumFrameInterval:CMTimeMakeWithSeconds(1 / frameRate, 1)];
+ return config;
+}
+
@end
namespace content {
@@ -95,7 +145,6 @@ void OnShareableContentCreated(
}
base::scoped_nsobject<SCContentFilter> filter;
- gfx::Size content_size;
switch (source_.type) {
case DesktopMediaID::TYPE_SCREEN:
for (SCDisplay* display : [content displays]) {
@@ -112,7 +161,8 @@ void OnShareableContentCreated(
filter.reset([[SCContentFilter alloc]
initWithDisplay:display
excludingWindows:exclude_windows]);
- content_size = gfx::Size([display width], [display height]);
+ stream_config_content_size_ =
+ gfx::Size([display width], [display height]);
break;
}
}
@@ -123,7 +173,7 @@ void OnShareableContentCreated(
filter.reset([[SCContentFilter alloc]
initWithDesktopIndependentWindow:window]);
CGRect frame = [window frame];
- content_size = gfx::Size(frame.size);
+ stream_config_content_size_ = gfx::Size(frame.size);
break;
}
}
@@ -142,22 +192,16 @@ void OnShareableContentCreated(
gfx::RectF dest_rect_in_frame;
actual_capture_format_ = capture_params().requested_format;
actual_capture_format_.pixel_format = media::PIXEL_FORMAT_NV12;
- ComputeFrameSizeAndDestRect(content_size, actual_capture_format_.frame_size,
+ ComputeFrameSizeAndDestRect(stream_config_content_size_,
+ actual_capture_format_.frame_size,
dest_rect_in_frame);
-
- base::scoped_nsobject<SCStreamConfiguration> config(
- [[SCStreamConfiguration alloc] init]);
- [config setWidth:actual_capture_format_.frame_size.width()];
- [config setHeight:actual_capture_format_.frame_size.height()];
- [config setPixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange];
- [config setDestinationRect:dest_rect_in_frame.ToCGRect()];
- [config setBackgroundColor:CGColorGetConstantColor(kCGColorBlack)];
- [config setScalesToFit:YES];
- [config setShowsCursor:YES];
- [config setColorSpaceName:kCGColorSpaceSRGB];
- [config
- setMinimumFrameInterval:CMTimeMakeWithSeconds(
- 1 / actual_capture_format_.frame_rate, 1)];
+ base::scoped_nsobject<SCStreamConfiguration> config =
+ [ScreenCaptureKitDeviceHelper
+ createStreamConfigurationWithFrameSize:actual_capture_format_
+ .frame_size
+ destRectInFrame:dest_rect_in_frame
+ frameRate:actual_capture_format_
+ .frame_rate];
stream_.reset([[SCStream alloc] initWithFilter:filter
configuration:config
delegate:helper_]);
@@ -203,9 +247,71 @@ void OnStreamStopped(bool error) {
return;
}
}
- void OnStreamSample(gfx::ScopedInUseIOSurface io_surface) {
- // TODO(https://crbug.com/1309653): Reconfigure the stream if the IOSurface
- // should be resized.
+ void OnStreamSample(gfx::ScopedInUseIOSurface io_surface,
+ absl::optional<gfx::Size> content_size) {
+ if (requested_capture_format_) {
+ // Does the size of io_surface match the requested format?
+ size_t io_surface_width = IOSurfaceGetWidth(io_surface);
+ size_t io_surface_height = IOSurfaceGetHeight(io_surface);
+ DVLOG(3) << "Waiting for new capture format, "
+ << requested_capture_format_->frame_size.width() << " x "
+ << requested_capture_format_->frame_size.height()
+ << ". IO surface size " << io_surface_width << " x "
+ << io_surface_height;
+ if (static_cast<size_t>(requested_capture_format_->frame_size.width()) ==
+ io_surface_width &&
+ static_cast<size_t>(requested_capture_format_->frame_size.height()) ==
+ io_surface_height) {
+ actual_capture_format_ = requested_capture_format_.value();
+ requested_capture_format_.reset();
+ }
+ } else {
+ // No current request for new capture format. Check to see if content_size
+ // has changed and requires an updated configuration.
+ if (content_size &&
+ (stream_config_content_size_.width() != content_size->width() ||
+ stream_config_content_size_.height() != content_size->height())) {
+ DVLOG(3) << "Content size changed to " << content_size->width() << " x "
+ << content_size->height() << ". It was "
+ << stream_config_content_size_.width() << " x "
+ << stream_config_content_size_.height();
+ stream_config_content_size_ = content_size.value();
+ gfx::RectF dest_rect_in_frame;
+ gfx::Size new_frame_size;
+ ComputeFrameSizeAndDestRect(stream_config_content_size_, new_frame_size,
+ dest_rect_in_frame);
+ if (new_frame_size.width() !=
+ actual_capture_format_.frame_size.width() ||
+ new_frame_size.height() !=
+ actual_capture_format_.frame_size.height()) {
+ DVLOG(3) << "Calling updateConfiguration with new frame size: "
+ << new_frame_size.width() << " x "
+ << new_frame_size.height();
+ requested_capture_format_ = actual_capture_format_;
+ requested_capture_format_->frame_size = new_frame_size;
+ // Update stream configuration.
+ base::scoped_nsobject<SCStreamConfiguration> config =
+ [ScreenCaptureKitDeviceHelper
+ createStreamConfigurationWithFrameSize:
+ requested_capture_format_->frame_size
+ destRectInFrame:dest_rect_in_frame
+ frameRate:
+ requested_capture_format_->
+ frame_rate];
+ [stream_
+ updateConfiguration:config
+ completionHandler:^(NSError* _Nullable error) {
+ if (error) {
+ client()->OnError(
+ media::VideoCaptureError::kScreenCaptureKitStreamError,
+ FROM_HERE, "Error on updateConfiguration");
+ }
+ }];
+ }
+ }
+ }
+ // TODO(https://crbug.com/1352405): Set visible rect to make it possible to
+ // crop the frame when it's rendered/encoded.
OnReceivedIOSurfaceFromStream(io_surface, actual_capture_format_);
}
void OnStreamError() {
@@ -257,6 +363,13 @@ void OnStop() override {
// The actual format of the video frames that are sent to `client`.
media::VideoCaptureFormat actual_capture_format_;
+ // The requested format if a request to update the configuration has been
+ // sent.
+ absl::optional<media::VideoCaptureFormat> requested_capture_format_;
+
+ // The size of the content at the time that we configured the stream.
+ gfx::Size stream_config_content_size_;
+
// Helper class that acts as output and delegate for `stream_`.
base::scoped_nsobject<ScreenCaptureKitDeviceHelper> helper_;
View Git Blame Copy Permalink