From edd2cc807c82013ef9b0da236b32af129858d55d Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Fri, 7 May 2021 18:05:26 -0700 Subject: [PATCH] browser(ff): migrate screencast to client interfaces --- browser_patches/firefox/BUILD_NUMBER | 4 +- .../firefox/juggler/TargetRegistry.js | 42 ++++++--- .../juggler/protocol/BrowserHandler.js | 9 +- .../firefox/juggler/protocol/PageHandler.js | 4 +- .../screencast/HeadlessWindowCapturer.cpp | 9 +- .../screencast/nsIScreencastService.idl | 10 +- .../screencast/nsScreencastService.cpp | 94 ++++++++----------- 7 files changed, 94 insertions(+), 78 deletions(-) diff --git a/browser_patches/firefox/BUILD_NUMBER b/browser_patches/firefox/BUILD_NUMBER index ba4a7d0862..5849ce373e 100644 --- a/browser_patches/firefox/BUILD_NUMBER +++ b/browser_patches/firefox/BUILD_NUMBER @@ -1,2 +1,2 @@ -1255 -Changed: pavel.feldman@gmail.com Thu 06 May 2021 06:33:59 PM PDT +1256 +Changed: pavel.feldman@gmail.com Fri 07 May 2021 09:00:05 PM PDT diff --git a/browser_patches/firefox/juggler/TargetRegistry.js b/browser_patches/firefox/juggler/TargetRegistry.js index e0142f254f..9613fc83c6 100644 --- a/browser_patches/firefox/juggler/TargetRegistry.js +++ b/browser_patches/firefox/juggler/TargetRegistry.js @@ -88,6 +88,8 @@ class DownloadInterceptor { } } +const screencastService = Cc['@mozilla.org/juggler/screencast;1'].getService(Ci.nsIScreencastService); + class TargetRegistry { constructor() { EventEmitter.decorate(this); @@ -353,7 +355,6 @@ class PageTarget { this._registry._browserBrowsingContextToTarget.set(this._linkedBrowser.browsingContext, this); this._registry.emit(TargetRegistry.Events.TargetCreated, this); - this._screencast = Cc['@mozilla.org/juggler/screencast;1'].getService(Ci.nsIScreencastService); } dialog(dialogId) { @@ -503,7 +504,17 @@ class PageTarget { // Exclude address bar and navigation control from the video. const rect = this.linkedBrowser().getBoundingClientRect(); const devicePixelRatio = this._window.devicePixelRatio; - const sessionId = this._screencast.startVideoRecording(docShell, true, file, width, height, 0, devicePixelRatio * rect.top); + let sessionId; + const registry = this._registry; + const screencastClient = { + QueryInterface: ChromeUtils.generateQI([Ci.nsIScreencastServiceClient]), + screencastFrame(data, deviceWidth, deviceHeight) { + }, + screencastStopped() { + registry.emit(TargetRegistry.Events.ScreencastStopped, sessionId); + }, + }; + sessionId = screencastService.startVideoRecording(screencastClient, docShell, true, file, width, height, 0, devicePixelRatio * rect.top); this._videoRecordingInfo = { sessionId, file }; this.emit(PageTarget.Events.ScreencastStarted); } @@ -513,7 +524,7 @@ class PageTarget { throw new Error('No video recording in progress'); const videoRecordingInfo = this._videoRecordingInfo; this._videoRecordingInfo = undefined; - this._screencast.stopVideoRecording(videoRecordingInfo.sessionId); + screencastService.stopVideoRecording(videoRecordingInfo.sessionId); } videoRecordingInfo() { @@ -532,28 +543,34 @@ class PageTarget { // Exclude address bar and navigation control from the video. const rect = this.linkedBrowser().getBoundingClientRect(); const devicePixelRatio = this._window.devicePixelRatio; - const screencastId = this._screencast.startVideoRecording(docShell, false, '', width, height, quality || 90, devicePixelRatio * rect.top); - const onFrame = (subject, topic, data) => { - this.emit(PageTarget.Events.ScreencastFrame, data); + + const self = this; + const screencastClient = { + QueryInterface: ChromeUtils.generateQI([Ci.nsIScreencastServiceClient]), + screencastFrame(data, deviceWidth, deviceHeight) { + if (self._screencastRecordingInfo) + self.emit(PageTarget.Events.ScreencastFrame, { data, deviceWidth, deviceHeight }); + }, + screencastStopped() { + }, }; - Services.obs.addObserver(onFrame, 'juggler-screencast-frame'); - this._screencastRecordingInfo = { screencastId, onFrame }; + const screencastId = screencastService.startVideoRecording(screencastClient, docShell, false, '', width, height, quality || 90, devicePixelRatio * rect.top); + this._screencastRecordingInfo = { screencastId }; return { screencastId }; } screencastFrameAck({ screencastId }) { if (!this._screencastRecordingInfo || this._screencastRecordingInfo.screencastId !== screencastId) return; - this._screencast.screencastFrameAck(screencastId); + screencastService.screencastFrameAck(screencastId); } stopScreencast() { if (!this._screencastRecordingInfo) throw new Error('No screencast in progress'); - const screencastInfo = this._screencastRecordingInfo; - Services.obs.removeObserver(screencastInfo.onFrame, 'juggler-screencast-frame'); + const { screencastId } = this._screencastRecordingInfo; this._screencastRecordingInfo = undefined; - this._screencast.stopVideoRecording(screencastInfo.screencastId); + screencastService.stopVideoRecording(screencastId); } dispose() { @@ -934,6 +951,7 @@ TargetRegistry.Events = { TargetDestroyed: Symbol('TargetRegistry.Events.TargetDestroyed'), DownloadCreated: Symbol('TargetRegistry.Events.DownloadCreated'), DownloadFinished: Symbol('TargetRegistry.Events.DownloadFinished'), + ScreencastStopped: Symbol('TargetRegistry.ScreencastStopped'), }; var EXPORTED_SYMBOLS = ['TargetRegistry', 'PageTarget']; diff --git a/browser_patches/firefox/juggler/protocol/BrowserHandler.js b/browser_patches/firefox/juggler/protocol/BrowserHandler.js index 7663b8d284..48c43ab7f3 100644 --- a/browser_patches/firefox/juggler/protocol/BrowserHandler.js +++ b/browser_patches/firefox/juggler/protocol/BrowserHandler.js @@ -37,14 +37,11 @@ class BrowserHandler { helper.on(this._targetRegistry, TargetRegistry.Events.TargetDestroyed, this._onTargetDestroyed.bind(this)), helper.on(this._targetRegistry, TargetRegistry.Events.DownloadCreated, this._onDownloadCreated.bind(this)), helper.on(this._targetRegistry, TargetRegistry.Events.DownloadFinished, this._onDownloadFinished.bind(this)), + helper.on(this._targetRegistry, TargetRegistry.Events.ScreencastStopped, sessionId => { + this._session.emitEvent('Browser.videoRecordingFinished', {screencastId: '' + sessionId}); + }) ]; - const onScreencastStopped = (subject, topic, data) => { - this._session.emitEvent('Browser.videoRecordingFinished', {screencastId: '' + data}); - }; - Services.obs.addObserver(onScreencastStopped, 'juggler-screencast-stopped'); - this._eventListeners.push(() => Services.obs.removeObserver(onScreencastStopped, 'juggler-screencast-stopped')); - for (const target of this._targetRegistry.targets()) this._onTargetCreated(target); diff --git a/browser_patches/firefox/juggler/protocol/PageHandler.js b/browser_patches/firefox/juggler/protocol/PageHandler.js index c6484a3837..964742b00d 100644 --- a/browser_patches/firefox/juggler/protocol/PageHandler.js +++ b/browser_patches/firefox/juggler/protocol/PageHandler.js @@ -152,8 +152,8 @@ class PageHandler { this._session.emitEvent('Page.videoRecordingStarted', { screencastId: info.sessionId, file: info.file }); } - _onScreencastFrame(data) { - this._session.emitEvent('Page.screencastFrame', { data, deviceWidth: 0, deviceHeight: 0 }); + _onScreencastFrame(params) { + this._session.emitEvent('Page.screencastFrame', params); } _onPageReady(event) { diff --git a/browser_patches/firefox/juggler/screencast/HeadlessWindowCapturer.cpp b/browser_patches/firefox/juggler/screencast/HeadlessWindowCapturer.cpp index 416164d7e4..66cb94b0c8 100644 --- a/browser_patches/firefox/juggler/screencast/HeadlessWindowCapturer.cpp +++ b/browser_patches/firefox/juggler/screencast/HeadlessWindowCapturer.cpp @@ -89,8 +89,13 @@ int32_t HeadlessWindowCapturer::StartCapture(const VideoCaptureCapability& capab frameInfo.videoType = VideoType::kBGRA; #endif - for (auto rawFrameCallback : _rawFrameCallbacks) { - rawFrameCallback->OnRawFrame(dataSurface->GetData(), dataSurface->Stride(), frameInfo); + { + rtc::CritScope lock2(&_callBackCs); + for (auto rawFrameCallback : _rawFrameCallbacks) { + rawFrameCallback->OnRawFrame(dataSurface->GetData(), dataSurface->Stride(), frameInfo); + } + if (!_dataCallBacks.size()) + return; } int width = dataSurface->GetSize().width; diff --git a/browser_patches/firefox/juggler/screencast/nsIScreencastService.idl b/browser_patches/firefox/juggler/screencast/nsIScreencastService.idl index 3d205e2fbe..302a47ce7c 100644 --- a/browser_patches/firefox/juggler/screencast/nsIScreencastService.idl +++ b/browser_patches/firefox/juggler/screencast/nsIScreencastService.idl @@ -6,13 +6,21 @@ interface nsIDocShell; +[scriptable, uuid(0b5d32c4-aeeb-11eb-8529-0242ac130003)] +interface nsIScreencastServiceClient : nsISupports +{ + void screencastFrame(in AString frame, in uint32_t deviceWidth, in uint32_t deviceHeight); + + void screencastStopped(); +}; + /** * Service for recording window video. */ [scriptable, uuid(d8c4d9e0-9462-445e-9e43-68d3872ad1de)] interface nsIScreencastService : nsISupports { - AString startVideoRecording(in nsIDocShell docShell, in boolean isVideo, in ACString fileName, in uint32_t width, in uint32_t height, in uint32_t quality, in int32_t offset_top); + AString startVideoRecording(in nsIScreencastServiceClient client, in nsIDocShell docShell, in boolean isVideo, in ACString fileName, in uint32_t width, in uint32_t height, in uint32_t quality, in int32_t offset_top); /** * Will emit 'juggler-screencast-stopped' when the video file is saved. diff --git a/browser_patches/firefox/juggler/screencast/nsScreencastService.cpp b/browser_patches/firefox/juggler/screencast/nsScreencastService.cpp index 5e006255e1..90699235e9 100644 --- a/browser_patches/firefox/juggler/screencast/nsScreencastService.cpp +++ b/browser_patches/firefox/juggler/screencast/nsScreencastService.cpp @@ -16,7 +16,6 @@ #include "nsIRandomGenerator.h" #include "nsISupportsPrimitives.h" #include "nsThreadManager.h" -#include "nsReadableUtils.h" #include "nsView.h" #include "nsViewManager.h" #include "webrtc/modules/desktop_capture/desktop_capturer.h" @@ -59,28 +58,6 @@ rtc::scoped_refptr CreateWindowCapturer(nsIWidget* return webrtc::DesktopCaptureImpl::Create(++moduleId, windowId.get(), webrtc::CaptureDeviceType::Window, captureCursor); } -void NotifyScreencastStopped(const nsString& sessionId) { - nsCOMPtr observerService = mozilla::services::GetObserverService(); - if (!observerService) { - fprintf(stderr, "NotifyScreencastStopped error: no observer service\n"); - return; - } - - observerService->NotifyObservers(nullptr, "juggler-screencast-stopped", sessionId.get()); -} - -void NotifyScreencastFrame(const nsCString& frameData) { - nsString wideString; - CopyASCIItoUTF16(frameData, wideString); - nsCOMPtr observerService = mozilla::services::GetObserverService(); - if (!observerService) { - fprintf(stderr, "NotifyScreencastFrame error: no observer service\n"); - return; - } - - observerService->NotifyObservers(nullptr, "juggler-screencast-frame", wideString.get()); -} - nsresult generateUid(nsString& uid) { nsresult rv = NS_OK; nsCOMPtr rg = do_GetService("@mozilla.org/security/random-generator;1", &rv); @@ -102,8 +79,9 @@ nsresult generateUid(nsString& uid) { class nsScreencastService::Session : public rtc::VideoSinkInterface, public webrtc::RawFrameCallback { public: - Session(rtc::scoped_refptr&& capturer, RefPtr&& encoder, gfx::IntMargin margin, uint32_t jpegQuality) - : mCaptureModule(std::move(capturer)) + Session(nsIScreencastServiceClient* client, rtc::scoped_refptr&& capturer, RefPtr&& encoder, gfx::IntMargin margin, uint32_t jpegQuality) + : mClient(client) + , mCaptureModule(std::move(capturer)) , mEncoder(std::move(encoder)) , mJpegQuality(jpegQuality) , mMargin(margin) { @@ -129,23 +107,32 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface&& callback) { + void Stop() { if (mEncoder) mCaptureModule->DeRegisterCaptureDataCallback(this); else - mCaptureModule->RegisterRawFrameCallback(this); + mCaptureModule->DeRegisterRawFrameCallback(this); int error = mCaptureModule->StopCapture(); if (error) { fprintf(stderr, "StopCapture error %d\n", error); } - if (mEncoder) - mEncoder->finish(std::move(callback)); - else - callback(); + if (mEncoder) { + rtc::CritScope lock(&mCaptureCallbackCs); + mEncoder->finish([client = std::move(mClient)] { + NS_DispatchToMainThread(NS_NewRunnableFunction( + "NotifyScreencastStopped", [client = std::move(client)]() -> void { + client->ScreencastStopped(); + })); + }); + } else { + rtc::CritScope lock(&mCaptureCallbackCs); + mClient->ScreencastStopped(); + mClient = nullptr; + } } void ScreencastFrameAck() { - rtc::CritScope lock(&mFramesInFlightCs); + rtc::CritScope lock(&mCaptureCallbackCs); --mFramesInFlight; } @@ -158,13 +145,14 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface= kMaxFramesInFlight) + rtc::CritScope lock(&mCaptureCallbackCs); + if (mFramesInFlight >= kMaxFramesInFlight) { return; + } ++mFramesInFlight; + if (!mClient) + return; } jpeg_compress_struct info; @@ -211,22 +199,27 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface(bufferPtr), bufferSize, base64); - if (NS_WARN_IF(NS_FAILED(rv))) - return; - - NS_DispatchToMainThread(NS_NewRunnableFunction( - "NotifyScreencastFrame", [base64]() -> void { - NotifyScreencastFrame(base64); - })); - free(bufferPtr); + if (NS_WARN_IF(NS_FAILED(rv))) { + return; + } + + uint32_t deviceWidth = info.image_width; + uint32_t deviceHeight = info.image_height; + nsIScreencastServiceClient* client = mClient.get(); + NS_DispatchToMainThread(NS_NewRunnableFunction( + "NotifyScreencastFrame", [client, base64, deviceWidth, deviceHeight]() -> void { + NS_ConvertUTF8toUTF16 utf16(base64); + client->ScreencastFrame(utf16, deviceWidth, deviceHeight); + })); } private: + RefPtr mClient; rtc::scoped_refptr mCaptureModule; RefPtr mEncoder; uint32_t mJpegQuality; - rtc::CriticalSection mFramesInFlightCs; + rtc::CriticalSection mCaptureCallbackCs; uint32_t mFramesInFlight = 0; gfx::IntMargin mMargin; }; @@ -248,7 +241,7 @@ nsScreencastService::nsScreencastService() = default; nsScreencastService::~nsScreencastService() { } -nsresult nsScreencastService::StartVideoRecording(nsIDocShell* aDocShell, bool isVideo, const nsACString& aVideoFileName, uint32_t width, uint32_t height, uint32_t quality, int32_t offsetTop, nsAString& sessionId) { +nsresult nsScreencastService::StartVideoRecording(nsIScreencastServiceClient* aClient, nsIDocShell* aDocShell, bool isVideo, const nsACString& aVideoFileName, uint32_t width, uint32_t height, uint32_t quality, int32_t offsetTop, nsAString& sessionId) { MOZ_RELEASE_ASSERT(NS_IsMainThread(), "Screencast service must be started on the Main thread."); PresShell* presShell = aDocShell->GetPresShell(); @@ -289,7 +282,7 @@ nsresult nsScreencastService::StartVideoRecording(nsIDocShell* aDocShell, bool i NS_ENSURE_SUCCESS(rv, rv); sessionId = uid; - auto session = std::make_unique(std::move(capturer), std::move(encoder), margin, isVideo ? 0 : quality); + auto session = std::make_unique(aClient, std::move(capturer), std::move(encoder), margin, isVideo ? 0 : quality); if (!session->Start()) return NS_ERROR_FAILURE; mIdToSession.emplace(sessionId, std::move(session)); @@ -301,12 +294,7 @@ nsresult nsScreencastService::StopVideoRecording(const nsAString& aSessionId) { auto it = mIdToSession.find(sessionId); if (it == mIdToSession.end()) return NS_ERROR_INVALID_ARG; - it->second->Stop([sessionId] { - NS_DispatchToMainThread(NS_NewRunnableFunction( - "NotifyScreencastStopped", [sessionId]() -> void { - NotifyScreencastStopped(sessionId); - })); - }); + it->second->Stop(); mIdToSession.erase(it); return NS_OK; }