From 79009e87cd00c47141d248a79947aec051f09137 Mon Sep 17 00:00:00 2001 From: Johan von Forstner <5310424+johan12345@users.noreply.github.com> Date: Thu, 23 Mar 2023 16:50:19 +0100 Subject: [PATCH] Add getDisplayMedia and PipeWire support (#1477) I'm picking up @RickStanley's abandoned PR #1321 again to add screensharing support (fixes #927), with the following additional changes: - In newer Electron versions, `desktopCapturer.getSources` must be called from the main process, so I solved this with an IPC call. - Importing from `./helpers/helpers` in 'preload.ts' does not work, as was mentioned by @DimICE in https://github.com/nativefier/nativefier/pull/1321#issuecomment-1001518035. I'm not very familiar with TypeScript or Electron, so not sure why that is and how it could be solved - for now I just copied the referenced `isWayland` function to `preload.ts`. - Add a screensharing test to the manual test script, as requested by @ronjouch in https://github.com/nativefier/nativefier/pull/1321#issuecomment-1006725818 As far as I understood from the discussion in #1321, the last point was basically the only thing that was missing to get this merged, correct? --------- Co-authored-by: Rick Stanley Co-authored-by: Rick Stanley Co-authored-by: Ronan Jouchet --- .github/manual-test | 20 +++ app/src/components/mainWindow.ts | 19 ++- app/src/helpers/helpers.ts | 10 ++ app/src/main.ts | 11 +- app/src/preload.ts | 241 ++++++++++++++++++++++++++++++- 5 files changed, 298 insertions(+), 3 deletions(-) diff --git a/.github/manual-test b/.github/manual-test index 4ad0390..1ec44f4 100755 --- a/.github/manual-test +++ b/.github/manual-test @@ -101,3 +101,23 @@ printf '\n***** SMOKE TEST 3: Test checklist ***** launch_app "$tmp_dir" "$name" request_feedback "$tmp_dir" + +# ------------------------------------------------------------------------------ + +printf '\n***** SMOKE TEST 4: Setting up test and building app... *****\n' +tmp_dir=$(mktemp -d -t nativefier-manual-test-start-in-tray-XXXXX) +name='nativefier-smoke-test-4' +node ./lib/cli.js 'https://meet.jit.si/nativefier-test' \ + --name "$name" \ + "$tmp_dir" + +printf '\n***** SMOKE TEST 4: Test checklist ***** +- Join the Jitsi meeting and try to share your screen + (third button from the left in the bottom bar) +- An overlay should appear where you can select a screen/window to share +- After selecting a screen, a thumbnail of the shared screen should appear on + the top right +- Console: no Electron runtime deprecation warnings/error logged' + +launch_app "$tmp_dir" "$name" +request_feedback "$tmp_dir" diff --git a/app/src/components/mainWindow.ts b/app/src/components/mainWindow.ts index 41731c1..da0ae63 100644 --- a/app/src/components/mainWindow.ts +++ b/app/src/components/mainWindow.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; -import { ipcMain, BrowserWindow, Event } from 'electron'; +import { ipcMain, desktopCapturer, BrowserWindow, Event } from 'electron'; import windowStateKeeper from 'electron-window-state'; import { initContextMenu } from './contextMenu'; @@ -155,6 +155,7 @@ export async function createMainWindow( }); setupSessionInteraction(options, mainWindow); + setupSessionPermissionHandler(mainWindow); if (options.clearCache) { await clearCache(mainWindow); @@ -230,6 +231,22 @@ function setupCounter( }); } +function setupSessionPermissionHandler(window: BrowserWindow): void { + window.webContents.session.setPermissionCheckHandler(() => { + return true; + }); + window.webContents.session.setPermissionRequestHandler( + (_webContents, _permission, callback) => { + callback(true); + }, + ); + ipcMain.handle('desktop-capturer-get-sources', () => { + return desktopCapturer.getSources({ + types: ['screen', 'window'], + }); + }); +} + function setupNotificationBadge( options: OutputOptions, window: BrowserWindow, diff --git a/app/src/helpers/helpers.ts b/app/src/helpers/helpers.ts index 17b5608..335768c 100644 --- a/app/src/helpers/helpers.ts +++ b/app/src/helpers/helpers.ts @@ -284,6 +284,16 @@ export function openExternal( return shell.openExternal(url, options); } +// Copy-pastaed as unable to get imports to work in preload. +// If modifying, update also app/src/preload.ts +export function isWayland(): boolean { + return ( + isLinux() && + (Boolean(process.env.WAYLAND_DISPLAY) || + process.env.XDG_SESSION_TYPE === 'wayland') + ); +} + export function removeUserAgentSpecifics( userAgentFallback: string, appName: string, diff --git a/app/src/main.ts b/app/src/main.ts index 865a9b5..99c874e 100644 --- a/app/src/main.ts +++ b/app/src/main.ts @@ -20,7 +20,12 @@ import { APP_ARGS_FILE_PATH, } from './components/mainWindow'; import { createTrayIcon } from './components/trayIcon'; -import { isOSX, isWindows, removeUserAgentSpecifics } from './helpers/helpers'; +import { + isOSX, + isWayland, + isWindows, + removeUserAgentSpecifics, +} from './helpers/helpers'; import { inferFlashPath } from './helpers/inferFlash'; import * as log from './helpers/loggingHelper'; import { @@ -176,6 +181,10 @@ if (appArgs.basicAuthPassword) { ); } +if (isWayland()) { + app.commandLine.appendSwitch('enable-features', 'WebRTCPipeWireCapturer'); +} + if (appArgs.lang) { const langParts = appArgs.lang.split(','); // Convert locales to languages, because for some reason locales don't work. Stupid Chromium diff --git a/app/src/preload.ts b/app/src/preload.ts index e884455..ada7334 100644 --- a/app/src/preload.ts +++ b/app/src/preload.ts @@ -8,6 +8,7 @@ document.addEventListener('DOMContentLoaded', () => { }); import * as fs from 'fs'; +import * as os from 'os'; import * as path from 'path'; import { ipcRenderer } from 'electron'; @@ -61,6 +62,229 @@ function setNotificationCallback( window.Notification = newNotify; } +async function getDisplayMedia( + sourceId: number | string, +): Promise { + type OriginalVideoPropertyType = boolean | MediaTrackConstraints | undefined; + // Electron supports an outdated specification for mediaDevices, + // see https://www.electronjs.org/docs/latest/api/desktop-capturer/ + const stream = await window.navigator.mediaDevices.getUserMedia({ + audio: false, + video: { + mandatory: { + chromeMediaSource: 'desktop', + chromeMediaSourceId: sourceId, + }, + } as unknown as OriginalVideoPropertyType, + }); + + return stream; +} + +function setupScreenSharePickerStyles(id: string): void { + const screenShareStyles = document.createElement('style'); + screenShareStyles.id = id; + screenShareStyles.innerHTML = ` + .desktop-capturer-selection { + --overlay-color: hsla(0, 0%, 11.8%, 0.75); + --highlight-color: highlight; + --text-content-color: #fff; + --selection-button-color: hsl(180, 1.3%, 14.7%); + } + .desktop-capturer-selection { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100vh; + background: var(--overlay-color); + color: var(--text-content-color); + z-index: 10000000; + display: flex; + align-items: center; + justify-content: center; + } + .desktop-capturer-selection__close { + -moz-appearance: none; + -webkit-appearance: none; + appearance: none; + padding: 1rem; + color: inherit; + position: absolute; + left: 1rem; + top: 1rem; + cursor: pointer; + } + .desktop-capturer-selection__scroller { + width: 100%; + max-height: 100vh; + overflow-y: auto; + } + .desktop-capturer-selection__list { + max-width: calc(100% - 100px); + margin: 50px; + padding: 0; + display: flex; + flex-wrap: wrap; + list-style: none; + overflow: hidden; + justify-content: center; + } + .desktop-capturer-selection__item { + display: flex; + margin: 4px; + } + .desktop-capturer-selection__btn { + display: flex; + flex-direction: column; + align-items: stretch; + width: 145px; + margin: 0; + border: 0; + border-radius: 3px; + padding: 4px; + background: var(--selection-button-color); + text-align: left; + transition: background-color .15s, box-shadow .15s; + } + .desktop-capturer-selection__btn:hover, + .desktop-capturer-selection__btn:focus { + background: var(--highlight-color); + } + .desktop-capturer-selection__thumbnail { + width: 100%; + height: 81px; + object-fit: cover; + } + .desktop-capturer-selection__name { + margin: 6px 0 6px; + white-space: nowrap; + text-overflow: ellipsis; + overflow: hidden; + } + @media (prefers-color-scheme: light) { + .desktop-capturer-selection { + --overlay-color: hsla(0, 0%, 90.2%, 0.75); + --text-content-color: hsl(0, 0%, 12.9%); + --selection-button-color: hsl(180, 1.3%, 85.3%); + } + }`; + document.head.appendChild(screenShareStyles); +} + +function setupScreenSharePickerElement( + id: string, + sources: Electron.DesktopCapturerSource[], +): void { + const selectionElem = document.createElement('div'); + selectionElem.classList.add('desktop-capturer-selection'); + selectionElem.id = id; + selectionElem.innerHTML = ` + +
+
    + ${sources + .map( + ({ id, name, thumbnail }) => ` +
  • + +
    • + `, + ) + .join('')} +
+
+ `; + document.body.appendChild(selectionElem); +} + +function setupScreenSharePicker( + resolve: (value: MediaStream | PromiseLike) => void, + reject: (reason?: unknown) => void, + sources: Electron.DesktopCapturerSource[], +): void { + const baseElementsId = 'native-screen-share-picker'; + const pickerStylesElementId = baseElementsId + '-styles'; + + setupScreenSharePickerElement(baseElementsId, sources); + setupScreenSharePickerStyles(pickerStylesElementId); + + const clearElements = (): void => { + document.getElementById(pickerStylesElementId)?.remove(); + document.getElementById(baseElementsId)?.remove(); + }; + + document + .getElementById(`${baseElementsId}-close`) + ?.addEventListener('click', () => { + clearElements(); + reject('Screen share was cancelled by the user.'); + }); + + document + .querySelectorAll('.desktop-capturer-selection__btn') + .forEach((button) => { + button.addEventListener('click', () => { + const id = button.getAttribute('data-id'); + if (!id) { + log.error("Couldn't find `data-id` of element"); + clearElements(); + return; + } + const source = sources.find((source) => source.id === id); + if (!source) { + log.error(`Source with id "${id}" does not exist`); + clearElements(); + return; + } + + getDisplayMedia(source.id) + .then((stream) => { + resolve(stream); + }) + .catch((err) => { + log.error('Error selecting desktop capture source:', err); + reject(err); + }) + .finally(() => { + clearElements(); + }); + }); + }); +} + +function setDisplayMediaPromise(): void { + // Since no implementation for `getDisplayMedia` exists in Electron we write our own. + window.navigator.mediaDevices.getDisplayMedia = (): Promise => { + return new Promise((resolve, reject) => { + const sources = ipcRenderer.invoke( + 'desktop-capturer-get-sources', + ) as Promise; + sources + .then(async (sources) => { + if (isWayland()) { + // No documentation is provided wether the first element is always PipeWire-picked or not + // i.e. maybe it's not deterministic, we are only taking a guess here. + const stream = await getDisplayMedia(sources[0].id); + resolve(stream); + } else { + setupScreenSharePicker(resolve, reject, sources); + } + }) + .catch((err) => { + reject(err); + }); + }); + }; +} + function injectScripts(): void { const needToInject = fs.existsSync(INJECT_DIR); if (!needToInject) { @@ -95,13 +319,28 @@ function notifyNotificationClick(): void { // @ts-expect-error TypeScript thinks these are incompatible but they aren't setNotificationCallback(notifyNotificationCreate, notifyNotificationClick); +setDisplayMediaPromise(); ipcRenderer.on('params', (event, message: string) => { log.debug('ipcRenderer.params', { event, message }); - const appArgs = JSON.parse(message) as OutputOptions; + const appArgs: unknown = JSON.parse(message) as OutputOptions; log.info('nativefier.json', appArgs); }); ipcRenderer.on('debug', (event, message: string) => { log.debug('ipcRenderer.debug', { event, message }); }); + +// Copy-pastaed as unable to get imports to work in preload. +// If modifying, update also app/src/helpers/helpers.ts +function isWayland(): boolean { + return ( + isLinux() && + (Boolean(process.env.WAYLAND_DISPLAY) || + process.env.XDG_SESSION_TYPE === 'wayland') + ); +} + +function isLinux(): boolean { + return os.platform() === 'linux'; +}