Add getDisplayMedia and PipeWire support (#1477)

I'm picking up @RickStanley's abandoned PR #1321 again to add
screensharing support (fixes #927), with the following additional
changes:

- In newer Electron versions, `desktopCapturer.getSources` must be
called from the main process, so I solved this with an IPC call.
- Importing from `./helpers/helpers` in 'preload.ts' does not work, as
was mentioned by @DimICE in
https://github.com/nativefier/nativefier/pull/1321#issuecomment-1001518035.
I'm not very familiar with TypeScript or Electron, so not sure why that
is and how it could be solved - for now I just copied the referenced
`isWayland` function to `preload.ts`.
- Add a screensharing test to the manual test script, as requested by
@ronjouch in
https://github.com/nativefier/nativefier/pull/1321#issuecomment-1006725818

As far as I understood from the discussion in #1321, the last point was
basically the only thing that was missing to get this merged, correct?

---------

Co-authored-by: Rick Stanley <rick-stanley@outlook.com>
Co-authored-by: Rick Stanley <rick.stanley@lambda3.com.br>
Co-authored-by: Ronan Jouchet <ronan@jouchet.fr>

.github/manual-test

@@ -101,3 +101,23 @@ printf '\n***** SMOKE TEST 3: Test checklist *****
 
 launch_app "$tmp_dir" "$name"
 request_feedback "$tmp_dir"
+
+# ------------------------------------------------------------------------------
+
+printf '\n***** SMOKE TEST 4: Setting up test and building app... *****\n'
+tmp_dir=$(mktemp -d -t nativefier-manual-test-start-in-tray-XXXXX)
+name='nativefier-smoke-test-4'
+node ./lib/cli.js 'https://meet.jit.si/nativefier-test' \
+  --name "$name" \
+  "$tmp_dir"
+
+printf '\n***** SMOKE TEST 4: Test checklist *****
+- Join the Jitsi meeting and try to share your screen
+  (third button from the left in the bottom bar)
+- An overlay should appear where you can select a screen/window to share
+- After selecting a screen, a thumbnail of the shared screen should appear on
+  the top right
+- Console: no Electron runtime deprecation warnings/error logged'
+
+launch_app "$tmp_dir" "$name"
+request_feedback "$tmp_dir"

app/src/components/mainWindow.ts

@@ -1,7 +1,7 @@
 import * as fs from 'fs';
 import * as path from 'path';
 
-import { ipcMain, BrowserWindow, Event } from 'electron';
+import { ipcMain, desktopCapturer, BrowserWindow, Event } from 'electron';
 import windowStateKeeper from 'electron-window-state';
 
 import { initContextMenu } from './contextMenu';
@@ -155,6 +155,7 @@ export async function createMainWindow(
   });
 
   setupSessionInteraction(options, mainWindow);
+  setupSessionPermissionHandler(mainWindow);
 
   if (options.clearCache) {
     await clearCache(mainWindow);
@@ -230,6 +231,22 @@ function setupCounter(
   });
 }
 
+function setupSessionPermissionHandler(window: BrowserWindow): void {
+  window.webContents.session.setPermissionCheckHandler(() => {
+    return true;
+  });
+  window.webContents.session.setPermissionRequestHandler(
+    (_webContents, _permission, callback) => {
+      callback(true);
+    },
+  );
+  ipcMain.handle('desktop-capturer-get-sources', () => {
+    return desktopCapturer.getSources({
+      types: ['screen', 'window'],
+    });
+  });
+}
+
 function setupNotificationBadge(
   options: OutputOptions,
   window: BrowserWindow,

app/src/helpers/helpers.ts

@@ -284,6 +284,16 @@ export function openExternal(
   return shell.openExternal(url, options);
 }
 
+// Copy-pastaed as unable to get imports to work in preload.
+// If modifying, update also app/src/preload.ts
+export function isWayland(): boolean {
+  return (
+    isLinux() &&
+    (Boolean(process.env.WAYLAND_DISPLAY) ||
+      process.env.XDG_SESSION_TYPE === 'wayland')
+  );
+}
+
 export function removeUserAgentSpecifics(
   userAgentFallback: string,
   appName: string,

app/src/main.ts

@@ -20,7 +20,12 @@ import {
   APP_ARGS_FILE_PATH,
 } from './components/mainWindow';
 import { createTrayIcon } from './components/trayIcon';
-import { isOSX, isWindows, removeUserAgentSpecifics } from './helpers/helpers';
+import {
+  isOSX,
+  isWayland,
+  isWindows,
+  removeUserAgentSpecifics,
+} from './helpers/helpers';
 import { inferFlashPath } from './helpers/inferFlash';
 import * as log from './helpers/loggingHelper';
 import {
@@ -176,6 +181,10 @@ if (appArgs.basicAuthPassword) {
   );
 }
 
+if (isWayland()) {
+  app.commandLine.appendSwitch('enable-features', 'WebRTCPipeWireCapturer');
+}
+
 if (appArgs.lang) {
   const langParts = appArgs.lang.split(',');
   // Convert locales to languages, because for some reason locales don't work. Stupid Chromium

app/src/preload.ts

@@ -8,6 +8,7 @@ document.addEventListener('DOMContentLoaded', () => {
 });
 
 import * as fs from 'fs';
+import * as os from 'os';
 import * as path from 'path';
 
 import { ipcRenderer } from 'electron';
@@ -61,6 +62,229 @@ function setNotificationCallback(
   window.Notification = newNotify;
 }
 
+async function getDisplayMedia(
+  sourceId: number | string,
+): Promise<MediaStream> {
+  type OriginalVideoPropertyType = boolean | MediaTrackConstraints | undefined;
+  // Electron supports an outdated specification for mediaDevices,
+  // see https://www.electronjs.org/docs/latest/api/desktop-capturer/
+  const stream = await window.navigator.mediaDevices.getUserMedia({
+    audio: false,
+    video: {
+      mandatory: {
+        chromeMediaSource: 'desktop',
+        chromeMediaSourceId: sourceId,
+      },
+    } as unknown as OriginalVideoPropertyType,
+  });
+
+  return stream;
+}
+
+function setupScreenSharePickerStyles(id: string): void {
+  const screenShareStyles = document.createElement('style');
+  screenShareStyles.id = id;
+  screenShareStyles.innerHTML = `
+  .desktop-capturer-selection {
+    --overlay-color: hsla(0, 0%, 11.8%, 0.75);
+    --highlight-color: highlight;
+    --text-content-color: #fff;
+    --selection-button-color: hsl(180, 1.3%, 14.7%);
+  }
+  .desktop-capturer-selection {
+    position: fixed;
+    top: 0;
+    left: 0;
+    width: 100%;
+    height: 100vh;
+    background: var(--overlay-color);
+    color: var(--text-content-color);
+    z-index: 10000000;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+  }
+  .desktop-capturer-selection__close {
+    -moz-appearance: none;
+    -webkit-appearance: none;
+    appearance: none;
+    padding: 1rem;
+    color: inherit;
+    position: absolute;
+    left: 1rem;
+    top: 1rem;
+    cursor: pointer;
+  }
+  .desktop-capturer-selection__scroller {
+    width: 100%;
+    max-height: 100vh;
+    overflow-y: auto;
+  }
+  .desktop-capturer-selection__list {
+    max-width: calc(100% - 100px);
+    margin: 50px;
+    padding: 0;
+    display: flex;
+    flex-wrap: wrap;
+    list-style: none;
+    overflow: hidden;
+    justify-content: center;
+  }
+  .desktop-capturer-selection__item {
+    display: flex;
+    margin: 4px;
+  }
+  .desktop-capturer-selection__btn {
+    display: flex;
+    flex-direction: column;
+    align-items: stretch;
+    width: 145px;
+    margin: 0;
+    border: 0;
+    border-radius: 3px;
+    padding: 4px;
+    background: var(--selection-button-color);
+    text-align: left;
+    transition: background-color .15s, box-shadow .15s;
+  }
+  .desktop-capturer-selection__btn:hover,
+  .desktop-capturer-selection__btn:focus {
+    background: var(--highlight-color);
+  }
+  .desktop-capturer-selection__thumbnail {
+    width: 100%;
+    height: 81px;
+    object-fit: cover;
+  }
+  .desktop-capturer-selection__name {
+    margin: 6px 0 6px;
+    white-space: nowrap;
+    text-overflow: ellipsis;
+    overflow: hidden;
+  }
+  @media (prefers-color-scheme: light) {
+    .desktop-capturer-selection {
+      --overlay-color: hsla(0, 0%, 90.2%, 0.75);
+      --text-content-color: hsl(0, 0%, 12.9%);
+      --selection-button-color: hsl(180, 1.3%, 85.3%);
+    }
+  }`;
+  document.head.appendChild(screenShareStyles);
+}
+
+function setupScreenSharePickerElement(
+  id: string,
+  sources: Electron.DesktopCapturerSource[],
+): void {
+  const selectionElem = document.createElement('div');
+  selectionElem.classList.add('desktop-capturer-selection');
+  selectionElem.id = id;
+  selectionElem.innerHTML = `
+    <button class="desktop-capturer-selection__close" id="${id}-close" aria-label="Close screen share picker" type="button">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="32" height="32">
+      <path fill="currentColor" d="m12 10.586 4.95-4.95 1.414 1.414-4.95 4.95 4.95 4.95-1.414 1.414-4.95-4.95-4.95 4.95-1.414-1.414 4.95-4.95-4.95-4.95L7.05 5.636z"/>   
+      </svg>
+    </button>
+    <div class="desktop-capturer-selection__scroller">
+      <ul class="desktop-capturer-selection__list">
+        ${sources
+          .map(
+            ({ id, name, thumbnail }) => `
+          <li class="desktop-capturer-selection__item">
+            <button class="desktop-capturer-selection__btn" data-id="${id}" title="${name}">
+              <img class="desktop-capturer-selection__thumbnail" src="${thumbnail.toDataURL()}" />
+              <span class="desktop-capturer-selection__name">${name}</span>
+            </button>
+          </li>
+        `,
+          )
+          .join('')}
+      </ul>
+    </div>
+    `;
+  document.body.appendChild(selectionElem);
+}
+
+function setupScreenSharePicker(
+  resolve: (value: MediaStream | PromiseLike<MediaStream>) => void,
+  reject: (reason?: unknown) => void,
+  sources: Electron.DesktopCapturerSource[],
+): void {
+  const baseElementsId = 'native-screen-share-picker';
+  const pickerStylesElementId = baseElementsId + '-styles';
+
+  setupScreenSharePickerElement(baseElementsId, sources);
+  setupScreenSharePickerStyles(pickerStylesElementId);
+
+  const clearElements = (): void => {
+    document.getElementById(pickerStylesElementId)?.remove();
+    document.getElementById(baseElementsId)?.remove();
+  };
+
+  document
+    .getElementById(`${baseElementsId}-close`)
+    ?.addEventListener('click', () => {
+      clearElements();
+      reject('Screen share was cancelled by the user.');
+    });
+
+  document
+    .querySelectorAll('.desktop-capturer-selection__btn')
+    .forEach((button) => {
+      button.addEventListener('click', () => {
+        const id = button.getAttribute('data-id');
+        if (!id) {
+          log.error("Couldn't find `data-id` of element");
+          clearElements();
+          return;
+        }
+        const source = sources.find((source) => source.id === id);
+        if (!source) {
+          log.error(`Source with id "${id}" does not exist`);
+          clearElements();
+          return;
+        }
+
+        getDisplayMedia(source.id)
+          .then((stream) => {
+            resolve(stream);
+          })
+          .catch((err) => {
+            log.error('Error selecting desktop capture source:', err);
+            reject(err);
+          })
+          .finally(() => {
+            clearElements();
+          });
+      });
+    });
+}
+
+function setDisplayMediaPromise(): void {
+  // Since no implementation for `getDisplayMedia` exists in Electron we write our own.
+  window.navigator.mediaDevices.getDisplayMedia = (): Promise<MediaStream> => {
+    return new Promise((resolve, reject) => {
+      const sources = ipcRenderer.invoke(
+        'desktop-capturer-get-sources',
+      ) as Promise<Electron.DesktopCapturerSource[]>;
+      sources
+        .then(async (sources) => {
+          if (isWayland()) {
+            // No documentation is provided wether the first element is always PipeWire-picked or not
+            // i.e. maybe it's not deterministic, we are only taking a guess here.
+            const stream = await getDisplayMedia(sources[0].id);
+            resolve(stream);
+          } else {
+            setupScreenSharePicker(resolve, reject, sources);
+          }
+        })
+        .catch((err) => {
+          reject(err);
+        });
+    });
+  };
+}
+
 function injectScripts(): void {
   const needToInject = fs.existsSync(INJECT_DIR);
   if (!needToInject) {
@@ -95,13 +319,28 @@ function notifyNotificationClick(): void {
 
 // @ts-expect-error TypeScript thinks these are incompatible but they aren't
 setNotificationCallback(notifyNotificationCreate, notifyNotificationClick);
+setDisplayMediaPromise();
 
 ipcRenderer.on('params', (event, message: string) => {
   log.debug('ipcRenderer.params', { event, message });
-  const appArgs = JSON.parse(message) as OutputOptions;
+  const appArgs: unknown = JSON.parse(message) as OutputOptions;
   log.info('nativefier.json', appArgs);
 });
 
 ipcRenderer.on('debug', (event, message: string) => {
   log.debug('ipcRenderer.debug', { event, message });
 });
+
+// Copy-pastaed as unable to get imports to work in preload.
+// If modifying, update also app/src/helpers/helpers.ts
+function isWayland(): boolean {
+  return (
+    isLinux() &&
+    (Boolean(process.env.WAYLAND_DISPLAY) ||
+      process.env.XDG_SESSION_TYPE === 'wayland')
+  );
+}
+
+function isLinux(): boolean {
+  return os.platform() === 'linux';
+}