How to use parallel offthread video?

[gajraj-gan]

I have written this component which is very slow due to the Video component I am assuming. How do I optimize this by using offthread video? Right now the video on preview looks good, but comes as black on render.

import React, { useEffect, useRef, useState } from 'react';
import {
  AbsoluteFill,
  continueRender,
  delayRender,
  Sequence,
  useCurrentFrame,
  useVideoConfig,
  Video
} from 'remotion';

type CustomOffthreadVideoProps = {
  inputVideoSrc: string;
  segmentationVideoSrc: string;
};

export const CustomOffthreadVideo: React.FC<CustomOffthreadVideoProps> = ({
  inputVideoSrc,
  segmentationVideoSrc
}) => {
  const inputVideoRef = useRef<HTMLVideoElement>(null);
  const segmentationVideoRef = useRef<HTMLVideoElement>(null);
  const canvasRef = useRef<HTMLCanvasElement>(null);
  const frame = useCurrentFrame();
  const { width, height, durationInFrames } = useVideoConfig();
  const [handle] = useState(() => delayRender('Fetching dimensions...'));
  const [video1Loaded] = useState(true);
  const [video2Loaded] = useState(true);

  useEffect(() => {
    if (video1Loaded && video2Loaded) {
      continueRender(handle);
    }
  }, [video1Loaded, video2Loaded]);

  useEffect(() => {
    const inputVideo = inputVideoRef.current;
    const segmentationVideo = segmentationVideoRef.current;
    const canvas = canvasRef.current;

    if (!video1Loaded || !video2Loaded || !inputVideo || !segmentationVideo || !canvas) {
      return;
    }

    const context = canvas.getContext('2d', {
      willReadFrequently: true
    });

    if (!context) {
      return;
    }

    const renderFrame = () => {
      if (inputVideo.readyState >= 2 && segmentationVideo.readyState >= 2) {
        context.clearRect(0, 0, width, height);

        context.drawImage(inputVideo, 0, 0, width, height);
        const imageData = context.getImageData(0, 0, width, height);
        context.clearRect(0, 0, width, height);

        context.drawImage(segmentationVideo, 0, 0, width, height);
        const segmentationData = context.getImageData(0, 0, width, height);

        for (let i = 0; i < imageData.data.length; i += 4) {
          // check if black or white pixel and then assign alpha
          const alpha = segmentationData.data[i] >= 128 ? 0 : 1;
          imageData.data[i + 3] = alpha * 255;
        }

        context.putImageData(imageData, 0, 0);
      }
    };

    renderFrame();
  }, [frame, width, height, video1Loaded, video2Loaded]);

  return (
    <Sequence from={0} durationInFrames={durationInFrames} premountFor={500}>
      <AbsoluteFill>
        <Video
          ref={inputVideoRef}
          src={inputVideoSrc}
          style={{ opacity: 0 }}
          crossOrigin="anonymous"
        />
        <Video
          ref={segmentationVideoRef}
          src={segmentationVideoSrc}
          style={{ opacity: 0 }}
          crossOrigin="anonymous"
        />
      </AbsoluteFill>
      <AbsoluteFill>
        <canvas ref={canvasRef} width={width} height={height} />
      </AbsoluteFill>
    </Sequence>
  );
};

[gitsult4n]

Drop the refs and the delayRender juggling
use onVideoFrame on two <OffthreadVideo> and composite once both frames arrived

import React, {useCallback, useRef} from 'react';
import {AbsoluteFill, OffthreadVideo, useVideoConfig} from 'remotion';
import type {OnVideoFrame} from 'remotion';

export const MaskedVideo: React.FC<{
  inputVideoSrc: string;
  segmentationVideoSrc: string;
}> = ({inputVideoSrc, segmentationVideoSrc}) => {
  const {width, height} = useVideoConfig();
  const canvasRef = useRef<HTMLCanvasElement>(null);
  const input = useRef<CanvasImageSource | null>(null);
  const mask = useRef<CanvasImageSource | null>(null);

  const draw = useCallback(() => {
    if (!input.current || !mask.current || !canvasRef.current) {
      return;
    }

    const ctx = canvasRef.current.getContext('2d', {willReadFrequently: true});
    if (!ctx) {
      return;
    }

    ctx.clearRect(0, 0, width, height);
    ctx.drawImage(input.current, 0, 0, width, height);
    const image = ctx.getImageData(0, 0, width, height);

    ctx.clearRect(0, 0, width, height);
    ctx.drawImage(mask.current, 0, 0, width, height);
    const segmentation = ctx.getImageData(0, 0, width, height);

    for (let i = 0; i < image.data.length; i += 4) {
      image.data[i + 3] = segmentation.data[i] >= 128 ? 0 : 255;
    }

    ctx.putImageData(image, 0, 0);
  }, [width, height]);

  const onInputFrame: OnVideoFrame = useCallback(
    (frame) => {
      input.current = frame;
      draw();
    },
    [draw],
  );

  const onMaskFrame: OnVideoFrame = useCallback(
    (frame) => {
      mask.current = frame;
      draw();
    },
    [draw],
  );

  return (
    <AbsoluteFill>
      <AbsoluteFill style={{opacity: 0}}>
        <OffthreadVideo src={inputVideoSrc} onVideoFrame={onInputFrame} />
        <OffthreadVideo src={segmentationVideoSrc} onVideoFrame={onMaskFrame} />
      </AbsoluteFill>
      <AbsoluteFill>
        <canvas ref={canvasRef} width={width} height={height} />
      </AbsoluteFill>
    </AbsoluteFill>
  );
};

why refs give you black on render
during rendering <OffthreadVideo> is an <img> not a <video> so readyState >= 2 never becomes true packages/core/src/video/OffthreadVideoForRendering.tsx:299
and with <Video> the seek finishes in a promise long after your useEffect already drew packages/core/src/video/VideoForRendering.tsx:210

both callbacks fire before the frame is captured because onImageFrame runs right before continueRender packages/core/src/Img.tsx:276

onVideoFrame hands you an HTMLVideoElement in preview and an HTMLImageElement in render
https://remotion.dev/docs/videos/video-manipulation

no crossOrigin needed either
preview sets it to "anonymous" once onVideoFrame is set packages/core/src/get-cross-origin-value.ts:21
and rendering strips the prop entirely packages/core/src/video/OffthreadVideoForRendering.tsx:52

needs Remotion 4.0.190 or newer

on current versions <Video> from @remotion/media takes the same onVideoFrame and has headless so you can skip the hidden tag entirely
https://remotion.dev/docs/media/video#headless