Question

59

MediaStream Capture Canvas and Audio Simultaneously

rated 0 times [ 66] [ 7] / answers: 1 / hits: 20043 / 8 Years ago, sat, september 3, 2016, 12:00:00

I'm working on a project in which I'd like to:

Load a video js and display it on the canvas.

Use filters to alter the appearance of the canvas (and therefore the video).

Use the MediaStream captureStream() method and a MediaRecorder object to record the surface of the canvas and the audio of the original video.

Play the stream of both the canvas and the audio in an HTML video element.

I've been able to display the canvas recording in a video element by tweaking this WebRTC demo code: https://webrtc.github.io/samples/src/content/capture/canvas-record/

That said, I can't figure out how to record the video's audio alongside the canvas. Is it possible to create a MediaStream containing MediaStreamTrack instances from two different sources/elements?

According to the MediaStream API's specs there should theoretically be some way to accomplish this:
https://w3c.github.io/mediacapture-main/#introduction

The two main components in the MediaStream API are the MediaStreamTrack and MediaStream interfaces. The MediaStreamTrack object represents media of a single type that originates from one media source in the User Agent, e.g. video produced by a web camera. A MediaStream is used to group several MediaStreamTrack objects into one unit that can be recorded or rendered in a media element.

Answers

Only authorized users can answer the question. Please sign in first, or register a free account.

pranavrorys

Add To Favorites

Follow

Total Points: 466

Total Questions: 87

Total Answers: 115

Location: Barbados

Member since Sun, Nov 27, 2022

2 Years ago

pranavrorys questions

1 'unsafe-eval' is not an allowed source of script

Fri, May 27, 22, 00:00, 2 Years ago

1 How do I run Hardhat with the --constructor-args parameter?

Thu, Oct 28, 21, 00:00, 3 Years ago

1 Task :react-native-webview:compileDebugJavaWithJavac FAILED

Sat, May 30, 20, 00:00, 4 Years ago

1 In React-Table, how to check/uncheck all checkboxes in a row or column

Tue, Mar 3, 20, 00:00, 4 Years ago

1 How to get value of jquery nice select

Fri, Dec 20, 19, 00:00, 5 Years ago

View All

answered 8 Years ago darennevina · Accepted Answer

Is it possible to create a MediaStream containing MediaStreamTrack instances from two different sources/elements?

Yes, you can do it using the MediaStream.addTrack() method, or new MediaStream([track1, track2]).

OP already known how to get all of it, but here is a reminder for future readers :

To get a video stream track from a <canvas>, you can call canvas.captureStream(framerate) method.

To get an audio stream track from a <video> element you can use the Web Audio API and it's createMediaStreamDestination method.
This will return a MediaStreamAudioDestinationNode node (dest) containing our audio stream. You'll then have to connect a MediaElementAudioSourceNode created from your <video> element, to this dest.
If you need to add more audio tracks to this stream, you should connect all these sources to dest.

Now that we've got two streams, one for the <canvas> video and one for the audio, we can either add the audio track to the canvas stream before we initialize the recorder:

canvasStream.addTrack(audioStream.getAudioTracks()[0]);

const recorder = new MediaRecorder(canvasStream)

or we can create a third MediaStream object from these two tracks:

const [videoTrack] = canvasStream.getVideoTracks();

const [audioTrack] = audioStream.getAudioTracks();

const recordedStream = new MediaStream(videoTrack, audioTrack)

const recorder = new MediaRecorder(recordedStream);

Here is a complete example:

var

  btn = document.querySelector(button),

  canvas,

  cStream,

  aStream,

  vid,

  recorder,

  analyser,

  dataArray,

  bufferLength,

  chunks = [];



function clickHandler() {



  btn.textContent = 'stop recording';

  

  if (!aStream) {

    initAudioStream();

  }

  

  cStream = canvas.captureStream(30);

  cStream.addTrack(aStream.getAudioTracks()[0]);



  recorder = new MediaRecorder(cStream);

  recorder.start();



  recorder.ondataavailable = saveChunks;



  recorder.onstop = exportStream;



  btn.onclick = stopRecording;



};



function exportStream(e) {



  if (chunks.length) {



    var blob = new Blob(chunks, { type: chunks[0].type });

    var vidURL = URL.createObjectURL(blob);

    var vid = document.createElement('video');

    vid.controls = true;

    vid.src = vidURL;

    vid.onend = function() {

      URL.revokeObjectURL(vidURL);

    }

    document.body.insertBefore(vid, canvas);



  } else {



    document.body.insertBefore(document.createTextNode('no data saved'), canvas);



  }

}



function saveChunks(e) {



  e.data.size && chunks.push(e.data);



}



function stopRecording() {



  vid.pause();

  btn.remove();

  recorder.stop();



}



function initAudioStream() {



  var audioCtx = new AudioContext();

  // create a stream from our AudioContext

  var dest = audioCtx.createMediaStreamDestination();

  aStream = dest.stream;

  // connect our video element's output to the stream

  var sourceNode = audioCtx.createMediaElementSource(vid);

  sourceNode.connect(dest)

  // start the video

  vid.play();



  // just for the fancy canvas drawings

  analyser = audioCtx.createAnalyser();

  sourceNode.connect(analyser);



  analyser.fftSize = 2048;

  bufferLength = analyser.frequencyBinCount;

  dataArray = new Uint8Array(bufferLength);

  analyser.getByteTimeDomainData(dataArray);



  // output to our headphones

  sourceNode.connect(audioCtx.destination)



  startCanvasAnim();

  

}

function enableButton() {



  vid.oncanplay = null;

  btn.onclick = clickHandler;

  btn.disabled = false;



};



var loadVideo = function() {



  vid = document.createElement('video');

  vid.crossOrigin = 'anonymous';

  vid.oncanplay = enableButton;

  vid.src = 'https://dl.dropboxusercontent.com/s/bch2j17v6ny4ako/movie720p.mp4';



}



function startCanvasAnim() {

  // from MDN https://developer.mozilla.org/en/docs/Web/API/AnalyserNode#Examples

  canvas = Object.assign(document.createElement(canvas), { width: 500, height: 200});

  document.body.prepend(canvas);

  var canvasCtx = canvas.getContext('2d');



  canvasCtx.fillStyle = 'rgb(200, 200, 200)';

  canvasCtx.lineWidth = 2;

  canvasCtx.strokeStyle = 'rgb(0, 0, 0)';



  var draw = function() {



    var drawVisual = requestAnimationFrame(draw);



    analyser.getByteTimeDomainData(dataArray);



    canvasCtx.fillRect(0, 0, canvas.width, canvas.height);

    canvasCtx.beginPath();



    var sliceWidth = canvas.width * 1.0 / bufferLength;

    var x = 0;



    for (var i = 0; i < bufferLength; i++) {



      var v = dataArray[i] / 128.0;

      var y = v * canvas.height / 2;



      if (i === 0) {

        canvasCtx.moveTo(x, y);

      } else {

        canvasCtx.lineTo(x, y);

      }



      x += sliceWidth;

    }



    canvasCtx.lineTo(canvas.width, canvas.height / 2);

    canvasCtx.stroke();



  };



  draw();



}



loadVideo();

button { vertical-align: top }

<button disabled>record</button>