Я работаю над проектом, который конвертирует поток rtsp (ffmpeg) и воспроизводит его на веб-странице (signalr + mse).
Пока что он работает почти так же, как я ожидал в последней версии Edge и Firefox, но не в Chrome.
вот код
public class WebmMediaStreamContext
{
private Process _ffProcess;
private readonly string _cmd;
private byte[] _initSegment;
private Task _readMediaStreamTask;
private CancellationTokenSource _cancellationTokenSource;
private const string _CmdTemplate = "-i {0} -c:v libvpx -tile-columns 4 -frame-parallel 1 -keyint_min 90 -g 90 -f webm -dash 1 pipe:";
public static readonly byte[] ClusterStart = { 0x1F, 0x43, 0xB6, 0x75, 0x01, 0x00, 0x00, 0x00 };
public event EventHandler<ClusterReadyEventArgs> ClusterReadyEvent;
public WebmMediaStreamContext(string rtspFeed)
{
_cmd = string.Format(_CmdTemplate, rtspFeed);
}
public async Task StartConverting()
{
if (_ffProcess != null)
throw new InvalidOperationException();
_ffProcess = new Process();
_ffProcess.StartInfo = new ProcessStartInfo
{
FileName = "ffmpeg/ffmpeg.exe",
Arguments = _cmd,
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardOutput = true
};
_ffProcess.Start();
_initSegment = await ParseInitSegmentAndStartReadMediaStream();
}
public byte[] GetInitSegment()
{
return _initSegment;
}
// Find the first cluster， and everything before it is the InitSegment
private async Task<byte[]> ParseInitSegmentAndStartReadMediaStream()
{
Memory<byte> buffer = new byte[10 * 1024];
int length = 0;
while (length != buffer.Length)
{
length += await _ffProcess.StandardOutput.BaseStream.ReadAsync(buffer.Slice(length));
int cluster = buffer.Span.IndexOf(ClusterStart);
if (cluster >= 0)
{
_cancellationTokenSource = new CancellationTokenSource();
_readMediaStreamTask = new Task(() => ReadMediaStreamProc(buffer.Slice(cluster, length - cluster).ToArray(), _cancellationTokenSource.Token), _cancellationTokenSource.Token, TaskCreationOptions.LongRunning);
_readMediaStreamTask.Start();
return buffer.Slice(0, cluster).ToArray();
}
}
throw new InvalidOperationException();
}
private void ReadMoreBytes(Span<byte> buffer)
{
int size = buffer.Length;
while (size > 0)
{
int len = _ffProcess.StandardOutput.BaseStream.Read(buffer.Slice(buffer.Length - size));
size -= len;
}
}
// Parse every single cluster and fire ClusterReadyEvent
private void ReadMediaStreamProc(byte[] bytesRead, CancellationToken cancel)
{
Span<byte> buffer = new byte[5 * 1024 * 1024];
bytesRead.CopyTo(buffer);
int bufferEmptyIndex = bytesRead.Length;
do
{
if (bufferEmptyIndex < ClusterStart.Length + 4)
{
ReadMoreBytes(buffer.Slice(bufferEmptyIndex, 1024));
bufferEmptyIndex += 1024;
}
int clusterDataSize = BitConverter.ToInt32(
buffer.Slice(ClusterStart.Length, 4)
.ToArray()
.Reverse()
.ToArray()
);
int clusterSize = ClusterStart.Length + 4 + clusterDataSize;
if (clusterSize > buffer.Length)
{
byte[] newBuffer = new byte[clusterSize];
buffer.Slice(0, bufferEmptyIndex).CopyTo(newBuffer);
buffer = newBuffer;
}
if (bufferEmptyIndex < clusterSize)
{
ReadMoreBytes(buffer.Slice(bufferEmptyIndex, clusterSize - bufferEmptyIndex));
bufferEmptyIndex = clusterSize;
}
ClusterReadyEvent?.Invoke(this, new ClusterReadyEventArgs(buffer.Slice(0, bufferEmptyIndex).ToArray()));
bufferEmptyIndex = 0;
} while (!cancel.IsCancellationRequested);
}
}
Я использую ffmpeg для преобразования потока rtsp в поток байтов WEBM vp8 и разбора его на «Init Segment» (ebml head 、 info 、 track ...) и «Media Segment» (cluster) ", а затем отправляю его в браузер через signalR
$(function () {
var mediaSource = new MediaSource();
var mimeCodec = 'video/webm; codecs="vp8"';
var video = document.getElementById('video');
mediaSource.addEventListener('sourceopen', callback, false);
function callback(e) {
var sourceBuffer = mediaSource.addSourceBuffer(mimeCodec);
var queue = [];
sourceBuffer.addEventListener('updateend', function () {
if (queue.length === 0) {
return;
}
var base64 = queue[0];
if (base64.length === 0) {
mediaSource.endOfStream();
queue.shift();
return;
} else {
var buffer = new Uint8Array(atob(base64).split("").map(function (c) {
return c.charCodeAt(0);
}));
sourceBuffer.appendBuffer(buffer);
queue.shift();
}
}, false);
var connection = new signalR.HubConnectionBuilder()
.withUrl("/signalr-video")
.configureLogging(signalR.LogLevel.Information)
.build();
connection.start().then(function () {
connection.stream("InitVideoReceive")
.subscribe({
next: function(item) {
if (queue.length === 0 && !!!sourceBuffer.updating) {
var buffer = new Uint8Array(atob(item).split("").map(function (c) {
return c.charCodeAt(0);
}));
sourceBuffer.appendBuffer(buffer);
console.log(blockindex++ + " : " + buffer.byteLength);
} else {
queue.push(item);
}
},
complete: function () {
queue.push('');
},
error: function (err) {
console.error(err);
}
});
});
}
video.src = window.URL.createObjectURL(mediaSource);
})
chrome просто воспроизводит видео в течение 3 ~ 5 секунд, а затем останавливается для буферизации, даже несмотря на то, что в SourceBuffer передается большое количество кластеров.
вот информация в chrome: // media-internals /
Свойства игрока:
render_id: 217
player_id: 1
origin_url: http://localhost:52531/
frame_url: http://localhost:52531/
frame_title: Home Page
url: blob:http://localhost:52531/dcb25d89-9830-40a5-ba88-33c13b5c03eb
info: Selected FFmpegVideoDecoder for video decoding, config: codec: vp8 format: 1 profile: vp8 coded size: [1280,720] visible rect: [0,0,1280,720] natural size: [1280,720] has extra data? false encryption scheme: Unencrypted rotation: 0°
pipeline_state: kSuspended
found_video_stream: true
video_codec_name: vp8
video_dds: false
video_decoder: FFmpegVideoDecoder
duration: unknown
height: 720
width: 1280
video_buffering_state: BUFFERING_HAVE_NOTHING
for_suspended_start: false
pipeline_buffering_state: BUFFERING_HAVE_NOTHING
event: PAUSE
Вход
Timestamp Property Value
00:00:00 00 origin_url http://localhost:52531/
00:00:00 00 frame_url http://localhost:52531/
00:00:00 00 frame_title Home Page
00:00:00 00 url blob:http://localhost:52531/dcb25d89-9830-40a5-ba88-33c13b5c03eb
00:00:00 00 info ChunkDemuxer: buffering by DTS
00:00:00 35 pipeline_state kStarting
00:00:15 213 found_video_stream true
00:00:15 213 video_codec_name vp8
00:00:15 216 video_dds false
00:00:15 216 video_decoder FFmpegVideoDecoder
00:00:15 216 info Selected FFmpegVideoDecoder for video decoding, config: codec: vp8 format: 1 profile: vp8 coded size: [1280,720] visible rect: [0,0,1280,720] natural size: [1280,720] has extra data? false encryption scheme: Unencrypted rotation: 0°
00:00:15 216 pipeline_state kPlaying
00:00:15 213 duration unknown
00:00:16 661 height 720
00:00:16 661 width 1280
00:00:16 665 video_buffering_state BUFFERING_HAVE_ENOUGH
00:00:16 665 for_suspended_start false
00:00:16 665 pipeline_buffering_state BUFFERING_HAVE_ENOUGH
00:00:16 667 pipeline_state kSuspending
00:00:16 670 pipeline_state kSuspended
00:00:52 759 info Effective playback rate changed from 0 to 1
00:00:52 759 event PLAY
00:00:52 759 pipeline_state kResuming
00:00:52 760 video_dds false
00:00:52 760 video_decoder FFmpegVideoDecoder
00:00:52 760 info Selected FFmpegVideoDecoder for video decoding, config: codec: vp8 format: 1 profile: vp8 coded size: [1280,720] visible rect: [0,0,1280,720] natural size: [1280,720] has extra data? false encryption scheme: Unencrypted rotation: 0°
00:00:52 760 pipeline_state kPlaying
00:00:52 793 height 720
00:00:52 793 width 1280
00:00:52 798 video_buffering_state BUFFERING_HAVE_ENOUGH
00:00:52 798 for_suspended_start false
00:00:52 798 pipeline_buffering_state BUFFERING_HAVE_ENOUGH
00:00:56 278 video_buffering_state BUFFERING_HAVE_NOTHING
00:00:56 295 for_suspended_start false
00:00:56 295 pipeline_buffering_state BUFFERING_HAVE_NOTHING
00:01:20 717 event PAUSE
00:01:33 538 event PLAY
00:01:35 94 event PAUSE
00:01:55 561 pipeline_state kSuspending
00:01:55 563 pipeline_state kSuspended
Может кто-нибудь сказать мне, что не так с моим кодом, или доза хрома требует магической конфигурации для работы?
Спасибо ~
Прошу прощения за мой английский:)