Здесь я адаптировал код и интегрировал его с http-сервером и элементами управления socket.io: https://github.com/phoboslab/jsmpeg
Сервер:
V4L2 -> FFMPEG (MPEG1 TS) -> NODE HTTPСервер -> NODE Websocket broadcast
Клиент:
Websocket -> Javascript (Декодирование MPEG1 TS и рисование в html canvas) -> Html Canvas
Этот стек достигает 640x480 @ 20FPSс задержкой 240 мс.Все еще далеко от моей цели, но достаточно хорош как MVP.Задержка управления в обоих направлениях составляет 7 мс, что превосходно.
Этот стек удерживается этапом транскодирования и декодирования, и RPI действительно сильно нагревается.Транспортировка необработанных данных через websocket выглядит хорошо, я собираюсь профилировать задержку каждого шага в будущем.
Выполнение:
pi@MazeRunner:~ $ node node.js &
pi@MazeRunner:~ $ ffmpeg -f v4l2 -framerate 20 -video_size 640x480 -i /dev/video0 -f mpegts -codec:v mpeg1video -s 640x480 -b:v 600k -bf 0 http://localhost:8080/mystream
На стороне сервера NODE.JS
//operating system library. Used to get local IP address
var os = require("os");
//file system library. Used to load file stored inside back end server (https://nodejs.org/api/fs.html)
var fs = require("fs");
//http system library. Handles basic html requests
var http = require("http").createServer(http_handler);
//url library. Used to process html url requests
var url = require("url");
//Websocket
var io = require("socket.io")(http);
//Websocket used to stream video
var websocket = require("ws");
//-----------------------------------------------------------------------------------
// CONFIGURATION
//-----------------------------------------------------------------------------------
//Port the server will listen to
var server_port = 8080;
var websocket_stream_port = 8082;
//Path of the http and css files for the http server
var file_index_name = "index.html";
var file_css_name = "style.css";
var file_jsplayer_name = "jsmpeg.min.js";
//Http and css files loaded into memory for fast access
var file_index;
var file_css;
var file_jsplayer;
//Name of the local video stream
var stream_name = "mystream";
//-----------------------------------------------------------------------------------
// DETECT SERVER OWN IP
//-----------------------------------------------------------------------------------
//If just one interface, store the server IP Here
var server_ip;
//Get local IP address of the server
///3246903/poluchit-lokalnyi-ip-adres-v-node-js
var ifaces = os.networkInterfaces();
Object.keys(ifaces).forEach
(
function (ifname)
{
var alias = 0;
ifaces[ifname].forEach
(
function (iface)
{
if ('IPv4' !== iface.family || iface.internal !== false)
{
// skip over internal (i.e. 127.0.0.1) and non-ipv4 addresses
return;
}
if (alias >= 1)
{
// this single interface has multiple ipv4 addresses
console.log('INFO: Server interface ' +alias +' - ' + ifname + ':' + alias, iface.address);
}
else
{
server_ip = iface.address;
// this interface has only one ipv4 adress
console.log('INFO: Server interface - ' +ifname, iface.address);
}
++alias;
}
);
}
);
//-----------------------------------------------------------------------------------
// HTTP SERVER
//-----------------------------------------------------------------------------------
// Fetch and serves local files to client
//Create http server and listen to the given port
http.listen
(
server_port,
function( )
{
console.log('INFO: ' +server_ip +' listening to html requests on port ' +server_port);
//Pre-load http, css and js files into memory to improve http request latency
file_index = load_file( file_index_name );
file_css = load_file( file_css_name );
file_jsplayer = load_file( file_jsplayer_name );
}
);
//-----------------------------------------------------------------------------------
// HTTP REQUESTS HANDLER
//-----------------------------------------------------------------------------------
// Answer to client http requests. Serve http, css and js files
function http_handler(req, res)
{
//If client asks for root
if (req.url == '/')
{
//Request main page
res.writeHead( 200, {"Content-Type": detect_content(file_index_name),"Content-Length":file_index.length} );
res.write(file_index);
res.end();
console.log("INFO: Serving file: " +req.url);
}
//If client asks for css file
else if (req.url == ("/" +file_css_name))
{
//Request main page
res.writeHead( 200, {"Content-Type": detect_content(file_css_name),"Content-Length" :file_css.length} );
res.write(file_css);
res.end();
console.log("INFO: Serving file: " +req.url);
}
//If client asks for css file
else if (req.url == ("/" +file_jsplayer_name))
{
//Request main page
res.writeHead( 200, {"Content-Type": detect_content(file_jsplayer_name),"Content-Length" :file_jsplayer.length} );
res.write(file_jsplayer);
res.end();
console.log("INFO: Serving file: " +req.url);
}
//Listening to the port the stream from ffmpeg will flow into
else if (req.url = "/mystream")
{
res.connection.setTimeout(0);
console.log( "Stream Connected: " +req.socket.remoteAddress + ":" +req.socket.remotePort );
req.on
(
"data",
function(data)
{
streaming_websocket.broadcast(data);
/*
if (req.socket.recording)
{
req.socket.recording.write(data);
}
*/
//console.log("broadcast: ", data.length);
}
);
req.on
(
"end",
function()
{
console.log("local stream has ended");
if (req.socket.recording)
{
req.socket.recording.close();
}
}
);
}
//If client asks for an unhandled path
else
{
res.end();
console.log("ERR: Invalid file request" +req.url);
}
}
//-----------------------------------------------------------------------------------
// WEBSOCKET SERVER: CONTROL/FEEDBACK REQUESTS
//-----------------------------------------------------------------------------------
// Handle websocket connection to the client
io.on
(
"connection",
function (socket)
{
console.log("connecting...");
socket.emit("welcome", { payload: "Server says hello" });
//Periodically send the current server time to the client in string form
setInterval
(
function()
{
socket.emit("server_time", { server_time: get_server_time() });
},
//Send every 333ms
333
);
socket.on
(
"myclick",
function (data)
{
timestamp_ms = get_timestamp_ms();
socket.emit("profile_ping", { timestamp: timestamp_ms });
console.log("button event: " +" client says: " +data.payload);
}
);
//"ArrowLeft"
socket.on
(
"keyboard",
function (data)
{
timestamp_ms = get_timestamp_ms();
socket.emit("profile_ping", { timestamp: timestamp_ms });
console.log("keyboard event: " +" client says: " +data.payload);
}
);
//profile packets from the client are answer that allows to compute roundway trip time
socket.on
(
"profile_pong",
function (data)
{
timestamp_ms_pong = get_timestamp_ms();
timestamp_ms_ping = data.timestamp;
console.log("Pong received. Round trip time[ms]: " +(timestamp_ms_pong -timestamp_ms_ping));
}
);
}
);
//-----------------------------------------------------------------------------------
// WEBSOCKET SERVER: STREAMING VIDEO
//-----------------------------------------------------------------------------------
// Websocket Server
var streaming_websocket = new websocket.Server({port: websocket_stream_port, perMessageDeflate: false});
streaming_websocket.connectionCount = 0;
streaming_websocket.on
(
"connection",
function(socket, upgradeReq)
{
streaming_websocket.connectionCount++;
console.log
(
'New websocket Connection: ',
(upgradeReq || socket.upgradeReq).socket.remoteAddress,
(upgradeReq || socket.upgradeReq).headers['user-agent'],
'('+streaming_websocket.connectionCount+" total)"
);
socket.on
(
'close',
function(code, message)
{
streaming_websocket.connectionCount--;
console.log('Disconnected websocket ('+streaming_websocket.connectionCount+' total)');
}
);
}
);
streaming_websocket.broadcast = function(data)
{
streaming_websocket.clients.forEach
(
function each(client)
{
if (client.readyState === websocket.OPEN)
{
client.send(data);
}
}
);
};
//-----------------------------------------------------------------------------------
// FUNCTIONS
//-----------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------
// SERVER DATE&TIME
//-----------------------------------------------------------------------------------
// Get server time in string form
function get_server_time()
{
my_date = new Date();
return my_date.toUTCString();
}
//-----------------------------------------------------------------------------------
// TIMESTAMP
//-----------------------------------------------------------------------------------
// Profile performance in ms
function get_timestamp_ms()
{
my_date = new Date();
return 1000.0* my_date.getSeconds() +my_date.getMilliseconds()
}
//-----------------------------------------------------------------------------------
// FILE LOADER
//-----------------------------------------------------------------------------------
// Load files into memory for improved latency
function load_file( file_name )
{
var file_tmp;
var file_path = __dirname +"/" +file_name;
//HTML index file
try
{
file_tmp = fs.readFileSync( file_path );
}
catch (err)
{
console.log("ERR: " +err.code +" failed to load: " +file_path);
throw err;
}
console.log("INFO: " +file_path +" has been loaded into memory");
return file_tmp;
}
//-----------------------------------------------------------------------------------
// CONTENT TYPE DETECTOR
//-----------------------------------------------------------------------------------
// Return the right content type to give correct information to the client browser
function detect_content( file_name )
{
if (file_name.includes(".html"))
{
return "text/html";
}
else if (file_name.includes(".css"))
{
return "text/css";
}
else if (file_name.includes(".js"))
{
return "application/javascript";
}
else
{
throw "invalid extension";
}
}
Клиентский html
<!DOCTYPE html>
<meta charset="utf-8"/>
<html>
<head>
<title>Maze Runner</title>
<link rel="stylesheet" href="style.css">
<script type="text/javascript" src="/socket.io/socket.io.js"></script>
<script type="text/javascript">
var host_ip = document.location.hostname;
console.log("connecting to host: ", host_ip);
//Get references to the html controls
textbox_input1 = window.document.getElementById("my_text_box")
//Connect to the server via websocket
var mysocket = io("http://" +host_ip +":8080");
//Long lived frame object
var last_frame;
//-----------------------------------------
// CONNESSION ACKNOWLEDGE
//-----------------------------------------
// Link is initiated by the client
// Server sends a welcome message when link is estabilished
// Server could send an auth token to keep track of individual clients and login data
mysocket.on
(
"welcome",
(message) =>
{
console.log("Server websocket connession acknoweldged... " +message.payload);
}
)
//-----------------------------------------
// SERVER->CLIENT CONTROLS
//-----------------------------------------
// Server can send an async message to dinamically update the page without reloading
// This is an example message with the server local date and time in string form
mysocket.on
(
"server_time",
(message) =>
{
fill_label( message.server_time );
console.log("Server sent his local time... " +message.server_time);
}
)
function fill_label( payload )
{
textbox_input1.value=payload;
}
//-----------------------------------------
// CLIENT->SERVER CONTROLS
//-----------------------------------------
// Controls inside the webpage can emit async events to the server
// In this example I have a push button and I catch keyboard strokes
//Handler for a pushbutton
function socket_button_handler()
{
mysocket.emit("myclick", { payload: "button was clicked" });
console.log("Button was clicked...");
}
//Listen for keystrokes
window.document.addEventListener
(
"keypress",
function onEvent(event)
{
//Inform the server that a key has been pressed
mysocket.emit("keyboard", { payload: event.key });
console.log("Key press...");
}
);
//-----------------------------------------
// PING-PONG
//-----------------------------------------
// Server sends ping messages with a timestamp
// Client answers with pongs to allow server to profile latency of the channel
//profile messages means the server wants to compute roundway trip
mysocket.on
(
"profile_ping",
(message) =>
{
//Answer back with the received timestamp so that server can compute roundway trip
mysocket.emit("profile_pong", { timestamp: message.timestamp });
console.log( "server wants a pong. server absolute timestamp[ms]: " +message.timestamp );
}
);
</script>
</head>
<body>
<h1>Html+Css Server +low latency Websocket server</h1>
<!-- button control with socket emitter as handler -->
<p> This button will emit a websocket event. The server will be informed in real time of the event. </p>
<button id="my_button" type="button" onclick="socket_button_handler()">Websocket Button!</button>
<!-- input text control -->
<p> This input can be filled through websockets directly by the server in real time </p>
<input id="my_text_box" type="text" value="" size="40">
<!-- canvas object, it's painted by the javascript video decoder -->
<p> This canvas is painted by the javascript player and shows the live stream.'</p>
<canvas id="video-canvas" width=640 height=480></canvas>
<!-- Javascript video decoder, take in a data stream from a websocket and paint on a canvas -->
<script type="text/javascript" src="jsmpeg.min.js"></script>
<script type="text/javascript">
var mycanvas = document.getElementById("video-canvas");
var url = "ws://" + host_ip +":8082/";
var player = new JSMpeg.Player(url, {canvas: mycanvas});
</script>
</body>
</html>
Javascript Player
Вы можете получить проигрыватель JavaScript, который я использовал здесь: https://github.com/phoboslab/jsmpeg/blob/master/jsmpeg.min.js