Node.js зависает на HTTP get при загрузке файлов

Question

Я хочу загрузить серию файлов ts, но каждый раз, когда я запускаю этот код, он застревает в процессе загрузки файла без какого-либо сообщения об ошибке.

Однако, если вы вставите URL-адрес в программу в адресную строку браузера или используете инструмент, подобный wget, вы все равно сможете его загрузить.

const puppeteer = require('puppeteer');
var https = require('https');
var fs = require('fs');

async function doRequest(url) {
  return new Promise(function (resolve) {
  var req =  https.get(url, function (response) {
        resolve(response);
  });
  
  req.end('error', function(e) {   
  	sleep(3000).then(() => {
			 console.log('re try to get ts ...'+e);
  	   doRequest(url); })
  });
    
  req.on('error', function(e) {   
  	sleep(3000).then(() => {
			 console.log('re try to get ts ...'+e);
  	   doRequest(url); })
  });
    
  });
}

(async() => {	
			for(let i = 21; i<100; i++)
			{	
					var ts_url = 'https://hls2.videos.sproutvideo.com/1e7c8a8ff0518509452c7eb2e75a2a1f/e84883ff69cb66752bd6783cdbaa35fb/video/720_000'+i+'.ts?Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9obHMyLnZpZGVvcy5zcHJvdXR2aWRlby5jb20vMWU3YzhhOGZmMDUxODUwOTQ1MmM3ZWIyZTc1YTJhMWYvZTg0ODgzZmY2OWNiNjY3NTJiZDY3ODNjZGJhYTM1ZmIvKi50cz9zZXNzaW9uSUQ9MDZkN2ZhOWMtMDgxOS00Y2YwLTk0M2QtNzA2MGQzOGY2N2RkIiwiQ29uZGl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIjoxNTQ2NzY3NjY4fX19XX0_&Signature=DTCQWVIdInCe2YIf-fxD4RDEHOXGUDK2pUwxV0cKPi0m~WwlLYIEFSmQkAbK-oV-uLU93E1O2TGizvrMDp6voFVnm-jLaOur1JRlJDBCP7T8KEYrkkU3Y3grZAKHmi0gQiVpVIKRgo7gnDKwMZ1NjosQPbaf1XDMpuHxAyKfPGgIRLpSEp4BZ1dqcfzs-YyYQzNaK-a3tYONmpyID3bZnF8sn2pMZonArCz24BQL0wEfXeS3HqxwVv85z641kKxQBGd~8lG88qUTpJCvqWmIZhikzWjGQPY~6ezgJMKhjJQIoPMVGZehT~NcAzPwXo84kd5ksaOdbh4paHsUe1096A__&Key-Pair-Id=APKAIB5DGCGAQJ4GGIUQ&sessionID=06d7fa9c-0819-4cf0-943d-7060d38f67dd';				
					await console.log('downloading '+i);
			  	var file_ts_path = fs.createWriteStream(i+'.ts');
					let responseRes = await doRequest(ts_url);
					await responseRes.pipe(file_ts_path);	
			}				
})();

Answer 1

Вам нужно скачать файл, записать его в поток и затем передать его по конвейеру. https://nodejs.org/api/https.html#https_https_get_url_options_callback https://nodejs.org/api/fs.html#fs_fs_createwritestream_path_options

var https = require('https');
var fs = require('fs');

function doRequest(url, dest) {
  return new Promise(function (resolve) {
    https.get(url, res => {

      let data = '';
      res.on('data', (buffer) => {
        data += buffer; // save downloaded data
      });

      res.on('end', () => {
        var stream = fs.createWriteStream(dest);
        stream.write(data, 'binary');
        stream.on('finish', () => {
          resolve('File Saved !');
        });
        res.pipe(stream);
      });

    }).on('error', function (e) {
      reject('re try to get ts ...' + e);
    });
  });
}

for (let i = 21; i < 100; i++) {
  var ts_url =
    'https://hls2.videos.sproutvideo.com/1e7c8a8ff0518509452c7eb2e75a2a1f/e84883ff69cb66752bd6783cdbaa35fb/video/720_000' +
    i +
    '.ts?Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9obHMyLnZpZGVvcy5zcHJvdXR2aWRlby5jb20vMWU3YzhhOGZmMDUxODUwOTQ1MmM3ZWIyZTc1YTJhMWYvZTg0ODgzZmY2OWNiNjY3NTJiZDY3ODNjZGJhYTM1ZmIvKi50cz9zZXNzaW9uSUQ9MDZkN2ZhOWMtMDgxOS00Y2YwLTk0M2QtNzA2MGQzOGY2N2RkIiwiQ29uZGl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIjoxNTQ2NzY3NjY4fX19XX0_&Signature=DTCQWVIdInCe2YIf-fxD4RDEHOXGUDK2pUwxV0cKPi0m~WwlLYIEFSmQkAbK-oV-uLU93E1O2TGizvrMDp6voFVnm-jLaOur1JRlJDBCP7T8KEYrkkU3Y3grZAKHmi0gQiVpVIKRgo7gnDKwMZ1NjosQPbaf1XDMpuHxAyKfPGgIRLpSEp4BZ1dqcfzs-YyYQzNaK-a3tYONmpyID3bZnF8sn2pMZonArCz24BQL0wEfXeS3HqxwVv85z641kKxQBGd~8lG88qUTpJCvqWmIZhikzWjGQPY~6ezgJMKhjJQIoPMVGZehT~NcAzPwXo84kd5ksaOdbh4paHsUe1096A__&Key-Pair-Id=APKAIB5DGCGAQJ4GGIUQ&sessionID=06d7fa9c-0819-4cf0-943d-7060d38f67dd';

  doRequest(ts_url, i + '.ts')
    .then(console.log)
    .catch(console.log);
}

Answer 2

const puppeteer = require('puppeteer');
var https = require('https');
var fs = require('fs');

async function doRequest(url) {
  return new Promise(function (resolve) {
  var req =  https.get(url, function (response) {
        resolve(response);
  });
  
  req.end('error', function(e) {   
  	sleep(3000).then(() => {
			 console.log('re try to get ts ...'+e);
  	   doRequest(url); })
  });
    
  req.on('error', function(e) {   
  	sleep(3000).then(() => {
			 console.log('re try to get ts ...'+e);
  	   doRequest(url); })
  });
    
  });
}

(async() => {	
			for(let i = 21; i<100; i++)
			{	
					var ts_url = 'https://hls2.videos.sproutvideo.com/1e7c8a8ff0518509452c7eb2e75a2a1f/e84883ff69cb66752bd6783cdbaa35fb/video/720_000'+i+'.ts?Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9obHMyLnZpZGVvcy5zcHJvdXR2aWRlby5jb20vMWU3YzhhOGZmMDUxODUwOTQ1MmM3ZWIyZTc1YTJhMWYvZTg0ODgzZmY2OWNiNjY3NTJiZDY3ODNjZGJhYTM1ZmIvKi50cz9zZXNzaW9uSUQ9MDZkN2ZhOWMtMDgxOS00Y2YwLTk0M2QtNzA2MGQzOGY2N2RkIiwiQ29uZGl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIjoxNTQ2NzY3NjY4fX19XX0_&Signature=DTCQWVIdInCe2YIf-fxD4RDEHOXGUDK2pUwxV0cKPi0m~WwlLYIEFSmQkAbK-oV-uLU93E1O2TGizvrMDp6voFVnm-jLaOur1JRlJDBCP7T8KEYrkkU3Y3grZAKHmi0gQiVpVIKRgo7gnDKwMZ1NjosQPbaf1XDMpuHxAyKfPGgIRLpSEp4BZ1dqcfzs-YyYQzNaK-a3tYONmpyID3bZnF8sn2pMZonArCz24BQL0wEfXeS3HqxwVv85z641kKxQBGd~8lG88qUTpJCvqWmIZhikzWjGQPY~6ezgJMKhjJQIoPMVGZehT~NcAzPwXo84kd5ksaOdbh4paHsUe1096A__&Key-Pair-Id=APKAIB5DGCGAQJ4GGIUQ&sessionID=06d7fa9c-0819-4cf0-943d-7060d38f67dd';				
					await console.log('downloading '+i);
			  	var file_ts_path = fs.createWriteStream(i+'.ts');
					let responseRes = await doRequest(ts_url);
					await responseRes.pipe(file_ts_path);	
			}				
})();

const puppeteer = require('puppeteer');
var https = require('https');
var fs = require('fs');

async function doRequest(url) {
  return new Promise(function (resolve) {
  var req =  https.get(url, function (response) {
        resolve(response);
  });
  
  req.end('error', function(e) {   
  	sleep(3000).then(() => {
			 console.log('re try to get ts ...'+e);
  	   doRequest(url); })
  });
    
  req.on('error', function(e) {   
  	sleep(3000).then(() => {
			 console.log('re try to get ts ...'+e);
  	   doRequest(url); })
  });
    
  });
}

(async() => {	
			for(let i = 21; i<100; i++)
			{	
					var ts_url = 'https://hls2.videos.sproutvideo.com/1e7c8a8ff0518509452c7eb2e75a2a1f/e84883ff69cb66752bd6783cdbaa35fb/video/720_000'+i+'.ts?Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9obHMyLnZpZGVvcy5zcHJvdXR2aWRlby5jb20vMWU3YzhhOGZmMDUxODUwOTQ1MmM3ZWIyZTc1YTJhMWYvZTg0ODgzZmY2OWNiNjY3NTJiZDY3ODNjZGJhYTM1ZmIvKi50cz9zZXNzaW9uSUQ9MmFmN2JkNzEtODc3Mi00OThmLThjNzQtN2Y5NTQ3ZTgxYjA2IiwiQ29uZGl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIjoxNTQ2NzEzNzU4fX19XX0_&Signature=hFOZlf5Iqf7k-LqVms1FT4LKoEM8psMVkCsdjoHGhKztNTzROyGgfHg34RYr3ezffQaQV6drXBuID1NMypQhwSXgJ-ZRAoGC3KnKtrfm9bSpK2Wq97sZf97D5PbBn7wNaJhmfvbztym-cRknztepOqM2v~KvCz6~esS99TOsWbtQFBZWLPacp5MV3v5eZ4wh2WJXX1jDqI1XmpZ0jyU2oJCXOgVbvU1aF86E7duvniDrbhmS1R00~tWTAETBbmBSubDw-7fGq7XzeZcFRfXbdwb0a9KwsAGh54lj1UBUMsDzEtH8vI8r9aC~MnFIRub1KxsFSOzUlRLYRp4GsPJQiQ__&Key-Pair-Id=APKAIB5DGCGAQJ4GGIUQ&sessionID=2af7bd71-8772-498f-8c74-7f9547e81b06';				
					await console.log('downloading '+i);
			  	var file_ts_path = fs.createWriteStream(i+'.ts');
					let responseRes = await doRequest(ts_url);
					await responseRes.pipe(file_ts_path);	
			}				
})();