Ggraphicsmagick ломает mongoDB. Я не знаю почему - PullRequest
Новая
       62

Ggraphicsmagick ломает mongoDB. Я не знаю почему

0 голосов
/ 09 апреля 2020

Добрый день! У меня есть сервер, который снимает изображения, создает эскизы на фоне и записывает имена файлов в mongoDB сразу после запуска процесса создания эскизов.

У меня проблема. Когда я загружаю 30 картинок, сервер mon go прерывается (останавливается). Когда я загружаю 2 картинки, это нормально.

Почему это происходит? Кто-нибудь объясните мне, пожалуйста!

ОБНОВЛЕНИЕ: Проблема появляется только на выделенном сервере. У меня нет проблем на локальном хосте. ОБНОВЛЕНИЕ: Прикрепленные журналы mongoBD

загрузка. js

const
  fs = require('fs'),
  {Router} = require("express"),
  router = Router(),
  multer = require('multer'),
  colors = require('colors'),
  path = require('path'),
  cryptoRandomString = require('crypto-random-string');


const
  thumbnail = require('./thumbnail.js'),
  db_controllers = require('./db_controllers.js'),
  _meta = require('./meta.js');
  helpers = require("../helpers.js"),
  defaults = require("../defaults.js");


let storage = multer.diskStorage({

  destination: function(req, file, cb) {
    let uploads_dir = "public/models/"+req.body.model_id+"/"+req.body.album_id;
    fs.exists("./"+uploads_dir, function(err) {
      fs.mkdir(uploads_dir, {recursive: true}, function(err) {
        cb(null, uploads_dir);
        return;
      });
    });
  },
  filename: function (req, file, cb){
    let random_name = cryptoRandomString({length: 15});
    cb( null, file.originalname.replace(/[а-я-()]/gi, "").split('.').slice(0, -1).join('.') +"_"+ random_name + path.extname(file.originalname) );
  }
});

var upload = multer({ storage: storage });

router.post('/uploadmultiple', upload.array('myFiles', 200), async function (req, res, next) {

  const files = req.files;
  if (!files) {
    const error = new Error('Please choose files')
    error.httpStatusCode = 400
    return next(error)
  }

  let data = {
    content: [],
    covers: [],
  };

  let meta = _meta.GenerateMetaData(req);

  thumb_sizes = { w: defaults.app.thumb_size.w, /*h: defaults.app.thumb_size.h*/ undefined };

  for (file of files) {  
    console.log("++ "+file.originalname);

    if (file.mimetype == 'image/jpeg' || file.mimetype == 'image/png' || file.mimetype == 'image/webp') {
      extname = path.extname(file.filename);
      filename = file.filename.split('.').slice(0, -1).join('.');

      thumbnail.genThumbs(file.destination, file.filename, thumb_sizes);

      if ( helpers.findString( filename, ["cover"] ) ) {
        data.covers.push({
          image: filename+"_thumbnail"+extname,
          alt: meta.alt[ helpers.SetRandom( meta.alt.length ) ],
        });
      }

      data.content.push({
        original: file.filename,
        thumbnail: filename+"_thumbnail"+extname,
        type: "photo",
        alt: meta.alt[ helpers.SetRandom( meta.alt.length ) ],
      });

    }
  }

  console.log(data);

  data.title = meta.title,
  data.description = meta.description,
  data.album_id = req.body.album_id;
  data.name = req.body.album_name;
  data.model_id = req.body.model_id;
  data.model_name = req.body.model_name;
  data.sise = req.body.sise;
  data.seo_description = req.body.seo_description;
  data.tags = req.body.tags.split(',');
  data.date = new Date(Date.now() + parseInt(req.body.published_after));
  data.category = req.body.category;
  data.author = cryptoRandomString({length: 20, type: 'url-safe'});

  db_controllers.CreateAlbum(data, function() {
    res.send(files);
  });

});

миниатюра. js

const gm = require('gm').subClass({imageMagick: true});
const path = require('path');
const colors = require('colors');
const ffmpeg = require('fluent-ffmpeg');
const command = ffmpeg();


function genThumbs(origin_path, f_name, thumb_size) {

  f_ext = path.extname(f_name);
  f_name = f_name.split('.').slice(0, -1).join('.');

  gm(origin_path+"/"+f_name+f_ext)
  .resize(thumb_size.w, thumb_size.h)
  .noProfile()
  .write(origin_path+"/"+f_name+"_thumbnail"+f_ext, function(err) {
    if (!err) {
      console.log('thumbnails is generated');

      //return (origin_path+"/"+f_name+"_thumbnail"+f_ext);
    }
    else {
      console.log(err);
      return(undefined);
    }
  });
}

module.exports.genThumbs = genThumbs;

db_controllers. js

const
  colors = require('colors'),
  defaults = require("../defaults.js");

const MongoClient = require("mongodb").MongoClient;
const options = {
  useNewUrlParser: true,
  useUnifiedTopology: true,
  // keepAlive: 1,
  // connectTimeoutMS: 60000,
  // socketTimeoutMS: 60000,
  // auto_reconnect: true,
}
const mongoClient = new MongoClient(defaults.mongo.url, options);

db_name = defaults.mongo.db;
albums_coll = defaults.mongo.albums_coll;

function CreateAlbum (data, callback) {

  console.log("/upload/db_conttrollers.js CreateAlbum() – is working");

  // ERROR HAPPENS HERE

  mongoClient.connect(function(err, dbClient) {
    if(err) throw err;
    console.log("----- posts.js: CreateAlbumRecord(): client is created -----");

    let collection = dbClient.db(db_name).collection(albums_coll);
    data.views = 0;

    collection.insertOne(data, function(err, res) {
      if (err) throw err;
      console.log("-- posts.js: CreateAlbumRecord(): album_object is inserted --");
      callback();
    });
  });
}

module.exports.CreateAlbum = CreateAlbum;

Текст ошибки 

MongoServerSelectionError: connect ECONNREFUSED 127.0.0.1:27017
    at Timeout.<anonymous> (/root/bg/bg_site_node_hosting/node_modules/mongodb/lib/core/sdam/server_selection.js:307:9)
    at listOnTimeout (internal/timers.js:549:17)
    at processTimers (internal/timers.js:492:7) {
  name: 'MongoServerSelectionError',
  reason: TopologyDescription {
    type: 'Single',
    setName: null,
    maxSetVersion: null,
    maxElectionId: null,
    servers: Map { 'localhost:27017' => [ServerDescription] },
    stale: false,
    compatible: true,
    compatibilityError: null,
    logicalSessionTimeoutMinutes: null,
    heartbeatFrequencyMS: 10000,
    localThresholdMS: 15,
    commonWireVersion: null
  },
  [Symbol(mongoErrorContextSymbol)]: {}
}

Журнал MongoDB 

2020-04-09T03:01:33.204+0000 I  CONTROL  [main] ***** SERVER RESTARTED *****
2020-04-09T03:01:33.218+0000 I  CONTROL  [main] Automatically disabling TLS 1.0, to force-enable TLS 1.0 specify --sslDisabledProtocols 'none'
2020-04-09T03:01:33.311+0000 W  ASIO     [main] No TransportLayer configured during NetworkInterface startup
2020-04-09T03:01:33.313+0000 I  CONTROL  [initandlisten] MongoDB starting : pid=13500 port=27017 dbpath=/var/lib/mongodb 64-bit host=ubuntu-s-1vcpu-1gb-fra1-01
2020-04-09T03:01:33.313+0000 I  CONTROL  [initandlisten] db version v4.2.5
2020-04-09T03:01:33.313+0000 I  CONTROL  [initandlisten] git version: 2261279b51ea13df08ae708ff278f0679c59dc32
2020-04-09T03:01:33.313+0000 I  CONTROL  [initandlisten] OpenSSL version: OpenSSL 1.1.1  11 Sep 2018
2020-04-09T03:01:33.313+0000 I  CONTROL  [initandlisten] allocator: tcmalloc
2020-04-09T03:01:33.313+0000 I  CONTROL  [initandlisten] modules: none
2020-04-09T03:01:33.313+0000 I  CONTROL  [initandlisten] build environment:
2020-04-09T03:01:33.313+0000 I  CONTROL  [initandlisten]     distmod: ubuntu1804
2020-04-09T03:01:33.313+0000 I  CONTROL  [initandlisten]     distarch: x86_64
2020-04-09T03:01:33.313+0000 I  CONTROL  [initandlisten]     target_arch: x86_64
2020-04-09T03:01:33.313+0000 I  CONTROL  [initandlisten] options: { config: "/etc/mongod.conf", net: { bindIp: "127.0.0.1", port: 27017 }, processManagement: { timeZoneInfo: "/usr/share/zoneinfo" }, storage: { dbPath: "/var/lib/mongodb", journal: { enabled: true } }, systemLog: { destination: "file", logAppend: true, path: "/var/log/mongodb/mongod.log" } }
2020-04-09T03:01:33.318+0000 W  STORAGE  [initandlisten] Detected unclean shutdown - /var/lib/mongodb/mongod.lock is not empty.
2020-04-09T03:01:33.318+0000 I  STORAGE  [initandlisten] Detected data files in /var/lib/mongodb created by the 'wiredTiger' storage engine, so setting the active storage engine to 'wiredTiger'.
2020-04-09T03:01:33.318+0000 W  STORAGE  [initandlisten] Recovering data from the last clean checkpoint.
2020-04-09T03:01:33.318+0000 I  STORAGE  [initandlisten] 
2020-04-09T03:01:33.318+0000 I  STORAGE  [initandlisten] ** WARNING: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine
2020-04-09T03:01:33.318+0000 I  STORAGE  [initandlisten] **          See http://dochub.mongodb.org/core/prodnotes-filesystem
2020-04-09T03:01:33.318+0000 I  STORAGE  [initandlisten] wiredtiger_open config: create,cache_size=256M,cache_overflow=(file_max=0M),session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress],
2020-04-09T03:01:33.908+0000 I  STORAGE  [initandlisten] WiredTiger message [1586401293:908591][13500:0x7fbed1005b00], txn-recover: Recovering log 19 through 20
2020-04-09T03:01:33.987+0000 I  STORAGE  [initandlisten] WiredTiger message [1586401293:987910][13500:0x7fbed1005b00], txn-recover: Recovering log 20 through 20
2020-04-09T03:01:34.063+0000 I  STORAGE  [initandlisten] WiredTiger message [1586401294:63623][13500:0x7fbed1005b00], txn-recover: Main recovery loop: starting at 19/10112 to 20/256
2020-04-09T03:01:34.064+0000 I  STORAGE  [initandlisten] WiredTiger message [1586401294:64932][13500:0x7fbed1005b00], txn-recover: Recovering log 19 through 20
2020-04-09T03:01:34.125+0000 I  STORAGE  [initandlisten] WiredTiger message [1586401294:125783][13500:0x7fbed1005b00], txn-recover: Recovering log 20 through 20
2020-04-09T03:01:34.176+0000 I  STORAGE  [initandlisten] WiredTiger message [1586401294:176256][13500:0x7fbed1005b00], txn-recover: Set global recovery timestamp: (0, 0)
2020-04-09T03:01:34.228+0000 I  RECOVERY [initandlisten] WiredTiger recoveryTimestamp. Ts: Timestamp(0, 0)
2020-04-09T03:01:34.243+0000 I  STORAGE  [initandlisten] Timestamp monitor starting
2020-04-09T03:01:34.255+0000 I  CONTROL  [initandlisten] 
2020-04-09T03:01:34.255+0000 I  CONTROL  [initandlisten] ** WARNING: Access control is not enabled for the database.
2020-04-09T03:01:34.255+0000 I  CONTROL  [initandlisten] **          Read and write access to data and configuration is unrestricted.
2020-04-09T03:01:34.255+0000 I  CONTROL  [initandlisten] 
2020-04-09T03:01:34.273+0000 I  SHARDING [initandlisten] Marking collection local.system.replset as collection version: <unsharded>
2020-04-09T03:01:34.282+0000 I  STORAGE  [initandlisten] Flow Control is enabled on this deployment.
2020-04-09T03:01:34.284+0000 I  SHARDING [initandlisten] Marking collection admin.system.roles as collection version: <unsharded>
2020-04-09T03:01:34.285+0000 I  SHARDING [initandlisten] Marking collection admin.system.version as collection version: <unsharded>
2020-04-09T03:01:34.291+0000 I  SHARDING [initandlisten] Marking collection local.startup_log as collection version: <unsharded>
2020-04-09T03:01:34.291+0000 I  FTDC     [initandlisten] Initializing full-time diagnostic data capture with directory '/var/lib/mongodb/diagnostic.data'
2020-04-09T03:01:34.295+0000 I  SHARDING [LogicalSessionCacheRefresh] Marking collection config.system.sessions as collection version: <unsharded>
2020-04-09T03:01:34.296+0000 I  SHARDING [LogicalSessionCacheReap] Marking collection config.transactions as collection version: <unsharded>
2020-04-09T03:01:34.296+0000 I  NETWORK  [listener] Listening on /tmp/mongodb-27017.sock
2020-04-09T03:01:34.296+0000 I  NETWORK  [listener] Listening on 127.0.0.1
2020-04-09T03:01:34.296+0000 I  NETWORK  [listener] waiting for connections on port 27017
2020-04-09T03:01:35.005+0000 I  FTDC     [ftdc] Unclean full-time diagnostic data capture shutdown detected, found interim file, some metrics may have been lost. OK
2020-04-09T03:01:35.013+0000 I  SHARDING [ftdc] Marking collection local.oplog.rs as collection version: <unsharded>

1 Ответ

0 голосов
/ 09 апреля 2020

Приложение работает на простейшей капле DigitalOcean. Имеет 1 Гб оперативной памяти. Graphicsmagick / ImageMagick используют всю оперативную память и разбивают пн go. Чтобы избежать этого, я буду запускать Thumbnail Generator с задержкой. Теперь работает нормально. 

function genThumbs(origin_path, f_name, thumb_size, delay) {

  f_ext = path.extname(f_name);
  f_name = f_name.split('.').slice(0, -1).join('.');

  setTimeout(function() {
    gm(origin_path+"/"+f_name+f_ext)
    .resize(thumb_size.w, thumb_size.h)
    .noProfile()
    .write(origin_path+"/"+f_name+"_thumbnail"+f_ext, function(err) {
      if (!err) {
        console.log('thumbnails is generated');

        //return (origin_path+"/"+f_name+"_thumbnail"+f_ext);
      }
      else {
        console.log(err);
        return(undefined);
      }
    });
  }, delay);
}
...