kafka java .io.IOException: слишком много открытых файлов - PullRequest
Новая
       0

kafka java .io.IOException: слишком много открытых файлов

0 голосов
/ 27 апреля 2020

У нас есть развертывание кластера Kafka с 2 узлами с 50 темами (с 3 разделами), и срок хранения каждой топи c составляет 90 дней.

наши серверы имеют 16 ГБ ОЗУ и 12 Cors.

Мы встречаемся с этими ошибками каждую неделю: 100 

[2020-04-25 15:46:23,969] ERROR Error while accepting connection (kafka.network.Acceptor)
java.io.IOException: Too many open files
    at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
    at kafka.network.Acceptor.accept(SocketServer.scala:341)
    at kafka.network.Acceptor.run(SocketServer.scala:284)
    at java.lang.Thread.run(Thread.java:748)
[2020-04-25 15:46:23,969] ERROR Error while accepting connection (kafka.network.Acceptor)
java.io.IOException: Too many open files
    at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
    at kafka.network.Acceptor.accept(SocketServer.scala:341)
    at kafka.network.Acceptor.run(SocketServer.scala:284)
    at java.lang.Thread.run(Thread.java:748)
[2020-04-25 15:46:23,969] ERROR Error while accepting connection (kafka.network.Acceptor)
java.io.IOException: Too many open files
    at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
    at kafka.network.Acceptor.accept(SocketServer.scala:341)
    at kafka.network.Acceptor.run(SocketServer.scala:284)
    at java.lang.Thread.run(Thread.java:748)
[2020-04-25 15:46:23,969] ERROR Error while accepting connection (kafka.network.Acceptor)
java.io.IOException: Too many open files
    at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
    at kafka.network.Acceptor.accept(SocketServer.scala:341)
    at kafka.network.Acceptor.run(SocketServer.scala:284)
    at java.lang.Thread.run(Thread.java:748)
[2020-04-25 15:46:23,969] ERROR Error while accepting connection (kafka.network.Acceptor)
java.io.IOException: Too many open files
    at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
    at kafka.network.Acceptor.accept(SocketServer.scala:341)
    at kafka.network.Acceptor.run(SocketServer.scala:284)
    at java.lang.Thread.run(Thread.java:748)
[2020-04-25 15:46:23,969] ERROR Error while accepting connection (kafka.network.Acceptor)
java.io.IOException: Too many open files
    at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
    at kafka.network.Acceptor.accept(SocketServer.scala:341)
    at kafka.network.Acceptor.run(SocketServer.scala:284)
    at java.lang.Thread.run(Thread.java:748)
[2020-04-25 15:46:23,969] ERROR Error while accepting connection (kafka.network.Acceptor)
java.io.IOException: Too many open files
    at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
    at kafka.network.Acceptor.accept(SocketServer.scala:341)
    at kafka.network.Acceptor.run(SocketServer.scala:284)
    at java.lang.Thread.run(Thread.java:748)
[2020-04-25 15:46:23,969] ERROR Error while accepting connection (kafka.network.Acceptor)
java.io.IOException: Too many open files
    at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
    at kafka.network.Acceptor.accept(SocketServer.scala:341)
    at kafka.network.Acceptor.run(SocketServer.scala:284)
    at java.lang.Thread.run(Thread.java:748)
[2020-04-25 15:46:23,970] ERROR Error while accepting connection (kafka.network.Acceptor)
java.io.IOException: Too many open files
    at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
    at kafka.network.Acceptor.accept(SocketServer.scala:341)
    at kafka.network.Acceptor.run(SocketServer.scala:284)
    at java.lang.Thread.run(Thread.java:748)
[2020-04-25 15:46:23,970] ERROR Error while accepting connection (kafka.network.Acceptor)
java.io.IOException: Too many open files
    at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
    at kafka.network.Acceptor.accept(SocketServer.scala:341)
    at kafka.network.Acceptor.run(SocketServer.scala:284)
    at java.lang.Thread.run(Thread.java:748)
[2020-04-25 15:46:23,970] ERROR Error while accepting connection (kafka.network.Acceptor)
java.io.IOException: Too many open files
    at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
    at kafka.network.Acceptor.accept(SocketServer.scala:341)
    at kafka.network.Acceptor.run(SocketServer.scala:284)
    at java.lang.Thread.run(Thread.java:748)
[2020-04-25 15:46:23,970] ERROR Error while accepting connection (kafka.network.Acceptor)
java.io.IOException: Too many open files
    at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
    at kafka.network.Acceptor.accept(SocketServer.scala:341)
    at kafka.network.Acceptor.run(SocketServer.scala:284)
    at java.lang.Thread.run(Thread.java:748)

число открытых файлов: 761092

enter image description here

вывод ulimit -a | grep "open files": 999999

enter image description here

также я добавил эту строку в / etc / security / limit .conf : 

kafka soft nproc 999999
* soft nproc 999999
* hard nproc 999999
* soft nofile 999999
* hard nofile 999999

и добавил эту строку в / etc / sysctl.conf :

fs.file-max = 4097152

Делаем ли мы нужно объединить больше машин в кластер? или увеличить ресурсы, такие как Ram или CPU? или ...

Добро пожаловать на сайт PullRequest, где вы можете задавать вопросы и получать ответы от других членов сообщества.

Нет похожих вопросов

...