kafka角色运行挂掉
请教下有人知道原因吗
2022-12-04 06:08:45,908 WARN org.eclipse.jetty.server.AbstractConnector:
java.io.IOException: Too many open files in system
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:397)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
2022-12-04 06:08:45,908 WARN org.eclipse.jetty.server.AbstractConnector:
java.io.IOException: Too many open files in system
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:397)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
2022-12-04 06:08:45,908 WARN org.eclipse.jetty.server.AbstractConnector:
java.io.IOException: Too many open files in system
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:397)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
2022-12-04 06:08:45,908 WARN org.eclipse.jetty.server.AbstractConnector:
java.io.IOException: Too many open files in system
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:397)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
2022-12-04 06:08:48,795 ERROR kafka.server.LogDirFailureChannel: Error while writing to checkpoint file /var/local/kafka/data/replication-offset-checkpoint
java.io.FileNotFoundException: /var/local/kafka/data/replication-offset-checkpoint.tmp (Too many open files in system)
at java.io.FileOutputStream.open0(Native Method)
at java.io.FileOutputStream.open(FileOutputStream.java:270)
at java.io.FileOutputStream.<init>(FileOutputStream.java:213)
at java.io.FileOutputStream.<init>(FileOutputStream.java:162)
at kafka.server.checkpoints.CheckpointFile.liftedTree1$1(CheckpointFile.scala:52)
at kafka.server.checkpoints.CheckpointFile.write(CheckpointFile.scala:50)
at kafka.server.checkpoints.OffsetCheckpointFile.write(OffsetCheckpointFile.scala:59)
at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2$$anonfun$apply$53.apply(ReplicaManager.scala:1437)
at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2$$anonfun$apply$53.apply(ReplicaManager.scala:1437)
at scala.Option.foreach(Option.scala:257)
at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2.apply(ReplicaManager.scala:1437)
at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2.apply(ReplicaManager.scala:1434)
at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733)
at scala.collection.immutable.Map$Map1.foreach(Map.scala:116)
at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732)
at kafka.server.ReplicaManager.checkpointHighWatermarks(ReplicaManager.scala:1434)
at kafka.server.ReplicaManager$$anonfun$1.apply$mcV$sp(ReplicaManager.scala:256)
at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114)
at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2022-12-04 06:08:48,796 INFO kafka.server.ReplicaManager: [ReplicaManager broker=1032] Stopping serving replicas in dir /var/local/kafka/data
2022-12-04 06:08:48,796 ERROR kafka.server.ReplicaManager: [ReplicaManager broker=1032] Error while writing to highwatermark file in directory /var/local/kafka/data
org.apache.kafka.common.errors.KafkaStorageException: Error while writing to checkpoint file /var/local/kafka/data/replication-offset-checkpoint
Caused by: java.io.FileNotFoundException: /var/local/kafka/data/replication-offset-checkpoint.tmp (Too many open files in system)
at java.io.FileOutputStream.open0(Native Method)
at java.io.FileOutputStream.open(FileOutputStream.java:270)
at java.io.FileOutputStream.<init>(FileOutputStream.java:213)
at java.io.FileOutputStream.<init>(FileOutputStream.java:162)
at kafka.server.checkpoints.CheckpointFile.liftedTree1$1(CheckpointFile.scala:52)
at kafka.server.checkpoints.CheckpointFile.write(CheckpointFile.scala:50)
at kafka.server.checkpoints.OffsetCheckpointFile.write(OffsetCheckpointFile.scala:59)
at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2$$anonfun$apply$53.apply(ReplicaManager.scala:1437)
at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2$$anonfun$apply$53.apply(ReplicaManager.scala:1437)
at scala.Option.foreach(Option.scala:257)
at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2.apply(ReplicaManager.scala:1437)
at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2.apply(ReplicaManager.scala:1434)
at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733)
at scala.collection.immutable.Map$Map1.foreach(Map.scala:116)
at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732)
at kafka.server.ReplicaManager.checkpointHighWatermarks(ReplicaManager.scala:1434)
at kafka.server.ReplicaManager$$anonfun$1.apply$mcV$sp(ReplicaManager.scala:256)
at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114)
at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2022-12-04 06:08:48,804 INFO kafka.server.ReplicaFetcherManager: [ReplicaFetcherManager on broker 1032] Removed fetcher for partitions Set(__consumer_offsets-22, topic_test_04-3, beta-log-analysis-topic_x1-7, __consumer_offsets-30, beta-log-analysis-topic_x1-0, __consumer_offsets-8, __consumer_offsets-21, __consumer_offsets-4, __consumer_offsets-27, __consumer_offsets-7, __consumer_offsets-9, __consumer_offsets-46, topic_test_04-6, topic_test_05-7, topic_test_07-8, beta-log-analysis-topic_x1-1, __consumer_offsets-25, flink_test-11, __consumer_offsets-35, topic_test_05-4, __consumer_offsets-41, __consumer_offsets-33, __consumer_offsets-23, __consumer_offsets-49, topic_test_06-5, beta-log-analysis-topic_x1-8, beta-log-analysis-topic_x1-9, __consumer_offsets-47, __consumer_offsets-16, __consumer_offsets-28, __consumer_offsets-31, __consumer_offsets-36, __consumer_offsets-42, __consumer_offsets-3, topic_test_04-9, __consumer_offsets-18, beta-log-analysis-topic_f1-4, __consumer_offsets-37, topic_test_03-0, flink_test-9, __consumer_offsets-15, __consumer_offsets-24, flink_test-5, beta-log-analysis-topic_x1-2, flink_test-7, flink_test-3, beta-log-analysis-topic_f1-0, __consumer_offsets-38, __consumer_offsets-17, beta-log-analysis-topic_x1-5, __consumer_offsets-48, beta-log-analysis-topic_f1-7, __consumer_offsets-19, __consumer_offsets-11, beta-log-analysis-topic_x1-3, flink_test-8, topic_test_06-2, flink_test-4, topic_test_05-1, __consumer_offsets-13, __consumer_offsets-2, __consumer_offsets-43, __consumer_offsets-6, __consumer_offsets-14, beta-log-analysis-topic_f1-5, flink_test-1, topic_test_04-0, flink_test-2, topic_test_07-5, topic_test_06-8, __consumer_offsets-20, beta-log-analysis-topic_f1-1, __consumer_offsets-0, __consumer_offsets-44, __consumer_offsets-39, __consumer_offsets-12, beta-log-analysis-topic_x1-6, beta-log-analysis-topic_f1-6, flink_test-0, __consumer_offsets-45, __consumer_offsets-1, __consumer_offsets-5, __consumer_offsets-26, beta-log-analysis-topic_x1-4, __consumer_offsets-29, __consumer_offsets-34, __consumer_offsets-10, flink_test-10, flink_test-6, __consumer_offsets-32, __consumer_offsets-40, topic_test_07-2)
2022-12-04 06:08:48,804 INFO kafka.server.ReplicaAlterLogDirsManager: [ReplicaAlterLogDirsManager on broker 1032] Removed fetcher for partitions Set(__consumer_offsets-22, topic_test_04-3, beta-log-analysis-topic_x1-7, __consumer_offsets-30, beta-log-analysis-topic_x1-0, __consumer_offsets-8, __consumer_offsets-21, __consumer_offsets-4, __consumer_offsets-27, __consumer_offsets-7, __consumer_offsets-9, __consumer_offsets-46, topic_test_04-6, topic_test_05-7, topic_test_07-8, beta-log-analysis-topic_x1-1, __consumer_offsets-25, flink_test-11, __consumer_offsets-35, topic_test_05-4, __consumer_offsets-41, __consumer_offsets-33, __consumer_offsets-23, __consumer_offsets-49, topic_test_06-5, beta-log-analysis-topic_x1-8, beta-log-analysis-topic_x1-9, __consumer_offsets-47, __consumer_offsets-16, __consumer_offsets-28, __consumer_offsets-31, __consumer_offsets-36, __consumer_offsets-42, __consumer_offsets-3, topic_test_04-9, __consumer_offsets-18, beta-log-analysis-topic_f1-4, __consumer_offsets-37, topic_test_03-0, flink_test-9, __consumer_offsets-15, __consumer_offsets-24, flink_test-5, beta-log-analysis-topic_x1-2, flink_test-7, flink_test-3, beta-log-analysis-topic_f1-0, __consumer_offsets-38, __consumer_offsets-17, beta-log-analysis-topic_x1-5, __consumer_offsets-48, beta-log-analysis-topic_f1-7, __consumer_offsets-19, __consumer_offsets-11, beta-log-analysis-topic_x1-3, flink_test-8, topic_test_06-2, flink_test-4, topic_test_05-1, __consumer_offsets-13, __consumer_offsets-2, __consumer_offsets-43, __consumer_offsets-6, __consumer_offsets-14, beta-log-analysis-topic_f1-5, flink_test-1, topic_test_04-0, flink_test-2, topic_test_07-5, topic_test_06-8, __consumer_offsets-20, beta-log-analysis-topic_f1-1, __consumer_offsets-0, __consumer_offsets-44, __consumer_offsets-39, __consumer_offsets-12, beta-log-analysis-topic_x1-6, beta-log-analysis-topic_f1-6, flink_test-0, __consumer_offsets-45, __consumer_offsets-1, __consumer_offsets-5, __consumer_offsets-26, beta-log-analysis-topic_x1-4, __consumer_offsets-29, __consumer_offsets-34, __consumer_offsets-10, flink_test-10, flink_test-6, __consumer_offsets-32, __consumer_offsets-40, topic_test_07-2)
2022-12-04 06:08:48,852 INFO kafka.server.ReplicaManager: [ReplicaManager broker=1032] Broker 1032 stopped fetcher for partitions __consumer_offsets-22,topic_test_04-3,beta-log-analysis-topic_x1-7,__consumer_offsets-30,beta-log-analysis-topic_x1-0,__consumer_offsets-8,__consumer_offsets-21,__consumer_offsets-4,__consumer_offsets-27,__consumer_offsets-7,__consumer_offsets-9,__consumer_offsets-46,topic_test_04-6,topic_test_05-7,topic_test_07-8,beta-log-analysis-topic_x1-1,__consumer_offsets-25,flink_test-11,__consumer_offsets-35,topic_test_05-4,__consumer_offsets-41,__consumer_offsets-33,__consumer_offsets-23,__consumer_offsets-49,topic_test_06-5,beta-log-analysis-topic_x1-8,beta-log-analysis-topic_x1-9,__consumer_offsets-47,__consumer_offsets-16,__consumer_offsets-28,__consumer_offsets-31,__consumer_offsets-36,__consumer_offsets-42,__consumer_offsets-3,topic_test_04-9,__consumer_offsets-18,beta-log-analysis-topic_f1-4,__consumer_offsets-37,topic_test_03-0,flink_test-9,__consumer_offsets-15,__consumer_offsets-24,flink_test-5,beta-log-analysis-topic_x1-2,flink_test-7,flink_test-3,beta-log-analysis-topic_f1-0,__consumer_offsets-38,__consumer_offsets-17,beta-log-analysis-topic_x1-5,__consumer_offsets-48,beta-log-analysis-topic_f1-7,__consumer_offsets-19,__consumer_offsets-11,beta-log-analysis-topic_x1-3,flink_test-8,topic_test_06-2,flink_test-4,topic_test_05-1,__consumer_offsets-13,__consumer_offsets-2,__consumer_offsets-43,__consumer_offsets-6,__consumer_offsets-14,beta-log-analysis-topic_f1-5,flink_test-1,topic_test_04-0,flink_test-2,topic_test_07-5,topic_test_06-8,__consumer_offsets-20,beta-log-analysis-topic_f1-1,__consumer_offsets-0,__consumer_offsets-44,__consumer_offsets-39,__consumer_offsets-12,beta-log-analysis-topic_x1-6,beta-log-analysis-topic_f1-6,flink_test-0,__consumer_offsets-45,__consumer_offsets-1,__consumer_offsets-5,__consumer_offsets-26,beta-log-analysis-topic_x1-4,__consumer_offsets-29,__consumer_offsets-34,__consumer_offsets-10,flink_test-10,flink_test-6,__consumer_offsets-32,__consumer_offsets-40,topic_test_07-2 and stopped moving logs for partitions because they are in the failed log directory /var/local/kafka/data.
2022-12-04 06:08:48,852 INFO kafka.log.LogManager: Stopping serving logs in dir /var/local/kafka/data
2022-12-04 06:08:48,855 ERROR kafka.log.LogManager: Shutdown broker because all log dirs in /var/local/kafka/data have failed
打开文件太多了
解决办法：参考https://blog.csdn.net/weixin_45444133/article/details/103645762