В неопределенном цикле на сервере, где установлена Cassandra, папка с подсказками удаляется сама по себе, что предотвращает подключение к Cassandra. Журнал появления выглядит следующим образом.
[2020-03-01 10:08:17,045] [STORAGE] [INFO] [CompactionExecutor:298] AutoSavingCache.java:404 - Saved KeyCache (100 items) in 7 ms
[2020-03-01 10:09:05,880] [STORAGE] [INFO] [IndexSummaryManager:1] IndexSummaryRedistribution.java:84 - Redistributing index summaries
[2020-03-01 11:09:05,884] [STORAGE] [INFO] [IndexSummaryManager:1] IndexSummaryRedistribution.java:84 - Redistributing index summaries
[2020-03-01 11:25:31,759] [STORAGE] [WARN] [HintsWriteExecutor:1] NativeLibrary.java:316 - open(/tmp/cassandra/hints, O_RDONLY) failed, errno (2).
[2020-03-01 11:25:31,759] [STORAGE] [ERROR] [HintsWriteExecutor:1] HintsCatalog.java:166 - Unable to open directory /tmp/cassandra/hints
[2020-03-01 11:25:31,761] [STORAGE] [ERROR] [HintsWriteExecutor:1] StorageService.java:397 - Stopping gossiper
[2020-03-01 11:25:31,761] [STORAGE] [WARN] [HintsWriteExecutor:1] StorageService.java:307 - Stopping gossip by operator request
[2020-03-01 11:25:31,761] [STORAGE] [INFO] [HintsWriteExecutor:1] Gossiper.java:1801 - Announcing shutdown
[2020-03-01 11:25:31,762] [STORAGE] [INFO] [HintsWriteExecutor:1] StorageService.java:2498 - Node 192.168.0.50:7000 state jump to shutdown
[2020-03-01 11:25:31,763] [STORAGE] [INFO] [HintsWriteExecutor:1] StorageService.java:2498 - Node 192.168.0.50:7000 state jump to shutdown
[2020-03-01 11:25:33,765] [STORAGE] [ERROR] [HintsWriteExecutor:1] StorageService.java:402 - Stopping native transport
[2020-03-01 11:25:33,768] [STORAGE] [INFO] [HintsWriteExecutor:1] Server.java:213 - Stop listening for CQL clients
[2020-03-01 11:25:33,771] [STORAGE] [ERROR] [HintsWriteExecutor:1] CassandraDaemon.java:442 - Exception in thread Thread[HintsWriteExecutor:1,5,main]
org.apache.cassandra.io.FSWriteError: java.io.IOException: Unable to open hint directory /tmp/cassandra/hints
at org.apache.cassandra.hints.HintsCatalog.fsyncDirectory(HintsCatalog.java:167)
at org.apache.cassandra.hints.HintsWriteExecutor$FsyncWritersTask.run(HintsWriteExecutor.java:204)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.io.IOException: Unable to open hint directory /tmp/cassandra/hints
... 8 common frames omitted
[2020-03-01 11:25:33,771] [STORAGE] [ERROR] [BatchlogTasks:1] CassandraDaemon.java:442 - Exception in thread Thread[BatchlogTasks:1,5,main]
java.lang.RuntimeException: java.util.concurrent.ExecutionException: FSWriteError in /tmp/cassandra/hints
at org.apache.cassandra.hints.HintsWriteExecutor.fsyncWritersBlockingly(HintsWriteExecutor.java:107)
at org.apache.cassandra.hints.HintsService.flushAndFsyncBlockingly(HintsService.java:200)
at org.apache.cassandra.batchlog.BatchlogManager.processBatchlogEntries(BatchlogManager.java:313)
at org.apache.cassandra.batchlog.BatchlogManager.replayFailedBatches(BatchlogManager.java:226)
at org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor$UncomplainingRunnable.run(DebuggableScheduledThreadPoolExecutor.java:118)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.ExecutionException: FSWriteError in /tmp/cassandra/hints
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
at java.util.concurrent.FutureTask.get(FutureTask.java:192)
at org.apache.cassandra.hints.HintsWriteExecutor.fsyncWritersBlockingly(HintsWriteExecutor.java:103)
... 12 common frames omitted
Caused by: org.apache.cassandra.io.FSWriteError: java.io.IOException: Unable to open hint directory /tmp/cassandra/hints
at org.apache.cassandra.hints.HintsCatalog.fsyncDirectory(HintsCatalog.java:167)
at org.apache.cassandra.hints.HintsWriteExecutor$FsyncWritersTask.run(HintsWriteExecutor.java:204)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
... 4 common frames omitted
Caused by: java.io.IOException: Unable to open hint directory /tmp/cassandra/hints
... 8 common frames omitted
Это происходит в Cassandra db, которая используется в веб-системе на основе Spring Framework и не может быть подключена, пока не будет перезапущена. В Spring Framework планирование переноса данных из другой RDB в Cassandra выполняется периодически, а планировщик выполняется каждый день. Кроме того, время выполнения - полночь.
Мне было интересно, был ли случай, когда папка с подсказками была удалена из самой Кассандры, и есть ли способ определить причину, если это произойдет.
версия cassandra - это снимок cassandra-4.0.