В моем приложении txt-файл обрабатывается с использованием apache camel, приложение считывает весь файл в строку, используя объект обмена, и выполняет добавление разделителя в каждой строке и записывает в файл как целую строку, используя объект обмена * 1011. * верблюд. Я получаю ниже OutOfMemoryError. *** Я заметил, что это исключение возникает после того, как обработка файла завершена объектом обмена и когда он должен переместить файл в другую папку (конечную точку) * **.
Мое приложение имеет кучу 16 ГБ космическая поддержка . Размер файла до обработки составляет 685 МБ, а после добавления разделителя он может увеличиться на 200 МБ. Может ли кто-нибудь помочь мне здесь? Заранее спасибо.
from("file:<filepath>/processed/INPUT?filter=#fileFilter&delay=120000&exclusiveReadLockStrategy=#fileReadLockStrategy&delete=true&autoCreate=true")
.threads(10, 10)
.process(fileProcessor).log("Came out of file processor")
.recipientList(header(DESTIONATION_PATH));
Это простой маршрут, который читает файл из папки INPUT и fileProcessor обрабатывает содержимое файла, добавляя в него разделитель. DESTIONATION_PATH - это путь к файлу. Я вижу, что это печать журнала «Из файлового процессора», ниже Исключение наступает, когда файл собирается отправить в папку назначения.
Error has occurred:
org.apache.camel.CamelExecutionException: Exception occurred during execution on the exchange: Exchange[ID-x01btadmdts3a-vsi-uat-dbs-com-1586519346908-0-4]
at org.apache.camel.util.ObjectHelper.wrapCamelExecutionException(ObjectHelper.java:1842) ~[camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.impl.DefaultExchange.setException(DefaultExchange.java:385) ~[camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:64) ~[camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.MulticastProcessor.doProcessSequential(MulticastProcessor.java:715) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.MulticastProcessor.doProcessSequential(MulticastProcessor.java:638) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.MulticastProcessor.process(MulticastProcessor.java:248) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.RecipientList.sendToRecipientList(RecipientList.java:173) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.RecipientList.process(RecipientList.java:133) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.management.InstrumentationProcessor.process(InstrumentationProcessor.java:76) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.Pipeline.access$100(Pipeline.java:43) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.Pipeline$1.done(Pipeline.java:157) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.ThreadsProcessor$ProcessCall.run(ThreadsProcessor.java:87) [camel-core-2.24.1.jar!/:2.24.1]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_171]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_171]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_171]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_171]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_171]
Suppressed: org.apache.camel.CamelExecutionException: Exception occurred during execution on the exchange: Exchange[ID-x01btadmdts3a-vsi-uat-dbs-com-1586519346908-0-4]
... 26 common frames omitted
***Caused by: java.lang.OutOfMemoryError: Requested array size exceeds VM limit***
at java.lang.StringCoding$StringEncoder.encode(StringCoding.java:300)
at java.lang.StringCoding.encode(StringCoding.java:344)
at java.lang.String.getBytes(String.java:918)
at org.apache.camel.converter.IOConverter.toInputStream(IOConverter.java:198)
at org.apache.camel.converter.IOConverterOptimised.convertTo(IOConverterOptimised.java:55)
at org.apache.camel.impl.converter.OptimisedTypeConverter.convertTo(OptimisedTypeConverter.java:63)
at org.apache.camel.impl.converter.BaseTypeConverterRegistry.doConvertTo(BaseTypeConverterRegistry.java:299)
at org.apache.camel.impl.converter.BaseTypeConverterRegistry.mandatoryConvertTo(BaseTypeConverterRegistry.java:188)
at org.apache.camel.impl.MessageSupport.getMandatoryBody(MessageSupport.java:115)
at org.apache.camel.component.file.FileOperations.storeFile(FileOperations.java:333)
at org.apache.camel.component.file.GenericFileProducer.writeFile(GenericFileProducer.java:305)
at org.apache.camel.component.file.GenericFileProducer.processExchange(GenericFileProducer.java:169)
at org.apache.camel.component.file.GenericFileProducer.process(GenericFileProducer.java:80)
at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61)
at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548)
at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201)
at org.apache.camel.processor.MulticastProcessor.doProcessSequential(MulticastProcessor.java:715)
at org.apache.camel.processor.MulticastProcessor.doProcessSequential(MulticastProcessor.java:638)
at org.apache.camel.processor.MulticastProcessor.process(MulticastProcessor.java:248)
at org.apache.camel.processor.RecipientList.sendToRecipientList(RecipientList.java:173)
at org.apache.camel.processor.RecipientList.process(RecipientList.java:133)
at org.apache.camel.processor.Pipeline.process(Pipeline.java:138)
at org.apache.camel.processor.Pipeline.process(Pipeline.java:101)
at org.apache.camel.management.InstrumentationProcessor.process(InstrumentationProcessor.java:76)
at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548)
at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201)
at org.apache.camel.processor.Pipeline.process(Pipeline.java:138)
at org.apache.camel.processor.Pipeline.process(Pipeline.java:101)
at org.apache.camel.processor.Pipeline.process(Pipeline.java:138)
at org.apache.camel.processor.Pipeline.access$100(Pipeline.java:43)
at org.apache.camel.processor.Pipeline$1.done(Pipeline.java:157)
at org.apache.camel.processor.ThreadsProcessor$ProcessCall.run(ThreadsProcessor.java:87)
Caused by: java.lang.OutOfMemoryError: Requested array size exceeds VM limit
at java.lang.StringCoding$StringEncoder.encode(StringCoding.java:300) ~[na:1.8.0_171]
at java.lang.StringCoding.encode(StringCoding.java:344) ~[na:1.8.0_171]
at java.lang.String.getBytes(String.java:918) ~[na:1.8.0_171]
at org.apache.camel.converter.IOConverter.toInputStream(IOConverter.java:198) ~[camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.converter.IOConverterOptimised.convertTo(IOConverterOptimised.java:55) ~[camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.impl.converter.OptimisedTypeConverter.convertTo(OptimisedTypeConverter.java:63) ~[camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.impl.converter.BaseTypeConverterRegistry.doConvertTo(BaseTypeConverterRegistry.java:299) ~[camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.impl.converter.BaseTypeConverterRegistry.mandatoryConvertTo(BaseTypeConverterRegistry.java:188) ~[camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.impl.MessageSupport.getMandatoryBody(MessageSupport.java:115) ~[camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.component.file.FileOperations.storeFile(FileOperations.java:333) ~[camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.component.file.GenericFileProducer.writeFile(GenericFileProducer.java:305) ~[camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.component.file.GenericFileProducer.processExchange(GenericFileProducer.java:169) ~[camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.component.file.GenericFileProducer.process(GenericFileProducer.java:80) ~[camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) ~[camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.MulticastProcessor.doProcessSequential(MulticastProcessor.java:715) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.MulticastProcessor.doProcessSequential(MulticastProcessor.java:638) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.MulticastProcessor.process(MulticastProcessor.java:248) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.RecipientList.sendToRecipientList(RecipientList.java:173) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.RecipientList.process(RecipientList.java:133) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.management.InstrumentationProcessor.process(InstrumentationProcessor.java:76) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.Pipeline.access$100(Pipeline.java:43) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.Pipeline$1.done(Pipeline.java:157) [camel-core-2.24.1.jar!/:2.24.1]
at org.apache.camel.processor.ThreadsProcessor$ProcessCall.run(ThreadsProcessor.java:87) [camel-core-2.24.1.jar!/:2.24.1]