Я пытаюсь загрузить файлы в корзину s3 напрямую с SFTP-сервера. Он работает до 1 ГБ и после этого выдает ошибку, как показано ниже. Я использую Apache MINA для настройки виртуального каталога, а конечный узел структуры каталогов связан с хранилищем s3. Ниже приведен код.
public class FileRemotePathChannel extends FileChannel {
@Override
public int write(ByteBuffer src) throws IOException {
if (channel == null) {
pipedInputStream = new PipedInputStream();
pipedOutputStream = new PipedOutputStream(pipedInputStream);
channel = Channels.newChannel(pipedOutputStream);
createS3Thread(this.bucketName);
}
log.info("Writing buffer with size :{} to s3", src);
channel.write(src);
return src.remaining();
}
private void createS3Thread(String bucketName) {
if (s3WriteThread == null) {
log.debug("Transfering file : {} to object store in bucket :{}", path.substring(path.lastIndexOf("/") + 1),
bucketName);
s3WriteThread = new Thread() {
public void run() {
try {
getAmazonS3Instance().writeFileToBucket(bucketName, path.substring(path.lastIndexOf("/") + 1),
pipedInputStream);
pipedInputStream.close();
} catch (Exception e) {
log.error("Error writing file to S3", e);
}
}
};
s3WriteThread.start();
}
}
}
public void writeFileToBucket(String bucket, String fileName, InputStream inputStream) throws IOException {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType("application/octet-stream");
log.info("Uploading file to s3 bucket with bucketName :{}, fileName: {}", bucket, fileName);
s3Connection.awsConnection().putObject(bucket, fileName, inputStream, metadata);
}
Журнал ошибок при загрузке файла объемом более 1 ГБ:
2020-02-28 19:05:23.664 DEBUG com.sc.xyz.sshd.filesystem.FileRemotePathChannel - Setting new position :0
2020-02-28 19:05:23.664 DEBUG com.sc.xyz.sshd.filesystem.FileRemotePathChannel - Setting the position for Write
2020-02-28 19:05:23.664 INFO com.sc.xyz.sshd.filesystem.FileRemotePathChannel - Writing buffer with size :java.nio.HeapByteBuffer[pos=57 lim=32761 cap=32825] to s3
2020-02-28 19:05:23.665 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:23.665 DEBUG org.apache.sshd.server.subsystem.sftp.SftpSubsystem - doSendStatus(ServerSessionImpl[user1@/127.0.0.1:52705])[id=117493766] SSH_FXP_STATUS (substatus=SSH_FX_OK, lang=, msg=)
2020-02-28 19:05:23.666 DEBUG org.apache.sshd.server.subsystem.sftp.SftpSubsystem - process(ServerSessionImpl[user1@/127.0.0.1:52705])[length=32757, type=SSH_FXP_WRITE, id=117494022] processing
2020-02-28 19:05:23.666 DEBUG com.sc.xyz.sshd.server.xyzSFTPEventListener - Session is Writing for offset :1073672320, dataoffset :57, and file size :32704
2020-02-28 19:05:23.666 DEBUG com.sc.xyz.sshd.filesystem.FileRemotePathChannel - Setting new position :0
2020-02-28 19:05:23.666 DEBUG com.sc.xyz.sshd.filesystem.FileRemotePathChannel - Setting the position for Write
2020-02-28 19:05:23.666 INFO com.sc.xyz.sshd.filesystem.FileRemotePathChannel - Writing buffer with size :java.nio.HeapByteBuffer[pos=57 lim=32761 cap=32825] to s3
2020-02-28 19:05:23.666 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:23.666 DEBUG org.apache.sshd.server.subsystem.sftp.SftpSubsystem - doSendStatus(ServerSessionImpl[user1@/127.0.0.1:52705])[id=117494022] SSH_FXP_STATUS (substatus=SSH_FX_OK, lang=, msg=)
2020-02-28 19:05:23.667 DEBUG org.apache.sshd.server.subsystem.sftp.SftpSubsystem - process(ServerSessionImpl[user1@/127.0.0.1:52705])[length=32757, type=SSH_FXP_WRITE, id=117494278] processing
2020-02-28 19:05:23.667 DEBUG com.sc.xyz.sshd.server.xyzSFTPEventListener - Session is Writing for offset :1073705024, dataoffset :57, and file size :32704
2020-02-28 19:05:23.667 DEBUG com.sc.xyz.sshd.filesystem.FileRemotePathChannel - Setting new position :0
2020-02-28 19:05:23.667 DEBUG com.sc.xyz.sshd.filesystem.FileRemotePathChannel - Setting the position for Write
2020-02-28 19:05:23.667 INFO com.sc.xyz.sshd.filesystem.FileRemotePathChannel - Writing buffer with size :java.nio.HeapByteBuffer[pos=57 lim=32761 cap=32825] to s3
2020-02-28 19:05:23.668 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:23.668 DEBUG org.apache.sshd.server.subsystem.sftp.SftpSubsystem - doSendStatus(ServerSessionImpl[user1@/127.0.0.1:52705])[id=117494278] SSH_FXP_STATUS (substatus=SSH_FX_OK, lang=, msg=)
2020-02-28 19:05:23.668 DEBUG org.apache.sshd.server.subsystem.sftp.SftpSubsystem - process(ServerSessionImpl[user1@/127.0.0.1:52705])[length=32757, type=SSH_FXP_WRITE, id=117494534] processing
2020-02-28 19:05:23.669 DEBUG com.sc.xyz.sshd.server.xyzSFTPEventListener - Session is Writing for offset :1073737728, dataoffset :57, and file size :32704
2020-02-28 19:05:23.669 DEBUG com.sc.xyz.sshd.filesystem.FileRemotePathChannel - Setting new position :0
2020-02-28 19:05:23.669 DEBUG com.sc.xyz.sshd.filesystem.FileRemotePathChannel - Setting the position for Write
2020-02-28 19:05:23.669 INFO com.sc.xyz.sshd.filesystem.FileRemotePathChannel - Writing buffer with size :java.nio.HeapByteBuffer[pos=57 lim=32761 cap=32825] to s3
2020-02-28 19:05:23.670 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:23.671 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:23.673 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:23.675 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:23.677 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:23.678 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:23.680 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:24.040 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:24.041 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:24.041 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:24.042 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:24.042 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:24.042 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:24.043 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:24.043 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=32761
2020-02-28 19:05:24.043 DEBUG org.apache.sshd.server.channel.ChannelSession - handleData(ChannelSession[id=0, recipient=256]-ServerSessionImpl[user1@/127.0.0.1:52705]) SSH_MSG_CHANNEL_DATA len=448
2020-02-28 19:05:24.040 DEBUG com.amazonaws.http.AmazonHttpClient - FYI: failed to reset content inputstream before throwing up
java.io.IOException: Resetting to invalid mark
at java.io.BufferedInputStream.reset(BufferedInputStream.java:448) ~[?:1.8.0_241]
at com.amazonaws.internal.SdkBufferedInputStream.reset(SdkBufferedInputStream.java:106) ~[aws-java-sdk-core-1.11.232.jar:?]
at com.amazonaws.internal.SdkFilterInputStream.reset(SdkFilterInputStream.java:112) ~[aws-java-sdk-core-1.11.232.jar:?]
at com.amazonaws.event.ProgressInputStream.reset(ProgressInputStream.java:168) ~[aws-java-sdk-core-1.11.232.jar:?]
at com.amazonaws.internal.SdkFilterInputStream.reset(SdkFilterInputStream.java:112) ~[aws-java-sdk-core-1.11.232.jar:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.lastReset(AmazonHttpClient.java:1145) [aws-java-sdk-core-1.11.232.jar:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1072) [aws-java-sdk-core-1.11.232.jar:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743) [aws-java-sdk-core-1.11.232.jar:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717) [aws-java-sdk-core-1.11.232.jar:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699) [aws-java-sdk-core-1.11.232.jar:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667) [aws-java-sdk-core-1.11.232.jar:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649) [aws-java-sdk-core-1.11.232.jar:?]
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513) [aws-java-sdk-core-1.11.232.jar:?]
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4319) [aws-java-sdk-s3-1.11.232.jar:?]
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4266) [aws-java-sdk-s3-1.11.232.jar:?]
at com.amazonaws.services.s3.AmazonS3Client.putObject(AmazonS3Client.java:1746) [aws-java-sdk-s3-1.11.232.jar:?]
at com.amazonaws.services.s3.AmazonS3Client.putObject(AmazonS3Client.java:1608) [aws-java-sdk-s3-1.11.232.jar:?]
at com.sc.xyz.sshd.component.AmazonS3Component.writeFileToBucket(AmazonS3Component.java:96) [classes/:?]
at com.sc.xyz.sshd.filesystem.FileRemotePathChannel$1.run(FileRemotePathChannel.java:126) [classes/:?]
2020-02-28 19:05:24.670 DEBUG org.apache.sshd.server.subsystem.sftp.SftpSubsystem - doSendStatus(ServerSessionImpl[user1@/127.0.0.1:52705])[id=117494534] SSH_FXP_STATUS (substatus=SSH_FX_FAILURE, lang=, msg=General failure)
2020-02-28 19:05:24.671 DEBUG org.apache.sshd.server.subsystem.sftp.SftpSubsystem - process(ServerSessionImpl[user1@/127.0.0.1:52705])[length=32757, type=SSH_FXP_WRITE, id=117494790] processing
2020-02-28 19:05:24.671 DEBUG com.sc.xyz.sshd.server.xyzSFTPEventListener - Session is Writing for offset :1073770432, dataoffset :57, and file size :32704
2020-02-28 19:05:24.671 DEBUG com.sc.xyz.sshd.filesystem.FileRemotePathChannel - Setting new position :0
2020-02-28 19:05:24.671 DEBUG com.sc.xyz.sshd.filesystem.FileRemotePathChannel - Setting the position for Write
2020-02-28 19:05:24.671 INFO com.sc.xyz.sshd.filesystem.FileRemotePathChannel - Writing buffer with size :java.nio.HeapByteBuffer[pos=57 lim=32761 cap=32825] to s3
То, что я пытался решить, это передача этого значения в java arguments - Dcom.amazonaws.sdk.s3.defaultStreamBufferSize=5368709120 но проблема все еще сохраняется.
Пожалуйста, дайте мне знать, если есть какие-либо идеи или решения.