valitydev/file-storage
14
0
Fork 0
mirror of https://github.com/valitydev/file-storage.git synced 2024-11-06 00:35:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

BJ-314: fix pu object InputStream

Browse Source
This commit is contained in:
a.karlov 2018-12-04 15:12:52 +03:00
parent e8de96b4c8
commit f3181d9b88
6 changed files with 97 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/main/java/com/rbkmoney/file/storage/service/AmazonS3StorageService.java
View File

@ -27,9 +27,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.time.Instant;
import java.util.*;
import java.util.stream.Collectors;
@ -121,19 +118,22 @@ public class AmazonS3StorageService implements StorageService {
public void uploadFile(String fileId, MultipartFile multipartFile) throws StorageException, IOException {
log.info("Trying to upload file to storage, filename='{}', bucketId='{}'", fileId, bucketName);
Path tempFile = Files.createTempFile("", "temp");
try {
Files.copy(multipartFile.getInputStream(), tempFile, StandardCopyOption.REPLACE_EXISTING);
S3Object object = getS3Object(fileId);
checkFileStatus(object);
object.getObjectMetadata().addUserMetadata(FILE_UPLOADED, "true");
object.getObjectMetadata().addUserMetadata(FILEDATA_MD_5, calculateMd5(tempFile));
ObjectMetadata objectMetadata = object.getObjectMetadata();
objectMetadata.addUserMetadata(FILE_UPLOADED, "true");
objectMetadata.addUserMetadata(FILEDATA_MD_5, calculateMd5(multipartFile.getBytes()));
objectMetadata.setContentLength(multipartFile.getSize());
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, fileId, tempFile.toFile());
PutObjectRequest putObjectRequest = new PutObjectRequest(
bucketName,
fileId,
multipartFile.getInputStream(),
objectMetadata
);
putObjectRequest.setMetadata(object.getObjectMetadata());
s3Client.putObject(putObjectRequest);
@ -151,8 +151,6 @@ public class AmazonS3StorageService implements StorageService {
),
ex
);
} finally {
Files.deleteIfExists(tempFile);
}
}
@ -354,7 +352,7 @@ public class AmazonS3StorageService implements StorageService {
}
}
private String calculateMd5(Path tempFile) throws IOException {
return DigestUtils.md5Hex(Files.newInputStream(tempFile));
private String calculateMd5(byte[] data) throws IOException {
return DigestUtils.md5Hex(data);
}
}

11
src/test/java/com/rbkmoney/file/storage/FileStorageApplicationTests.java
View File

@ -1,11 +0,0 @@
package com.rbkmoney.file.storage;
import org.junit.Test;
public class FileStorageApplicationTests extends AbstractIntegrationTest {
@Test
public void contextLoads() {
}
}

85
src/test/java/com/rbkmoney/file/storage/FileStorageTest.java
View File

@ -1,10 +1,13 @@
package com.rbkmoney.file.storage;
import com.rbkmoney.file.storage.service.StorageService;
import org.apache.thrift.TException;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.FileSystemResource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.client.RestTemplate;
@ -20,8 +23,12 @@ import java.util.Collections;
import static org.junit.Assert.assertEquals;
// все тесты в 1 классе , чтобы сэкономить время на поднятии тест контейнера
public class FileStorageTest extends AbstractIntegrationTest {
@Autowired
private StorageService storageService;
@Test
public void uploadAndDownloadFileFromStorageTest() throws IOException, TException {
Path testFile = Files.createTempFile("", "test_file");
@ -65,4 +72,82 @@ public class FileStorageTest extends AbstractIntegrationTest {
Files.deleteIfExists(testActualFile);
}
}
@Test
public void downloadUrlTest() throws TException, IOException {
Path testFile = Files.createTempFile("", "test_file");
Path testActualFile = Files.createTempFile("", "test_actual_file");
try {
Files.write(testFile, new byte[0]);
// создание нового файла
NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), getDayInstant().toString());
// генерация url с доступом только для загрузки
URL url = storageService.generateDownloadUrl(fileResult.getFileData().getFileId(), getDayInstant());
// с данной ссылкой нельзя записывать
assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, true, "PUT").getResponseCode());
// можно читать
assertEquals(HttpStatus.OK.value(), getHttpURLConnection(url, false, "GET").getResponseCode());
// чтение данных
HttpURLConnection urlConnection = getHttpURLConnection(url, false, "GET");
InputStream inputStream = urlConnection.getInputStream();
// чтение записанного файла из хранилища
Files.copy(inputStream, testActualFile, StandardCopyOption.REPLACE_EXISTING);
assertEquals(Files.readAllLines(testFile), Files.readAllLines(testActualFile));
} finally {
Files.deleteIfExists(testFile);
Files.deleteIfExists(testActualFile);
}
}
@Test(expected = FileNotFound.class)
public void expiredTimeForFileDataInMetadataTest() throws TException, InterruptedException {
NewFileResult testFile = client.createNewFile("test_file", Collections.emptyMap(), getSecondInstant().toString());
Thread.sleep(1000);
client.getFileData(testFile.getFileData().getFileId());
}
@Test(expected = FileNotFound.class)
public void expiredTimeForGenerateUrlInMetadataTest() throws TException, InterruptedException {
NewFileResult testFile = client.createNewFile("test_file", Collections.emptyMap(), getSecondInstant().toString());
Thread.sleep(1000);
client.generateDownloadUrl(testFile.getFileData().getFileId(), getSecondInstant().toString());
}
@Test
public void expiredTimeForGenerateUrlConnectionInCephTest() throws TException, IOException, InterruptedException {
NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), getDayInstant().toString());
URL url = storageService.generateDownloadUrl(fileResult.getFileData().getFileId(), getSecondInstant());
assertEquals(HttpStatus.OK.value(), getHttpURLConnection(url, false, "GET").getResponseCode());
Thread.sleep(2000);
assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, false, "GET").getResponseCode());
}
@Test
public void extractMetadataTest() throws TException {
String fileName = "test_file";
NewFileResult fileResult = client.createNewFile(fileName, Collections.emptyMap(), getDayInstant().toString());
assertEquals(fileResult.getFileData().getFileName(), fileName);
FileData fileData = storageService.getFileData(fileResult.getFileData().getFileId());
assertEquals(fileData, fileResult.getFileData());
}
}

53
src/test/java/com/rbkmoney/file/storage/handler/ExpirationTimeStorageObjectTest.java
View File

@ -1,53 +0,0 @@
package com.rbkmoney.file.storage.handler;
import com.rbkmoney.file.storage.AbstractIntegrationTest;
import com.rbkmoney.file.storage.FileNotFound;
import com.rbkmoney.file.storage.NewFileResult;
import com.rbkmoney.file.storage.service.StorageService;
import org.apache.thrift.TException;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import java.io.IOException;
import java.net.URL;
import java.util.Collections;
import static org.junit.Assert.assertEquals;
public class ExpirationTimeStorageObjectTest extends AbstractIntegrationTest {
@Autowired
private StorageService storageService;
@Test(expected = FileNotFound.class)
public void expiredTimeForFileDataInMetadataTest() throws TException, InterruptedException {
NewFileResult testFile = client.createNewFile("test_file", Collections.emptyMap(), getSecondInstant().toString());
Thread.sleep(1000);
client.getFileData(testFile.getFileData().getFileId());
}
@Test(expected = FileNotFound.class)
public void expiredTimeForGenerateUrlInMetadataTest() throws TException, InterruptedException {
NewFileResult testFile = client.createNewFile("test_file", Collections.emptyMap(), getSecondInstant().toString());
Thread.sleep(1000);
client.generateDownloadUrl(testFile.getFileData().getFileId(), getSecondInstant().toString());
}
@Test
public void expiredTimeForGenerateUrlConnectionInCephTest() throws TException, IOException, InterruptedException {
NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), getDayInstant().toString());
URL url = storageService.generateDownloadUrl(fileResult.getFileData().getFileId(), getSecondInstant());
assertEquals(HttpStatus.OK.value(), getHttpURLConnection(url, false, "GET").getResponseCode());
Thread.sleep(2000);
assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, false, "GET").getResponseCode());
}
}

31
src/test/java/com/rbkmoney/file/storage/service/MetadataStorageObjectTest.java
View File

@ -1,31 +0,0 @@
package com.rbkmoney.file.storage.service;
import com.rbkmoney.file.storage.AbstractIntegrationTest;
import com.rbkmoney.file.storage.FileData;
import com.rbkmoney.file.storage.NewFileResult;
import org.apache.thrift.TException;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import java.io.FileNotFoundException;
import java.util.Collections;
import static org.junit.Assert.assertEquals;
public class MetadataStorageObjectTest extends AbstractIntegrationTest {
@Autowired
private StorageService storageService;
@Test
public void extractMetadataTest() throws FileNotFoundException, TException {
String fileName = "test_file";
NewFileResult fileResult = client.createNewFile(fileName, Collections.emptyMap(), getDayInstant().toString());
assertEquals(fileResult.getFileData().getFileName(), fileName);
FileData fileData = storageService.getFileData(fileResult.getFileData().getFileId());
assertEquals(fileData, fileResult.getFileData());
}
}

60
src/test/java/com/rbkmoney/file/storage/service/PresignedUrlAccessRightsTest.java
View File

@ -1,60 +0,0 @@
package com.rbkmoney.file.storage.service;
import com.rbkmoney.file.storage.AbstractIntegrationTest;
import com.rbkmoney.file.storage.NewFileResult;
import org.apache.thrift.TException;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.Collections;
import static org.junit.Assert.assertEquals;
public class PresignedUrlAccessRightsTest extends AbstractIntegrationTest {
@Autowired
private StorageService storageService;
@Test
public void downloadUrlTest() throws TException, IOException {
Path testFile = Files.createTempFile("", "test_file");
Path testActualFile = Files.createTempFile("", "test_actual_file");
try {
Files.write(testFile, new byte[0]);
// создание нового файла
NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), getDayInstant().toString());
// генерация url с доступом только для загрузки
URL url = storageService.generateDownloadUrl(fileResult.getFileData().getFileId(), getDayInstant());
// с данной ссылкой нельзя записывать
assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, true, "PUT").getResponseCode());
// можно читать
assertEquals(HttpStatus.OK.value(), getHttpURLConnection(url, false, "GET").getResponseCode());
// чтение данных
HttpURLConnection urlConnection = getHttpURLConnection(url, false, "GET");
InputStream inputStream = urlConnection.getInputStream();
// чтение записанного файла из хранилища
Files.copy(inputStream, testActualFile, StandardCopyOption.REPLACE_EXISTING);
assertEquals(Files.readAllLines(testFile), Files.readAllLines(testActualFile));
} finally {
Files.deleteIfExists(testFile);
Files.deleteIfExists(testActualFile);
}
}
}