JavaWeb/nas-service
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复OSS上传遇到大文件(5G内)无法计算MD5的问题

Browse Source
This commit is contained in:
Yutousama 2021-06-19 02:36:28 +08:00
parent 3fcf1e72cc
commit 3df9b97e85
5 changed files with 144 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/com/yutou/nas/NasApplication.java
View File

@ -12,7 +12,7 @@ import org.springframework.context.annotation.Import;
@Import(BTDownloadManager.class)
@SpringBootApplication
public class NasApplication {
public static final String version="1.1.4.2";
public static final String version="1.1.4.10";
public static void main(String[] args) {
SpringApplication.run(NasApplication.class, args);
AppData.defaultMusicPath = (String) ConfigTools.load(ConfigTools.CONFIG, "musicDir");

18
src/main/java/com/yutou/nas/utils/DepotManager.java
View File

@ -7,6 +7,8 @@ import org.springframework.util.StringUtils;
import java.io.File;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
public class DepotManager {
@ -24,14 +26,16 @@ public class DepotManager {
if (json.getInteger("code") == 1) {
JSONArray array = json.getJSONArray("data");
for (Object o : array) {
List<File> list = new ArrayList<>();
JSONObject item = JSONObject.parseObject(o.toString());
scanFile(new File(item.getString("path")), new DownloadInterface() {
@Override
public void onDownload(File file) {
super.onDownload(file);
OSSManager.upload("oss-name-" + item.getString("type"), item.getString("path"), file);
list.add(file);
}
});
OSSManager.upload("oss-name-" + item.getString("type"), item.getString("path"), list.toArray(new File[0]));
}
}
}
@ -53,4 +57,16 @@ public class DepotManager {
}
}
}
public static void main(String[] args) {
List<File> list = new ArrayList<>();
scanFile(new File("Z:\\相机"), new DownloadInterface() {
@Override
public void onDownload(File file) {
super.onDownload(file);
list.add(file);
}
});
System.out.println("list = "+list.size());
}
}

96
src/main/java/com/yutou/nas/utils/OSSManager.java
View File

@ -2,6 +2,10 @@ package com.yutou.nas.utils;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSClientBuilder;
import com.aliyun.oss.common.utils.BinaryUtil;
import com.aliyun.oss.event.ProgressEvent;
import com.aliyun.oss.event.ProgressEventType;
import com.aliyun.oss.event.ProgressListener;
import com.aliyun.oss.model.OSSObject;
import com.aliyun.oss.model.ObjectMetadata;
import com.aliyun.oss.model.PutObjectRequest;
@ -10,12 +14,20 @@ import com.yutou.nas.interfaces.DownloadInterface;
import java.io.File;
import java.io.FileNotFoundException;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.List;
public class OSSManager {
public static final String TYPE_MUSIC = "oss-name-music";
public static final String TYPE_PHOTO = "oss-name-photo";
public static final String TYPE_DEPOT = "oss-name-depot";
private static final List<String> uploadList = new ArrayList<>();
private static OSS getOssClient() {
return new OSSClientBuilder().build(ConfigTools.load(ConfigTools.CONFIG, "oss-url", String.class),
@ -29,6 +41,10 @@ public class OSSManager {
public static void upload(String type, String path, File... files) {
if (uploadList.contains(path)) {
Log.i("OSS", "当前路径正在上传 = " + path + ", 文件数量 = " + files.length);
return;
}
boolean music = false, photo = false, depot = false;
if (type.equals(TYPE_MUSIC) && !ConfigTools.load(ConfigTools.CONFIG, "oss-upload-music", boolean.class, false)) {
music = true;
@ -47,9 +63,13 @@ public class OSSManager {
System.out.println("------------------------------");
return;
}
Log.i("OSS", "上传文件数:" + files.length);
new Thread(() -> {
uploadList.add(path);
OSS client = getOssClient();
try {
for (File file : files) {
Log.i("OSS", "Next :" + file.getAbsolutePath());
if (file.isDirectory()) {
continue;
}
@ -57,19 +77,50 @@ public class OSSManager {
if (uploadPath.startsWith("/")) {
uploadPath = uploadPath.substring(1);
}
if(isExist(ConfigTools.load(ConfigTools.CONFIG,type,String.class),uploadPath)){
System.out.println("文件已存在:"+file.getName());
return;
String md5 = BinaryUtil.toBase64String(Tools.getFileMD5Byte(file));
if (isExist(client, ConfigTools.load(ConfigTools.CONFIG, type, String.class), uploadPath, md5)) {
Log.i("OSS", "文件已存在:" + file.getName());
continue;
}
Log.i("OSS", "即将上传:" + file.getName());
//client.putObject(ConfigTools.load(ConfigTools.CONFIG,type,String.class), uploadPath, file);
int limitSpeed = 100 * 1024 * 8;
int limitSpeed = 1024 * 1024 * 8;
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentMD5(md5);
PutObjectRequest request = new PutObjectRequest(ConfigTools.load(ConfigTools.CONFIG, type, String.class), uploadPath, file);
request.setTrafficLimit(limitSpeed);
client.putObject(request);
System.out.println(file.getName()+"已上传");
request.setMetadata(metadata);
/*request.withProgressListener(new ProgressListener() {
private long bytesWritten = 0;
private long totalBytes = -1;
private boolean succeed = false;
@Override
public void progressChanged(ProgressEvent progressEvent) {
long bytes = progressEvent.getBytes();
ProgressEventType eventType = progressEvent.getEventType();
switch (eventType){
case REQUEST_BYTE_TRANSFER_EVENT:
this.bytesWritten += bytes;
if (this.totalBytes != -1) {
int percent = (int)(this.bytesWritten * 100.0 / this.totalBytes);
System.out.println(file.getName()+" upload progress: " + percent + "%(" + this.bytesWritten + "/" + this.totalBytes + ")");
} else {
System.out.println( "upload ratio: unknown" + "(" + this.bytesWritten + "/...)");
}
break;
}
}
});*/
client.putObject(request);
Log.i("OSS", file.getName() + "已上传");
}
} catch (Exception e) {
e.printStackTrace();
}
Log.i("OSS", "上传完成:" + files.length);
closeClient(client);
uploadList.remove(path);
}).start();
}
@ -99,10 +150,35 @@ public class OSSManager {
closeClient(oss);
}).start();
}
private static boolean isExist(String type,String path){
OSS oss=getOssClient();
private static boolean isOssExist(OSS oss, String type, String path){
return oss.doesObjectExist(type, path, true);
}
public static boolean isExist(OSS oss, String type, String path, String md5) {
if (oss == null) {
return false;
}
boolean flag = oss.doesObjectExist(type, path, true);
closeClient(oss);
if (flag) {
if (getFileMd5(oss, type, path).equals(md5)) {
return true;
}
}
return flag;
}
private static String getFileMd5(OSS client, String type, String path) {
ObjectMetadata metadata = client.getObjectMetadata(type, path);
return metadata.getContentMD5();
}
public static void main(String[] args) {
//upload(TYPE_PHOTO, "Z:\\相机", new File[]{new File("Z:\\相机\\截图\\QQ截图20190630161109.png")});
// getFileMd5(getOssClient(),TYPE_PHOTO,"截图/QQ截图20190630161109.png");
File file=new File("Z:\\相机\\截图\\QQ截图20190630161109.png");
file=new File("Z:\\相机\\北京\\MVI_0067.MOV");
// String md5 = BinaryUtil.toBase64String(BinaryUtil.calculateMd5(StreamTools.fileToByte(new File("Z:\\相机\\北京\\MVI_0067.MOV"))));
String md5=BinaryUtil.toBase64String(Tools.getFileMD5Byte(file));
System.out.println("md5 =" +md5);
System.out.println(getFileMd5(getOssClient(), ConfigTools.load(ConfigTools.CONFIG, TYPE_PHOTO, String.class),"截图/QQ截图20190630161109.png"));
}
}

18
src/main/java/com/yutou/nas/utils/StreamTools.java
View File

@ -1,6 +1,8 @@
package com.yutou.nas.utils;
import java.io.*;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
public class StreamTools {
public static String streamReadLine(InputStream stream) {
@ -38,4 +40,20 @@ public class StreamTools {
}
return null;
}
public static byte[] fileToByte(File file){
byte[] buffer = new byte[1024];
ByteArrayOutputStream outputStream=new ByteArrayOutputStream();
int len;
try {
FileInputStream in = new FileInputStream(file);
while ((len = in.read(buffer, 0, 1024)) != -1) {
outputStream.write(buffer,0,len);
}
in.close();
} catch (Exception e) {
e.printStackTrace();
return null;
}
return outputStream.toByteArray();
}
}

8
src/main/java/com/yutou/nas/utils/Tools.java
View File

@ -310,14 +310,16 @@ public class Tools {
headers.add("ETag", String.valueOf(System.currentTimeMillis()));
return ResponseEntity.ok().headers(headers).contentLength(file.length()).contentType(MediaType.parseMediaType("application/octet-stream")).body(new FileSystemResource(file));
}
public static String getFileMD5(File file){
return bytesToHexString(getFileMD5Byte(file));
}
public static byte[] getFileMD5Byte(File file) {
if (!file.isFile()) {
return null;
}
MessageDigest digest = null;
FileInputStream in = null;
byte buffer[] = new byte[1024];
byte[] buffer = new byte[1024];
int len;
try {
digest = MessageDigest.getInstance("MD5");
@ -330,7 +332,7 @@ public class Tools {
e.printStackTrace();
return null;
}
return bytesToHexString(digest.digest());
return digest.digest();
}
private static String bytesToHexString(byte[] src) {