修复OSS上传遇到大文件(5G内)无法计算MD5的问题

src/main/java/com/yutou/nas/NasApplication.java

@@ -12,7 +12,7 @@ import org.springframework.context.annotation.Import;
 @Import(BTDownloadManager.class)
 @SpringBootApplication
 public class NasApplication {
-    public static final String version="1.1.4.2";
+    public static final String version="1.1.4.10";
     public static void main(String[] args) {
         SpringApplication.run(NasApplication.class, args);
         AppData.defaultMusicPath = (String) ConfigTools.load(ConfigTools.CONFIG, "musicDir");

src/main/java/com/yutou/nas/utils/DepotManager.java

@@ -7,6 +7,8 @@ import org.springframework.util.StringUtils;
 
 import java.io.File;
 import java.io.FileNotFoundException;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Objects;
 
 public class DepotManager {
@@ -24,14 +26,16 @@ public class DepotManager {
                     if (json.getInteger("code") == 1) {
                         JSONArray array = json.getJSONArray("data");
                         for (Object o : array) {
+                            List<File> list = new ArrayList<>();
                             JSONObject item = JSONObject.parseObject(o.toString());
                             scanFile(new File(item.getString("path")), new DownloadInterface() {
                                 @Override
                                 public void onDownload(File file) {
                                     super.onDownload(file);
-                                    OSSManager.upload("oss-name-" + item.getString("type"), item.getString("path"), file);
+                                    list.add(file);
                                 }
                             });
+                            OSSManager.upload("oss-name-" + item.getString("type"), item.getString("path"), list.toArray(new File[0]));
                         }
                     }
                 }
@@ -53,4 +57,16 @@ public class DepotManager {
             }
         }
     }
+
+    public static void main(String[] args) {
+        List<File> list = new ArrayList<>();
+        scanFile(new File("Z:\\相机"), new DownloadInterface() {
+            @Override
+            public void onDownload(File file) {
+                super.onDownload(file);
+                list.add(file);
+            }
+        });
+        System.out.println("list = "+list.size());
+    }
 }

src/main/java/com/yutou/nas/utils/OSSManager.java

@@ -2,6 +2,10 @@ package com.yutou.nas.utils;
 
 import com.aliyun.oss.OSS;
 import com.aliyun.oss.OSSClientBuilder;
+import com.aliyun.oss.common.utils.BinaryUtil;
+import com.aliyun.oss.event.ProgressEvent;
+import com.aliyun.oss.event.ProgressEventType;
+import com.aliyun.oss.event.ProgressListener;
 import com.aliyun.oss.model.OSSObject;
 import com.aliyun.oss.model.ObjectMetadata;
 import com.aliyun.oss.model.PutObjectRequest;
@@ -10,12 +14,20 @@ import com.yutou.nas.interfaces.DownloadInterface;
 
 import java.io.File;
 import java.io.FileNotFoundException;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Base64;
+import java.util.List;
 
 public class OSSManager {
     public static final String TYPE_MUSIC = "oss-name-music";
     public static final String TYPE_PHOTO = "oss-name-photo";
     public static final String TYPE_DEPOT = "oss-name-depot";
+    private static final List<String> uploadList = new ArrayList<>();
 
     private static OSS getOssClient() {
         return new OSSClientBuilder().build(ConfigTools.load(ConfigTools.CONFIG, "oss-url", String.class),
@@ -29,17 +41,21 @@ public class OSSManager {
 
 
     public static void upload(String type, String path, File... files) {
-        boolean music=false,photo=false,depot=false;
+        if (uploadList.contains(path)) {
+            Log.i("OSS", "当前路径正在上传 = " + path + ", 文件数量 = " + files.length);
+            return;
+        }
+        boolean music = false, photo = false, depot = false;
         if (type.equals(TYPE_MUSIC) && !ConfigTools.load(ConfigTools.CONFIG, "oss-upload-music", boolean.class, false)) {
-            music=true;
+            music = true;
         }
         if (type.equals(TYPE_PHOTO) && !ConfigTools.load(ConfigTools.CONFIG, "oss-upload-photo", boolean.class, false)) {
-            photo=true;
+            photo = true;
         }
         if (type.equals(TYPE_DEPOT) && !ConfigTools.load(ConfigTools.CONFIG, "oss-upload-depot", boolean.class, false)) {
-            depot=true;
+            depot = true;
         }
-        if(music||photo||depot){
+        if (music || photo || depot) {
             System.out.println("music = " + music);
             System.out.println("photo = " + photo);
             System.out.println("depot = " + depot);
@@ -47,29 +63,64 @@ public class OSSManager {
             System.out.println("------------------------------");
             return;
         }
+        Log.i("OSS", "上传文件数:" + files.length);
         new Thread(() -> {
+            uploadList.add(path);
             OSS client = getOssClient();
-            for (File file : files) {
+            try {
-                if (file.isDirectory()) {
+                for (File file : files) {
-                    continue;
+                    Log.i("OSS", "Next :" + file.getAbsolutePath());
-                }
+                    if (file.isDirectory()) {
-                String uploadPath = file.getAbsolutePath().replace(path, "").replace(File.separator, "/");
+                        continue;
-                if (uploadPath.startsWith("/")) {
+                    }
-                    uploadPath = uploadPath.substring(1);
+                    String uploadPath = file.getAbsolutePath().replace(path, "").replace(File.separator, "/");
-                }
+                    if (uploadPath.startsWith("/")) {
-                if(isExist(ConfigTools.load(ConfigTools.CONFIG,type,String.class),uploadPath)){
+                        uploadPath = uploadPath.substring(1);
-                    System.out.println("文件已存在:"+file.getName());
+                    }
-                    return;
+                    String md5 = BinaryUtil.toBase64String(Tools.getFileMD5Byte(file));
+                    if (isExist(client, ConfigTools.load(ConfigTools.CONFIG, type, String.class), uploadPath, md5)) {
+                        Log.i("OSS", "文件已存在:" + file.getName());
+                        continue;
+                    }
+                    Log.i("OSS", "即将上传:" + file.getName());
+                    //client.putObject(ConfigTools.load(ConfigTools.CONFIG,type,String.class), uploadPath, file);
+                    int limitSpeed = 1024 * 1024 * 8;
+                    ObjectMetadata metadata = new ObjectMetadata();
+                    metadata.setContentMD5(md5);
+                    PutObjectRequest request = new PutObjectRequest(ConfigTools.load(ConfigTools.CONFIG, type, String.class), uploadPath, file);
+                    request.setTrafficLimit(limitSpeed);
+                    request.setMetadata(metadata);
+                    /*request.withProgressListener(new ProgressListener() {
+                        private long bytesWritten = 0;
+                        private long totalBytes = -1;
+                        private boolean succeed = false;
+                        @Override
+                        public void progressChanged(ProgressEvent progressEvent) {
+                            long bytes = progressEvent.getBytes();
+                            ProgressEventType eventType = progressEvent.getEventType();
+                            switch (eventType){
+                                case REQUEST_BYTE_TRANSFER_EVENT:
+                                    this.bytesWritten += bytes;
+                                    if (this.totalBytes != -1) {
+                                        int percent = (int)(this.bytesWritten * 100.0 / this.totalBytes);
+                                        System.out.println(file.getName()+" upload progress: " + percent + "%(" + this.bytesWritten + "/" + this.totalBytes + ")");
+                                    } else {
+                                        System.out.println( "upload ratio: unknown" + "(" + this.bytesWritten + "/...)");
+                                    }
+                                    break;
+                            }
+                        }
+                    });*/
+                    client.putObject(request);
+                    Log.i("OSS", file.getName() + "已上传");
                 }
 
-                //client.putObject(ConfigTools.load(ConfigTools.CONFIG,type,String.class), uploadPath, file);
+            } catch (Exception e) {
-                int limitSpeed = 100 * 1024 * 8;
+                e.printStackTrace();
-                PutObjectRequest request=new PutObjectRequest(ConfigTools.load(ConfigTools.CONFIG,type,String.class),uploadPath,file);
-                request.setTrafficLimit(limitSpeed);
-                client.putObject(request);
-                System.out.println(file.getName()+"已上传");
             }
+            Log.i("OSS", "上传完成:" + files.length);
             closeClient(client);
+            uploadList.remove(path);
         }).start();
 
     }
@@ -99,10 +150,35 @@ public class OSSManager {
             closeClient(oss);
         }).start();
     }
-    private static boolean isExist(String type,String path){
+    private static boolean isOssExist(OSS oss, String type, String path){
-        OSS oss=getOssClient();
+        return  oss.doesObjectExist(type, path, true);
-        boolean flag=oss.doesObjectExist(type,path,true);
+    }
-        closeClient(oss);
+    public static boolean isExist(OSS oss, String type, String path, String md5) {
+        if (oss == null) {
+            return false;
+        }
+        boolean flag = oss.doesObjectExist(type, path, true);
+        if (flag) {
+            if (getFileMd5(oss, type, path).equals(md5)) {
+                return true;
+            }
+        }
         return flag;
     }
+
+    private static String getFileMd5(OSS client, String type, String path) {
+        ObjectMetadata metadata = client.getObjectMetadata(type, path);
+        return metadata.getContentMD5();
+    }
+
+    public static void main(String[] args) {
+        //upload(TYPE_PHOTO, "Z:\\相机", new File[]{new File("Z:\\相机\\截图\\QQ截图20190630161109.png")});
+        // getFileMd5(getOssClient(),TYPE_PHOTO,"截图/QQ截图20190630161109.png");
+        File file=new File("Z:\\相机\\截图\\QQ截图20190630161109.png");
+        file=new File("Z:\\相机\\北京\\MVI_0067.MOV");
+//        String md5 = BinaryUtil.toBase64String(BinaryUtil.calculateMd5(StreamTools.fileToByte(new File("Z:\\相机\\北京\\MVI_0067.MOV"))));
+       String  md5=BinaryUtil.toBase64String(Tools.getFileMD5Byte(file));
+        System.out.println("md5 =" +md5);
+        System.out.println(getFileMd5(getOssClient(), ConfigTools.load(ConfigTools.CONFIG, TYPE_PHOTO, String.class),"截图/QQ截图20190630161109.png"));
+    }
 }

src/main/java/com/yutou/nas/utils/StreamTools.java

@@ -1,6 +1,8 @@
 package com.yutou.nas.utils;
 
 import java.io.*;
+import java.nio.ByteBuffer;
+import java.security.MessageDigest;
 
 public class StreamTools {
     public static String streamReadLine(InputStream stream) {
@@ -38,4 +40,20 @@ public class StreamTools {
         }
         return null;
     }
+    public static byte[] fileToByte(File file){
+        byte[] buffer = new byte[1024];
+        ByteArrayOutputStream outputStream=new ByteArrayOutputStream();
+        int len;
+        try {
+            FileInputStream in = new FileInputStream(file);
+            while ((len = in.read(buffer, 0, 1024)) != -1) {
+                outputStream.write(buffer,0,len);
+            }
+            in.close();
+        } catch (Exception e) {
+            e.printStackTrace();
+            return null;
+        }
+        return outputStream.toByteArray();
+    }
 }

src/main/java/com/yutou/nas/utils/Tools.java

@@ -310,14 +310,16 @@ public class Tools {
         headers.add("ETag", String.valueOf(System.currentTimeMillis()));
         return ResponseEntity.ok().headers(headers).contentLength(file.length()).contentType(MediaType.parseMediaType("application/octet-stream")).body(new FileSystemResource(file));
     }
-
+    public static String getFileMD5(File file){
-    public static String getFileMD5(File file) {
+        return bytesToHexString(getFileMD5Byte(file));
+    }
+    public static byte[] getFileMD5Byte(File file) {
         if (!file.isFile()) {
             return null;
         }
         MessageDigest digest = null;
         FileInputStream in = null;
-        byte buffer[] = new byte[1024];
+        byte[] buffer = new byte[1024];
         int len;
         try {
             digest = MessageDigest.getInstance("MD5");
@@ -330,7 +332,7 @@ public class Tools {
             e.printStackTrace();
             return null;
         }
-        return bytesToHexString(digest.digest());
+        return digest.digest();
     }
 
     private static String bytesToHexString(byte[] src) {