From 3df9b97e85f29ed4c065368bd67e4bfba687c4ad Mon Sep 17 00:00:00 2001 From: Yutousama <583819556@qq.com> Date: Sat, 19 Jun 2021 02:36:28 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8DOSS=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E9=81=87=E5=88=B0=E5=A4=A7=E6=96=87=E4=BB=B6(5G=E5=86=85)?= =?UTF-8?q?=E6=97=A0=E6=B3=95=E8=AE=A1=E7=AE=97MD5=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/yutou/nas/NasApplication.java | 2 +- .../com/yutou/nas/utils/DepotManager.java | 18 ++- .../java/com/yutou/nas/utils/OSSManager.java | 128 ++++++++++++++---- .../java/com/yutou/nas/utils/StreamTools.java | 18 +++ src/main/java/com/yutou/nas/utils/Tools.java | 10 +- 5 files changed, 144 insertions(+), 32 deletions(-) diff --git a/src/main/java/com/yutou/nas/NasApplication.java b/src/main/java/com/yutou/nas/NasApplication.java index 6d02f39..c4851dd 100644 --- a/src/main/java/com/yutou/nas/NasApplication.java +++ b/src/main/java/com/yutou/nas/NasApplication.java @@ -12,7 +12,7 @@ import org.springframework.context.annotation.Import; @Import(BTDownloadManager.class) @SpringBootApplication public class NasApplication { - public static final String version="1.1.4.2"; + public static final String version="1.1.4.10"; public static void main(String[] args) { SpringApplication.run(NasApplication.class, args); AppData.defaultMusicPath = (String) ConfigTools.load(ConfigTools.CONFIG, "musicDir"); diff --git a/src/main/java/com/yutou/nas/utils/DepotManager.java b/src/main/java/com/yutou/nas/utils/DepotManager.java index 1c7af40..4e91ee2 100644 --- a/src/main/java/com/yutou/nas/utils/DepotManager.java +++ b/src/main/java/com/yutou/nas/utils/DepotManager.java @@ -7,6 +7,8 @@ import org.springframework.util.StringUtils; import java.io.File; import java.io.FileNotFoundException; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; public class DepotManager { @@ -24,14 +26,16 @@ public class DepotManager { if (json.getInteger("code") == 1) { JSONArray array = json.getJSONArray("data"); for (Object o : array) { + List list = new ArrayList<>(); JSONObject item = JSONObject.parseObject(o.toString()); scanFile(new File(item.getString("path")), new DownloadInterface() { @Override public void onDownload(File file) { super.onDownload(file); - OSSManager.upload("oss-name-" + item.getString("type"), item.getString("path"), file); + list.add(file); } }); + OSSManager.upload("oss-name-" + item.getString("type"), item.getString("path"), list.toArray(new File[0])); } } } @@ -53,4 +57,16 @@ public class DepotManager { } } } + + public static void main(String[] args) { + List list = new ArrayList<>(); + scanFile(new File("Z:\\相机"), new DownloadInterface() { + @Override + public void onDownload(File file) { + super.onDownload(file); + list.add(file); + } + }); + System.out.println("list = "+list.size()); + } } diff --git a/src/main/java/com/yutou/nas/utils/OSSManager.java b/src/main/java/com/yutou/nas/utils/OSSManager.java index 08ba3fe..915ec57 100644 --- a/src/main/java/com/yutou/nas/utils/OSSManager.java +++ b/src/main/java/com/yutou/nas/utils/OSSManager.java @@ -2,6 +2,10 @@ package com.yutou.nas.utils; import com.aliyun.oss.OSS; import com.aliyun.oss.OSSClientBuilder; +import com.aliyun.oss.common.utils.BinaryUtil; +import com.aliyun.oss.event.ProgressEvent; +import com.aliyun.oss.event.ProgressEventType; +import com.aliyun.oss.event.ProgressListener; import com.aliyun.oss.model.OSSObject; import com.aliyun.oss.model.ObjectMetadata; import com.aliyun.oss.model.PutObjectRequest; @@ -10,12 +14,20 @@ import com.yutou.nas.interfaces.DownloadInterface; import java.io.File; import java.io.FileNotFoundException; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; +import java.util.List; public class OSSManager { public static final String TYPE_MUSIC = "oss-name-music"; public static final String TYPE_PHOTO = "oss-name-photo"; public static final String TYPE_DEPOT = "oss-name-depot"; + private static final List uploadList = new ArrayList<>(); private static OSS getOssClient() { return new OSSClientBuilder().build(ConfigTools.load(ConfigTools.CONFIG, "oss-url", String.class), @@ -29,17 +41,21 @@ public class OSSManager { public static void upload(String type, String path, File... files) { - boolean music=false,photo=false,depot=false; + if (uploadList.contains(path)) { + Log.i("OSS", "当前路径正在上传 = " + path + ", 文件数量 = " + files.length); + return; + } + boolean music = false, photo = false, depot = false; if (type.equals(TYPE_MUSIC) && !ConfigTools.load(ConfigTools.CONFIG, "oss-upload-music", boolean.class, false)) { - music=true; + music = true; } if (type.equals(TYPE_PHOTO) && !ConfigTools.load(ConfigTools.CONFIG, "oss-upload-photo", boolean.class, false)) { - photo=true; + photo = true; } if (type.equals(TYPE_DEPOT) && !ConfigTools.load(ConfigTools.CONFIG, "oss-upload-depot", boolean.class, false)) { - depot=true; + depot = true; } - if(music||photo||depot){ + if (music || photo || depot) { System.out.println("music = " + music); System.out.println("photo = " + photo); System.out.println("depot = " + depot); @@ -47,29 +63,64 @@ public class OSSManager { System.out.println("------------------------------"); return; } + Log.i("OSS", "上传文件数:" + files.length); new Thread(() -> { + uploadList.add(path); OSS client = getOssClient(); - for (File file : files) { - if (file.isDirectory()) { - continue; - } - String uploadPath = file.getAbsolutePath().replace(path, "").replace(File.separator, "/"); - if (uploadPath.startsWith("/")) { - uploadPath = uploadPath.substring(1); - } - if(isExist(ConfigTools.load(ConfigTools.CONFIG,type,String.class),uploadPath)){ - System.out.println("文件已存在:"+file.getName()); - return; + try { + for (File file : files) { + Log.i("OSS", "Next :" + file.getAbsolutePath()); + if (file.isDirectory()) { + continue; + } + String uploadPath = file.getAbsolutePath().replace(path, "").replace(File.separator, "/"); + if (uploadPath.startsWith("/")) { + uploadPath = uploadPath.substring(1); + } + String md5 = BinaryUtil.toBase64String(Tools.getFileMD5Byte(file)); + if (isExist(client, ConfigTools.load(ConfigTools.CONFIG, type, String.class), uploadPath, md5)) { + Log.i("OSS", "文件已存在:" + file.getName()); + continue; + } + Log.i("OSS", "即将上传:" + file.getName()); + //client.putObject(ConfigTools.load(ConfigTools.CONFIG,type,String.class), uploadPath, file); + int limitSpeed = 1024 * 1024 * 8; + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentMD5(md5); + PutObjectRequest request = new PutObjectRequest(ConfigTools.load(ConfigTools.CONFIG, type, String.class), uploadPath, file); + request.setTrafficLimit(limitSpeed); + request.setMetadata(metadata); + /*request.withProgressListener(new ProgressListener() { + private long bytesWritten = 0; + private long totalBytes = -1; + private boolean succeed = false; + @Override + public void progressChanged(ProgressEvent progressEvent) { + long bytes = progressEvent.getBytes(); + ProgressEventType eventType = progressEvent.getEventType(); + switch (eventType){ + case REQUEST_BYTE_TRANSFER_EVENT: + this.bytesWritten += bytes; + if (this.totalBytes != -1) { + int percent = (int)(this.bytesWritten * 100.0 / this.totalBytes); + System.out.println(file.getName()+" upload progress: " + percent + "%(" + this.bytesWritten + "/" + this.totalBytes + ")"); + } else { + System.out.println( "upload ratio: unknown" + "(" + this.bytesWritten + "/...)"); + } + break; + } + } + });*/ + client.putObject(request); + Log.i("OSS", file.getName() + "已上传"); } - //client.putObject(ConfigTools.load(ConfigTools.CONFIG,type,String.class), uploadPath, file); - int limitSpeed = 100 * 1024 * 8; - PutObjectRequest request=new PutObjectRequest(ConfigTools.load(ConfigTools.CONFIG,type,String.class),uploadPath,file); - request.setTrafficLimit(limitSpeed); - client.putObject(request); - System.out.println(file.getName()+"已上传"); + } catch (Exception e) { + e.printStackTrace(); } + Log.i("OSS", "上传完成:" + files.length); closeClient(client); + uploadList.remove(path); }).start(); } @@ -99,10 +150,35 @@ public class OSSManager { closeClient(oss); }).start(); } - private static boolean isExist(String type,String path){ - OSS oss=getOssClient(); - boolean flag=oss.doesObjectExist(type,path,true); - closeClient(oss); + private static boolean isOssExist(OSS oss, String type, String path){ + return oss.doesObjectExist(type, path, true); + } + public static boolean isExist(OSS oss, String type, String path, String md5) { + if (oss == null) { + return false; + } + boolean flag = oss.doesObjectExist(type, path, true); + if (flag) { + if (getFileMd5(oss, type, path).equals(md5)) { + return true; + } + } return flag; } + + private static String getFileMd5(OSS client, String type, String path) { + ObjectMetadata metadata = client.getObjectMetadata(type, path); + return metadata.getContentMD5(); + } + + public static void main(String[] args) { + //upload(TYPE_PHOTO, "Z:\\相机", new File[]{new File("Z:\\相机\\截图\\QQ截图20190630161109.png")}); + // getFileMd5(getOssClient(),TYPE_PHOTO,"截图/QQ截图20190630161109.png"); + File file=new File("Z:\\相机\\截图\\QQ截图20190630161109.png"); + file=new File("Z:\\相机\\北京\\MVI_0067.MOV"); +// String md5 = BinaryUtil.toBase64String(BinaryUtil.calculateMd5(StreamTools.fileToByte(new File("Z:\\相机\\北京\\MVI_0067.MOV")))); + String md5=BinaryUtil.toBase64String(Tools.getFileMD5Byte(file)); + System.out.println("md5 =" +md5); + System.out.println(getFileMd5(getOssClient(), ConfigTools.load(ConfigTools.CONFIG, TYPE_PHOTO, String.class),"截图/QQ截图20190630161109.png")); + } } diff --git a/src/main/java/com/yutou/nas/utils/StreamTools.java b/src/main/java/com/yutou/nas/utils/StreamTools.java index b9086ca..7f0b813 100644 --- a/src/main/java/com/yutou/nas/utils/StreamTools.java +++ b/src/main/java/com/yutou/nas/utils/StreamTools.java @@ -1,6 +1,8 @@ package com.yutou.nas.utils; import java.io.*; +import java.nio.ByteBuffer; +import java.security.MessageDigest; public class StreamTools { public static String streamReadLine(InputStream stream) { @@ -38,4 +40,20 @@ public class StreamTools { } return null; } + public static byte[] fileToByte(File file){ + byte[] buffer = new byte[1024]; + ByteArrayOutputStream outputStream=new ByteArrayOutputStream(); + int len; + try { + FileInputStream in = new FileInputStream(file); + while ((len = in.read(buffer, 0, 1024)) != -1) { + outputStream.write(buffer,0,len); + } + in.close(); + } catch (Exception e) { + e.printStackTrace(); + return null; + } + return outputStream.toByteArray(); + } } diff --git a/src/main/java/com/yutou/nas/utils/Tools.java b/src/main/java/com/yutou/nas/utils/Tools.java index 44a9285..0066bb1 100644 --- a/src/main/java/com/yutou/nas/utils/Tools.java +++ b/src/main/java/com/yutou/nas/utils/Tools.java @@ -310,14 +310,16 @@ public class Tools { headers.add("ETag", String.valueOf(System.currentTimeMillis())); return ResponseEntity.ok().headers(headers).contentLength(file.length()).contentType(MediaType.parseMediaType("application/octet-stream")).body(new FileSystemResource(file)); } - - public static String getFileMD5(File file) { + public static String getFileMD5(File file){ + return bytesToHexString(getFileMD5Byte(file)); + } + public static byte[] getFileMD5Byte(File file) { if (!file.isFile()) { return null; } MessageDigest digest = null; FileInputStream in = null; - byte buffer[] = new byte[1024]; + byte[] buffer = new byte[1024]; int len; try { digest = MessageDigest.getInstance("MD5"); @@ -330,7 +332,7 @@ public class Tools { e.printStackTrace(); return null; } - return bytesToHexString(digest.digest()); + return digest.digest(); } private static String bytesToHexString(byte[] src) {