JavaWeb/nas-service
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

替换rss解析器为xml解析

Browse Source
This commit is contained in:
zlzw 2024-06-09 13:37:29 +08:00
parent a366e15765
commit dcb56e8873
4 changed files with 101 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pom.xml
View File

@ -137,6 +137,11 @@
<artifactId>snail</artifactId> <artifactId>snail</artifactId>
<version>1.17.1</version> <version>1.17.1</version>
</dependency> </dependency>
<dependency>
<groupId>org.dom4j</groupId>
<artifactId>dom4j</artifactId>
<version>2.1.4</version>
</dependency>
</dependencies> </dependencies>

2
src/main/java/com/yutou/nas/NasApplication.java
View File

@ -12,7 +12,7 @@ import org.springframework.context.annotation.Import;
@Import(DmhyRssDownloadManager.class) @Import(DmhyRssDownloadManager.class)
@SpringBootApplication @SpringBootApplication
public class NasApplication { public class NasApplication {
public static final String version = "1.3"; public static final String version = "1.4";
public static void main(String[] args) { public static void main(String[] args) {
SpringApplication.run(NasApplication.class, args); SpringApplication.run(NasApplication.class, args);

48
src/main/java/com/yutou/nas/utils/HttpTools.java
View File

@ -4,10 +4,7 @@ import com.alibaba.fastjson2.JSONObject;
import com.yutou.nas.interfaces.DownloadInterface; import com.yutou.nas.interfaces.DownloadInterface;
import java.io.*; import java.io.*;
import java.net.HttpURLConnection; import java.net.*;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
@ -17,7 +14,11 @@ public class HttpTools {
private static final int HttpRequestIndex = 3; private static final int HttpRequestIndex = 3;
public static String get(String url) { public static String get(String url) {
return https_get(url, null); return https_get(url, false, null);
}
public static String https_get(String url, Map<String, String> header) {
return https_get(url, false, header);
} }
public static String post(final String url, final byte[] body) { public static String post(final String url, final byte[] body) {
@ -28,16 +29,26 @@ public class HttpTools {
return new HttpTools().http_syncDownload(url, saveName); return new HttpTools().http_syncDownload(url, saveName);
} }
public static String https_get(String url, Map<String, String> header) { public static String https_get(String url, boolean isProxy, Map<String, String> header) {
try { try {
URLConnection connection; URLConnection connection;
connection = new URL(url).openConnection(); Proxy proxy = null;
if (isProxy) {
proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(ConfigTools.load(ConfigTools.CONFIG,"proxy.address",String.class),
ConfigTools.load(ConfigTools.CONFIG,"proxy.port",Integer.class)));
}
if (proxy == null) {
connection = new URL(url).openConnection();
} else {
connection = new URL(url).openConnection(proxy);
}
connection.setRequestProperty("User-Agent", getUa()); connection.setRequestProperty("User-Agent", getUa());
if (header != null) { if (header != null) {
for (String key : header.keySet()) { for (String key : header.keySet()) {
connection.addRequestProperty(key, header.get(key)); connection.addRequestProperty(key, header.get(key));
} }
} }
connection.connect(); connection.connect();
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
StringBuilder str = new StringBuilder(); StringBuilder str = new StringBuilder();
@ -65,8 +76,8 @@ public class HttpTools {
connection.addRequestProperty(key, headers.get(key)); connection.addRequestProperty(key, headers.get(key));
} }
} }
if(body==null){ if (body == null) {
body="".getBytes(); body = "".getBytes();
} }
connection.setDoOutput(true); connection.setDoOutput(true);
connection.setDoInput(true); connection.setDoInput(true);
@ -101,6 +112,7 @@ public class HttpTools {
} }
} }
} }
public static String http_post_form(String url, byte[] body, int index, Map<String, String> headers) { public static String http_post_form(String url, byte[] body, int index, Map<String, String> headers) {
String tmp; String tmp;
StringBuilder str = new StringBuilder(); StringBuilder str = new StringBuilder();
@ -112,17 +124,17 @@ public class HttpTools {
connection.addRequestProperty(key, headers.get(key)); connection.addRequestProperty(key, headers.get(key));
} }
} }
if(body==null){ if (body == null) {
body="".getBytes(); body = "".getBytes();
} }
connection.setDoOutput(true); connection.setDoOutput(true);
connection.setDoInput(true); connection.setDoInput(true);
connection.addRequestProperty("User-Agent", getUa()); connection.addRequestProperty("User-Agent", getUa());
connection.setConnectTimeout(5 * 1000); connection.setConnectTimeout(5 * 1000);
connection.setReadTimeout(10 * 1000); connection.setReadTimeout(10 * 1000);
connection.setRequestProperty( "Content-Type", "multipart/form-data;"); connection.setRequestProperty("Content-Type", "multipart/form-data;");
connection.setRequestProperty( "charset", "utf-8"); connection.setRequestProperty("charset", "utf-8");
connection.setRequestProperty( "Accept-Encoding", "gzip, deflate, br"); connection.setRequestProperty("Accept-Encoding", "gzip, deflate, br");
//connection.addRequestProperty("Connection", "keep-alive"); //connection.addRequestProperty("Connection", "keep-alive");
//connection.addRequestProperty("User-Agent", getExtUa()); //connection.addRequestProperty("User-Agent", getExtUa());
@ -186,11 +198,11 @@ public class HttpTools {
json.put("pid", "102"); json.put("pid", "102");
json.put("gid", "100584"); json.put("gid", "100584");
json.put("message", "0gha58u1c9FjZkeAsEmYIzTvp"); json.put("message", "0gha58u1c9FjZkeAsEmYIzTvp");
// json.put("access_token", "659c-S1gV0DwMXdYjPDlSrSLNYOvA8qUoCSvmdFEHvZugKgNX4Z2BCwF18A7W2gRdG7WiWfKsbZgF6YssZHhaozksI9RBn2QQFTXzmAHtbMd4ginEEtwdKmPCM4JbJGg1ollqoNE0PcGENpa4F3e7EdSOa_JFyE6XyUQN1iurJU3F8MZfLlTIcTR9USYoHX15vsAkCht_0mrapZblkeY1_8HFrmK8rlenbZLxccy7PrMz5eZ9uPPDJL5OYiEahyrtLENB8SVmlGofJfQw8wUjN8_XVZSfLMujdwz24"); // json.put("access_token", "659c-S1gV0DwMXdYjPDlSrSLNYOvA8qUoCSvmdFEHvZugKgNX4Z2BCwF18A7W2gRdG7WiWfKsbZgF6YssZHhaozksI9RBn2QQFTXzmAHtbMd4ginEEtwdKmPCM4JbJGg1ollqoNE0PcGENpa4F3e7EdSOa_JFyE6XyUQN1iurJU3F8MZfLlTIcTR9USYoHX15vsAkCht_0mrapZblkeY1_8HFrmK8rlenbZLxccy7PrMz5eZ9uPPDJL5OYiEahyrtLENB8SVmlGofJfQw8wUjN8_XVZSfLMujdwz24");
String url = "http://127.0.0.1:8002/qq/send.do"; String url = "http://127.0.0.1:8002/qq/send.do";
String data="data=abc&codd=1"; String data = "data=abc&codd=1";
data=json.toString(); data = json.toString();
http_post(url,data.getBytes(StandardCharsets.UTF_8),0,null); http_post(url, data.getBytes(StandardCharsets.UTF_8), 0, null);
/* ExecutorService service= Executors.newCachedThreadPool(); /* ExecutorService service= Executors.newCachedThreadPool();
for (int i = 0; i < 3000; i++) { for (int i = 0; i < 3000; i++) {
service.submit(new Runnable() { service.submit(new Runnable() {

74
src/main/java/com/yutou/nas/utils/RssXMLtoJson.java
View File

@ -1,23 +1,79 @@
package com.yutou.nas.utils; package com.yutou.nas.utils;
import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject; import com.alibaba.fastjson2.JSONObject;
import com.jcraft.jsch.jce.ARCFOUR;
import org.dom4j.Document;
import org.dom4j.Element;
import org.dom4j.io.SAXReader;
import java.io.BufferedReader;
import java.io.StringReader;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.List;
public class RssXMLtoJson { public class RssXMLtoJson {
public static JSONObject toJSON(String xmlUrl){ public static JSONObject toJSON(String xmlUrl) {
String url = "https://api.rss2json.com/v1/api.json?rss_url=%s&api_key=wtfm5pebya13pnl8rtu51wfgfpte0mb9sap1foll&count=500"; String xml = HttpTools.https_get(xmlUrl, true, null);
SAXReader reader = new SAXReader();
JSONObject json = new JSONObject();
try { try {
String ret = HttpTools.get(String.format(url, URLEncoder.encode(xmlUrl, "UTF-8"))); json.put("status", "ok");
if(ret!=null){ JSONArray array = new JSONArray();
return JSON.parseObject(ret, JSONObject.class); BufferedReader br = new BufferedReader(new StringReader(xml));
Document document = reader.read(br);
Element rootElement = document.getRootElement();
List<Element> elements = rootElement.element("channel").elements("item");
for (Element element : elements) {
JSONObject item = new JSONObject();
JSONObject enclosure = new JSONObject();
String title = element.element("title").content().get(0).getText();
String magnet = element.element("enclosure").attribute("url").getValue();
String time = element.element("pubDate").content().get(0).getText();
enclosure.put("link", magnet);
item.put("title", title);
item.put("enclosure", enclosure);
item.put("time", time);
array.add(item);
} }
} catch (UnsupportedEncodingException e) { json.put("items", array);
e.printStackTrace(); } catch (Exception e) {
return null; json.put("status", e.getMessage());
}
return json;
}
public static void main(String[] args) {
String xmlUrl = "https://share.dmhy.org/topics/rss/rss.xml?keyword=%E5%96%B5%E8%90%8C%E5%A5%B6%E8%8C%B6%E5%B1%8B%20%E8%8D%AF%E5%B1%8B%E5%B0%91%E5%A5%B3%E7%9A%84%E5%91%A2%E5%96%83%20%E7%AE%80%E6%97%A5";
String xml = HttpTools.https_get(xmlUrl, true, null);
SAXReader reader = new SAXReader();
JSONObject json = new JSONObject();
try {
json.put("status", "ok");
JSONArray array = new JSONArray();
BufferedReader br = new BufferedReader(new StringReader(xml));
Document document = reader.read(br);
Element rootElement = document.getRootElement();
List<Element> elements = rootElement.element("channel").elements("item");
for (Element element : elements) {
JSONObject item = new JSONObject();
JSONObject enclosure = new JSONObject();
String title = element.element("title").content().get(0).getText();
String magnet = element.element("enclosure").attribute("url").getValue();
String time = element.element("pubDate").content().get(0).getText();
enclosure.put("link", magnet);
item.put("title", title);
item.put("enclosure", enclosure);
item.put("time", time);
array.add(item);
}
json.put("items", array);
System.out.println(rootElement);
} catch (Exception e) {
json.put("status", e.getMessage());
} }
return null;
} }
} }