Merge remote-tracking branch 'origin/main'
This commit is contained in:
@@ -2,7 +2,7 @@
|
|||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
<packaging>jar</packaging>
|
<packaging>jar</packaging>
|
||||||
<groupId>com.budwk</groupId>
|
<groupId>com.budwk</groupId>
|
||||||
<artifactId>zhgh-whcp</artifactId>
|
<artifactId>zhgh_hdc</artifactId>
|
||||||
<version>5.6.0-plus</version>
|
<version>5.6.0-plus</version>
|
||||||
<properties>
|
<properties>
|
||||||
<nutzboot.version>2.6.0-SNAPSHOT</nutzboot.version>
|
<nutzboot.version>2.6.0-SNAPSHOT</nutzboot.version>
|
||||||
@@ -26,6 +26,19 @@
|
|||||||
</properties>
|
</properties>
|
||||||
<dependencies>
|
<dependencies>
|
||||||
|
|
||||||
|
<!-- HtmlUnit 核心依赖 -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>net.sourceforge.htmlunit</groupId>
|
||||||
|
<artifactId>htmlunit</artifactId>
|
||||||
|
<version>2.70.0</version>
|
||||||
|
</dependency>
|
||||||
|
<!-- 可选:解决部分 JS 兼容性问题 -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>net.sourceforge.htmlunit</groupId>
|
||||||
|
<artifactId>htmlunit-core-js</artifactId>
|
||||||
|
<version>2.70.0</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.github.javen205</groupId>
|
<groupId>com.github.javen205</groupId>
|
||||||
<artifactId>IJPay-All</artifactId>
|
<artifactId>IJPay-All</artifactId>
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ import com.budwk.app.sys.services.SysMenuService;
|
|||||||
import com.budwk.app.sys.services.SysUserService;
|
import com.budwk.app.sys.services.SysUserService;
|
||||||
import com.budwk.app.web.commons.auth.utils.SecurityUtil;
|
import com.budwk.app.web.commons.auth.utils.SecurityUtil;
|
||||||
import com.budwk.app.zhgh.activity.basic.models.ActivityUserScope;
|
import com.budwk.app.zhgh.activity.basic.models.ActivityUserScope;
|
||||||
|
import com.gargoylesoftware.htmlunit.WebClient;
|
||||||
|
import com.gargoylesoftware.htmlunit.html.HtmlPage;
|
||||||
import io.swagger.annotations.ApiOperation;
|
import io.swagger.annotations.ApiOperation;
|
||||||
import org.jsoup.Jsoup;
|
import org.jsoup.Jsoup;
|
||||||
import org.jsoup.nodes.Document;
|
import org.jsoup.nodes.Document;
|
||||||
@@ -264,71 +266,46 @@ public class SysHomeController {
|
|||||||
// @CacheResult(cacheKey = "news", ignoreNull = true, cacheLiveTime = 60 * 60 * 24)
|
// @CacheResult(cacheKey = "news", ignoreNull = true, cacheLiveTime = 60 * 60 * 24)
|
||||||
public Result getNews() {
|
public Result getNews() {
|
||||||
// 定义要爬取的URL
|
// 定义要爬取的URL
|
||||||
String domain = "https://gh.whcp.edu.cn";
|
String domain = "https://dept.hdc.edu.cn/gh";
|
||||||
try {
|
try (WebClient webClient = new WebClient()) {
|
||||||
// 使用Jsoup连接到URL并获取页面内容
|
// 关键配置:启用 JavaScript 执行(核心)
|
||||||
Document doc = Jsoup.connect(domain).get();
|
webClient.getOptions().setJavaScriptEnabled(true);
|
||||||
|
// 禁用 CSS 解析(节省资源,无需渲染样式)
|
||||||
|
webClient.getOptions().setCssEnabled(false);
|
||||||
|
// 忽略 JS 执行错误(避免页面个别 JS 报错导致程序中断)
|
||||||
|
webClient.getOptions().setThrowExceptionOnScriptError(false);
|
||||||
|
// 忽略页面加载异常
|
||||||
|
webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
|
||||||
|
// 设置 JS 执行超时时间(根据页面调整)
|
||||||
|
webClient.getOptions().setTimeout(10000);
|
||||||
|
// 设置页面加载超时时间
|
||||||
|
webClient.getOptions().setTimeout(15000);
|
||||||
|
|
||||||
|
// 2. 加载页面并执行 JS(核心步骤:自动渲染 DOM)
|
||||||
|
HtmlPage page = webClient.getPage(domain);
|
||||||
|
// 等待 JS 执行完成(确保动态内容加载)
|
||||||
|
webClient.waitForBackgroundJavaScript(3000);
|
||||||
|
|
||||||
|
// 3. 获取渲染后的完整 HTML 源码
|
||||||
|
String renderedHtml = page.asXml(); // 或 page.asHtml()
|
||||||
|
|
||||||
|
// 4. 用 Jsoup 解析最终的 DOM 节点
|
||||||
|
Document doc = Jsoup.parse(renderedHtml);
|
||||||
|
|
||||||
JSONObject root = new JSONObject();
|
JSONObject root = new JSONObject();
|
||||||
|
|
||||||
// 1. 新闻快讯(来自 .list1)
|
|
||||||
/* List<JSONObject> newsArray = new ArrayList<>();
|
|
||||||
Elements newsItems = doc.select(".ml .main_page2 .news_list li");
|
|
||||||
for (Element item : newsItems) {
|
|
||||||
String date = item.select(".date").text().trim();
|
|
||||||
Element link = item.select("a.text").first();
|
|
||||||
String title = link.select(".title").text().trim();
|
|
||||||
String url = link.attr("href").trim();
|
|
||||||
|
|
||||||
JSONObject node = new JSONObject();
|
|
||||||
node.set("title", title);
|
|
||||||
node.set("date", date);
|
|
||||||
node.set("url", domain + url);
|
|
||||||
newsArray.add(node);
|
|
||||||
}
|
|
||||||
root.set("news", newsArray);*/
|
|
||||||
|
|
||||||
// 2. 通知公告(来自 .list2)
|
|
||||||
List<JSONObject> noticesArray = new ArrayList<>();
|
List<JSONObject> noticesArray = new ArrayList<>();
|
||||||
Elements noticeItems = doc.select(".tu_text .news_list li");
|
Elements noticeItems = doc.select(".ul_show a");
|
||||||
for (Element item : noticeItems) {
|
for (Element item : noticeItems) {
|
||||||
String date = item.select(".news_meta").text().trim();
|
String date = item.select("p").text().trim();
|
||||||
Element link = item.select(".news_title a").first();
|
String title = item.select("span").text().trim();
|
||||||
String title = link.text().trim();
|
String url = item.attr("href").trim();
|
||||||
String url = link.attr("href").trim();
|
|
||||||
JSONObject node = new JSONObject();
|
JSONObject node = new JSONObject();
|
||||||
node.set("title", title);
|
node.set("title", title);
|
||||||
node.set("date", date);
|
node.set("date", date);
|
||||||
node.set("url", domain + url);
|
node.set("url", "https://dept.hdc.edu.cn" + url);
|
||||||
noticesArray.add(node);
|
noticesArray.add(node);
|
||||||
}
|
}
|
||||||
root.set("notices", noticesArray);
|
root.set("notices", noticesArray);
|
||||||
|
|
||||||
// 3. 基层风采(来自 .pic-list1)
|
|
||||||
/* List<JSONObject> grassrootsArray = new ArrayList<>();
|
|
||||||
Elements grassrootsItems = doc.select(".group3 .pic-list1 li");
|
|
||||||
for (Element item : grassrootsItems) {
|
|
||||||
Element link = item.select("a.img-scale").first();
|
|
||||||
String date = link.select(".date").text().trim();
|
|
||||||
String title = link.select(".title").text().trim();
|
|
||||||
String url = link.attr("href").trim();
|
|
||||||
String summary = link.select(".info").text().trim();
|
|
||||||
|
|
||||||
String imageUrl = "";
|
|
||||||
Element img = link.select("img").first();
|
|
||||||
if (img != null) {
|
|
||||||
imageUrl = img.attr("src").trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
JSONObject node = new JSONObject();
|
|
||||||
node.set("title", title);
|
|
||||||
node.set("date", date);
|
|
||||||
node.set("url", domain + url);
|
|
||||||
node.set("summary", summary);
|
|
||||||
node.set("image", domain + imageUrl);
|
|
||||||
grassrootsArray.add(node);
|
|
||||||
}
|
|
||||||
root.set("grassroots", grassrootsArray);*/
|
|
||||||
return Result.success(root);
|
return Result.success(root);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error(e);
|
log.error(e);
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
nutz:
|
nutz:
|
||||||
application:
|
application:
|
||||||
name: zhgh_whcp_new
|
name: zhgh_hdc
|
||||||
mvc:
|
mvc:
|
||||||
ignore: ^(.+[.])(jsp|png|gif|jpg|js|css|jspx|jpeg|html|mp3|mp4|ico|svg|vue)$
|
ignore: ^(.+[.])(jsp|png|gif|jpg|js|css|jspx|jpeg|html|mp3|mp4|ico|svg|vue)$
|
||||||
exclusions: /favicon/*,/assets/*,/druid/*,/upload/*,/components/*
|
exclusions: /favicon/*,/assets/*,/druid/*,/upload/*,/components/*
|
||||||
server:
|
server:
|
||||||
port: 8081
|
port: 80
|
||||||
host: 0.0.0.0
|
host: 0.0.0.0
|
||||||
|
|
||||||
jetty:
|
jetty:
|
||||||
@@ -21,7 +21,7 @@ jetty:
|
|||||||
#结合ftp使用,或用nginx代理ftp路径
|
#结合ftp使用,或用nginx代理ftp路径
|
||||||
#staticPath: /Users/wizzer/temp/files
|
#staticPath: /Users/wizzer/temp/files
|
||||||
#开发模式静态资源
|
#开发模式静态资源
|
||||||
staticPathLocal: D:/myProject/zhgh_whcp_new/src/main/resources/static
|
staticPathLocal: D:/project/zhgh_hdc/src/main/resources/static
|
||||||
|
|
||||||
security:
|
security:
|
||||||
tokenName: saToken
|
tokenName: saToken
|
||||||
@@ -86,7 +86,7 @@ xsssql:
|
|||||||
beetl:
|
beetl:
|
||||||
RESOURCE:
|
RESOURCE:
|
||||||
#本地路径,开发时设置本地路径,便于调试
|
#本地路径,开发时设置本地路径,便于调试
|
||||||
rootLocal: D:/myProject/zhgh_whcp_new/src/main/resources/views/
|
rootLocal: D:/project/zhgh_hdc/src/main/resources/views/
|
||||||
root: views/
|
root: views/
|
||||||
DELIMITER_STATEMENT_START: "<!--#"
|
DELIMITER_STATEMENT_START: "<!--#"
|
||||||
DELIMITER_STATEMENT_END: "#-->"
|
DELIMITER_STATEMENT_END: "#-->"
|
||||||
|
|||||||
Reference in New Issue
Block a user