From 81b0a4c50e952e7629eedb662e609545573983ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=8C=E5=85=88=E8=88=9C?= <2499071264@qq.com> Date: Tue, 10 Nov 2026 15:11:46 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BA=BA=E5=91=98=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=B2=BB=E7=90=86=E8=BF=81=E7=A7=BB5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 +- .../impl/SysDataUnitPullServiceImpl.java | 77 ++--- .../impl/SysDataUserPullServiceImpl.java | 263 +++++------------- 3 files changed, 108 insertions(+), 235 deletions(-) diff --git a/.gitignore b/.gitignore index de71ec4..6d7bb81 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ src/main/resources/application.yaml src/main/resources/application-dev.yaml .vscode bin -node-data-center-mock/ \ No newline at end of file +node-data-center-mock/ +.workbuddy \ No newline at end of file diff --git a/src/main/java/com/budwk/app/sys/services/impl/SysDataUnitPullServiceImpl.java b/src/main/java/com/budwk/app/sys/services/impl/SysDataUnitPullServiceImpl.java index 6fb1cb8..b823e44 100644 --- a/src/main/java/com/budwk/app/sys/services/impl/SysDataUnitPullServiceImpl.java +++ b/src/main/java/com/budwk/app/sys/services/impl/SysDataUnitPullServiceImpl.java @@ -1,9 +1,9 @@ package com.budwk.app.sys.services.impl; import cn.hutool.core.util.StrUtil; -import cn.hutool.core.util.URLUtil; import cn.hutool.http.HttpRequest; import cn.hutool.http.HttpUtil; +import cn.hutool.json.JSONArray; import cn.hutool.json.JSONObject; import cn.hutool.json.JSONUtil; import com.budwk.app.base.config.DataCenterProperties; @@ -13,16 +13,19 @@ import com.budwk.app.sys.enums.Api2UnitFiledMap; import com.budwk.app.sys.models.Sys_unit; import com.budwk.app.sys.services.SysDataUnitPullService; import com.budwk.app.sys.utils.DataCenterUtil; +import com.budwk.app.web.commons.base.Globals; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.nutz.aop.interceptor.ioc.TransAop; +import org.nutz.dao.Cnd; import org.nutz.dao.Dao; import org.nutz.dao.Sqls; +import org.nutz.dao.entity.Record; import org.nutz.dao.sql.Sql; import org.nutz.ioc.aop.Aop; import org.nutz.ioc.loader.annotation.Inject; import org.nutz.ioc.loader.annotation.IocBean; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -40,6 +43,10 @@ import java.util.stream.Collectors; @IocBean(args = {"refer:dao"}) public class SysDataUnitPullServiceImpl extends BaseServiceImpl implements SysDataUnitPullService { + private static final String URL = "http://api.jsahvc.edu.cn/getREST_XXZZJG_c515102514c74b4e913f01e05ecb0286"; + private static final String APPLY_ID = "2001030048_1778744549088"; + private static final String SECRET_KEY = "9ec51871-ee77-47ce-b981-d1c29d3477fa"; + @Inject private DataCenterProperties dcPro; @@ -64,54 +71,52 @@ public class SysDataUnitPullServiceImpl extends BaseServiceImpl implem @Override @Aop(TransAop.READ_COMMITTED) public void updateUnits(List unitIds) { - // token url参数 - Map tokenParams = Map.of( - "grant_type", "password", - "scope", "read", - "username", "3848A3C738EFBB2C2E76E0D59B419DACA16A80F3AB0354CD", - "password", "3848A3C738EFBB2C2E76E0D59B419DACA16A80F3AB0354CD" - ); - - String params = URLUtil.buildQuery(tokenParams, StandardCharsets.UTF_8); - String tokenUrl = "https://bd.whcp.edu.cn/sjjkfwpt/api/bd-api/oauth/token" + (StrUtil.isBlank(params) ? "" : "?" + params); - HttpRequest tokenHttpRequest = HttpUtil.createPost(tokenUrl); - tokenHttpRequest.header("Content-Type", "application/json"); - tokenHttpRequest.header("Authorization", "Basic MTE0OjU5M2UwNGI4YTE4NWRlMzkyNjY4ZmY0ZDJhZGMyZTEw"); - - log.info("请求token: {}", tokenHttpRequest); - String tokenResBody = tokenHttpRequest.execute().body(); - - log.info("请求token结果: {}", tokenResBody); - JSONObject tokenJsonBody = JSONUtil.parseObj(tokenResBody); // 请求数据 - HttpRequest httpRequest = HttpUtil.createGet("https://bd.whcp.edu.cn/sjjkfwpt/api/bd-api/get/YXSDWJBSJZLB"); - httpRequest.form(Map.of("access_token", tokenJsonBody.getStr("access_token"), "pageSize", 100, "page", 1)); - log.info("请求单位数据: {}", httpRequest); - String resBody = httpRequest.execute().body(); - log.info("请求单位数据结果: {}", resBody); - JSONObject jsonBody = JSONUtil.parseObj(resBody); + HttpRequest httpRequest = HttpUtil.createPost(URL); + httpRequest.header("Content-Type", "application/json"); + httpRequest.header("applyId", APPLY_ID); + httpRequest.header("secretKey", SECRET_KEY); + httpRequest.body(JSONUtil.toJsonStr(Map.of("page", 1, "pagesize", 1000))); - if (jsonBody.getInt("code") != 200) { - throw new BaseException("获取单位数据失败，错误码: " + jsonBody.getInt("code") + "；错误原因：" + jsonBody.getStr("msg")); + String resBody = httpRequest.execute().body(); + log.info("请求数据: {}", resBody); + JSONObject result = JSONUtil.parseObj(resBody); + + if (!"successful".equals(result.getStr("msg"))) { + throw new BaseException("获取人员数据失败，错误码: " + result.getInt("status") + "；错误原因：" + result.getStr("msg")); } // 需要新增的单位 List insertList = new ArrayList<>(); // 需要更新的单位 List updateList = new ArrayList<>(); - - // 拉取过来的数据，全部的单位，一万四千多条，这里只保留 “部门类” - List data = jsonBody.getJSONArray("data").stream() - .map(o -> (JSONObject) o) - .toList(); + JSONObject resultData = result.getJSONObject("data"); + JSONArray dataArray = resultData.getJSONArray("Rows"); + List data = dataArray.stream().map(v -> JSONUtil.parseObj(JSONUtil.toJsonStr(v))).toList(); Map api2db = Arrays.stream(Api2UnitFiledMap.values()) .collect(Collectors.toMap(e -> e.apiField, e -> e.dbColumn)); for (JSONObject row : data) { - Sys_unit unit = DataCenterUtil.mapJsonToBean(row, Sys_unit.class, api2db); + JSONObject jsonRow = JSONUtil.parseObj(JSONUtil.toJsonStr(row)); + Sys_unit unit = DataCenterUtil.mapJsonToBean(jsonRow, Sys_unit.class, api2db); + if("100".equals(unit.getParentId())) { + unit.setUnitcode("0"); + unit.setParentId("0"); + unit.setUnitLevel(1); + unit.setUnitType("根部门"); + unit.setUnitTypeCode(0); + } else if(StringUtils.isBlank(unit.getParentId())) { + unit.setParentId("0"); + unit.setUnitLevel(2); + unit.setUnitType("子部门"); + unit.setUnitTypeCode(0); + } else { + unit.setUnitLevel(3); + unit.setUnitType("子部门"); + unit.setUnitTypeCode(1); + } unit.setUnitcode(unit.getId()); - unit.setUnitTypeCode(1); if (unitIds.contains(unit.getId())) { // 存在则更新 diff --git a/src/main/java/com/budwk/app/sys/services/impl/SysDataUserPullServiceImpl.java b/src/main/java/com/budwk/app/sys/services/impl/SysDataUserPullServiceImpl.java index f2d38c8..46a3b7e 100644 --- a/src/main/java/com/budwk/app/sys/services/impl/SysDataUserPullServiceImpl.java +++ b/src/main/java/com/budwk/app/sys/services/impl/SysDataUserPullServiceImpl.java @@ -6,7 +6,6 @@ import cn.hutool.core.date.DateUtil; import cn.hutool.core.util.CharsetUtil; import cn.hutool.core.util.IdcardUtil; import cn.hutool.core.util.StrUtil; -import cn.hutool.core.util.URLUtil; import cn.hutool.crypto.digest.DigestUtil; import cn.hutool.http.Header; import cn.hutool.http.HttpRequest; @@ -33,6 +32,7 @@ import org.nutz.aop.interceptor.ioc.TransAop; import org.nutz.dao.Cnd; import org.nutz.dao.Dao; import org.nutz.dao.Sqls; +import org.nutz.dao.entity.Record; import org.nutz.dao.sql.Sql; import org.nutz.ioc.aop.Aop; import org.nutz.ioc.loader.annotation.Inject; @@ -42,7 +42,6 @@ import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import java.lang.reflect.Field; import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; import java.util.*; import java.util.stream.Collectors; @@ -50,6 +49,10 @@ import java.util.stream.Collectors; @Slf4j public class SysDataUserPullServiceImpl extends BaseServiceImpl implements SysDataUserPullService { + private static final String URL = "http://api.jsahvc.edu.cn/getREST_JZGJBXXOABG_529a0bf78f574e10a4de5c6acbb48410"; + private static final String APPLY_ID = "2001030048_1778744549088"; + private static final String SECRET_KEY = "9ec51871-ee77-47ce-b981-d1c29d3477fa"; + @Inject private SysDictService sysDictService; @Inject @@ -110,59 +113,47 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl @Aop(TransAop.READ_COMMITTED) public Date pull() { try { - // 获取数据中心的字典 - List dataDictList = dao().query(Sys_data_dict.class, Cnd.NEW()); - // 字典码表Map，key为父级编码，value为子级字典列表 - Map> dataDictMap = dataDictList.stream().collect(Collectors.groupingBy(Sys_data_dict::getParentCode)); - - // token url参数 - Map tokenParams = Map.of( - "grant_type", "password", - "scope", "read", - "username", "3848A3C738EFBB2C2E76E0D59B419DACA16A80F3AB0354CD", - "password", "3848A3C738EFBB2C2E76E0D59B419DACA16A80F3AB0354CD" - ); - - String params = URLUtil.buildQuery(tokenParams, StandardCharsets.UTF_8); - String tokenUrl = "https://bd.whcp.edu.cn/sjjkfwpt/api/bd-api/oauth/token" + (StrUtil.isBlank(params) ? "" : "?" + params); - HttpRequest tokenHttpRequest = HttpUtil.createPost(tokenUrl); - tokenHttpRequest.header("Content-Type", "application/json"); - tokenHttpRequest.header("Authorization", "Basic MTE0OjU5M2UwNGI4YTE4NWRlMzkyNjY4ZmY0ZDJhZGMyZTEw"); - - log.info("请求token: {}", tokenHttpRequest); - String tokenResBody = tokenHttpRequest.execute().body(); - log.info("请求token结果: {}", tokenResBody); - - JSONObject tokenJsonBody = JSONUtil.parseObj(tokenResBody); - - // 请求数据 - HttpRequest httpRequest = HttpUtil.createGet("https://bd.whcp.edu.cn/sjjkfwpt/api/bd-api/get/JZGJCSJZLB"); - httpRequest.form(Map.of("access_token", tokenJsonBody.getStr("access_token"), "pageSize", 5000, "page", 1)); - - log.info("请求人员数据: {}", httpRequest); - String resBody = httpRequest.execute().body(); - log.info("请求人员数据结果: {}", resBody); - JSONObject jsonBody = JSONUtil.parseObj(resBody); - - if (jsonBody.getInt("code") != 200) { - throw new BaseException("获取人员数据失败，错误码: " + jsonBody.getInt("code") + "；错误原因：" + jsonBody.getStr("msg")); - } - List rawDataList = new ArrayList<>(); - JSONArray data = jsonBody.getJSONArray("data"); + int page = 1; + int total; + final Integer pageSize = 1000; + do { + // 请求数据 + HttpRequest httpRequest = HttpUtil.createPost(URL); + httpRequest.header("Content-Type", "application/json"); + httpRequest.header("applyId", APPLY_ID); + httpRequest.header("secretKey", SECRET_KEY); + String jsonStr = JSONUtil.toJsonStr(Map.of("page", page, "pagesize", pageSize)); - if (!data.isEmpty()) { - // 先收集所有原始数据 - rawDataList = data.stream().map(v -> (JSONObject) v).toList(); - log.info("数据拉取进度: {}/{}", rawDataList.size(), jsonBody.getInt("total")); - } else { - log.warn("当前未获取到数据"); - } + httpRequest.body(jsonStr); + + String resBody = httpRequest.execute().body(); + log.info("请求数据: {}", resBody); + JSONObject result = JSONUtil.parseObj(resBody); + + if (!"successful".equals(result.getStr("msg"))) { + throw new BaseException("获取人员数据失败，错误码: " + result.getInt("status") + "；错误原因：" + result.getStr("msg")); + } + + JSONObject resultData = result.getJSONObject("data"); + total = result.getInt("total"); + JSONArray data = resultData.getJSONArray("Rows"); + + if (!data.isEmpty()) { + // 先收集所有原始数据 + rawDataList.addAll(data.stream().map(v -> JSONUtil.parseObj(JSONUtil.toJsonStr(v))).toList()); + log.info("数据拉取进度: 总数据量：{}，已经拉取{}，本次拉取：{}", total, rawDataList.size(), data.size()); + } else { + log.warn("当前未获取到数据"); + } + + page ++; + } while (page <= Math.ceil(total / (double) pageSize)); Date nowDate = new Date(); // 赋值 - List latestSourceList = rawDataList.stream().map(raw -> { + List latestSourceList = rawDataList.stream().filter(e -> e.getStr("BH").length() == 10).map(raw -> { Sys_user_source sysUser = new Sys_user_source(); // 使用缓存的字段映射 @@ -170,25 +161,9 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl try { // 根据字段类型设置值 if (mapping.field.getType() == String.class) { - - if (StrUtil.isNotBlank(mapping.dict)) { - Sys_data_dict sysDataDict = dataDictMap.get(mapping.dict).stream() - .filter(v -> raw.getStr(mapping.key) - .equals(v.getCode())).findFirst().orElse(new Sys_data_dict()); - mapping.field.set(sysUser, sysDataDict.getName()); - } else { - // 特殊处理 根据身份证号获取性别和出生年月 - if ("SFZJH".equals(mapping.field.getName())) { - String idCard = raw.getStr(mapping.key); - // 设置出生年月 - try { - sysUser.setBirthday(IdcardUtil.getBirthDate(idCard)); - } catch (Exception e) { - sysUser.setBirthday(null); - } - } else { - mapping.field.set(sysUser, raw.getStr(mapping.key)); - } + mapping.field.set(sysUser, raw.getStr(mapping.key)); + if("XB".equals(mapping.key) && StrUtil.isNotBlank(raw.getStr(mapping.key))) { + mapping.field.set(sysUser, raw.getStr(mapping.key).replace("性", "")); } } else if (mapping.field.getType() == Date.class) { mapping.field.set(sysUser, raw.getDate(mapping.key)); @@ -198,15 +173,15 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl } } - // 设置拉取时间 sysUser.setPullTime(nowDate); - return sysUser; }).toList(); log.info("用户数据初始化完成，等待插入，当前数据条数{}", latestSourceList.size()); // 插入到数据库 dao().insert(latestSourceList); + // 更新字典数据 + this.updateDict(latestSourceList); // 人员的单位数据和数据库的单位数据比较，如果人员里面有单位不存在，去更新单位数据 List sourceUnitIds = latestSourceList.stream().map(Sys_user_source::getUnitId).distinct().toList(); @@ -221,10 +196,16 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl } return nowDate; } catch (Exception e) { - throw new BaseException("数据拉取失败,{}", e.getMessage()); + log.error("数据拉取失败,{}", e.getMessage()); + throw new BaseException("数据拉取失败"); } } + @Override + public Map pullPostDoctoral() { + return null; + } + @Async private void updateDict(List userSources) { //判断是否要更新在职状态字典 @@ -234,9 +215,7 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl //判断是否要更新学位字典 List sourceAcademicDegrees = userSources.stream().map(Sys_user::getAcademicDegree).filter(StrUtil::isNotBlank).distinct().toList(); //判断是否要更新教职工类别字典 - List personTypes = userSources.stream().map(Sys_user::getPersonType).filter(StrUtil::isNotBlank).distinct().toList(); - //判断是否要更新身份类型字典 - List sourceIdentityTypes = userSources.stream().map(Sys_user::getIdentityType).filter(StrUtil::isNotBlank).distinct().toList(); + List sourcePersonTypes = userSources.stream().map(Sys_user::getPersonType).filter(StrUtil::isNotBlank).distinct().toList(); //系统在职状态字典 List sysUserStates = sysDictService.getSubListByCode("USER_STATE"); @@ -244,10 +223,7 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl List sysEducations = sysDictService.getSubListByCode("USER_EDUCATION"); //系统学位字典 List sysAcademicDegrees = sysDictService.getSubListByCode("USER_ACADEMIC_DEGREE"); - //系统教职工类别字典 - List sysStaffTypes = sysDictService.getSubListByCode("USER_STAFF_TYPE"); - //系统身份类型字典 - List sysIdentityTypes = sysDictService.getSubListByCode("USER_IDENTITY_TYPE"); + List sysPersonTypes = sysDictService.getSubListByCode("USER_PERSON_TYPE"); //系统中不存在的就插入 List insertUserStates = sourceUserStates.stream().filter(state -> sysUserStates.stream().noneMatch(dict -> dict.getCode().equals(state))).map(state -> { @@ -271,6 +247,13 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl return dict; }).toList(); + List insertPersonTypes = sourcePersonTypes.stream().filter(personType -> sysPersonTypes.stream().noneMatch(dict -> dict.getCode().equals(personType))).map(personType -> { + Sys_dict dict = new Sys_dict(); + dict.setCode(personType); + dict.setName(personType); + return dict; + }).toList(); + for (Sys_dict dict : insertUserStates) { sysDictService.saveByParentCode(dict, "USER_STATE"); } @@ -280,63 +263,12 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl for (Sys_dict dict : insertAcademicDegrees) { sysDictService.saveByParentCode(dict, "USER_ACADEMIC_DEGREE"); } + for (Sys_dict dict : insertPersonTypes) { + sysDictService.saveByParentCode(dict, "USER_PERSON_TYPE"); + } sysDictService.clearCache(); } - - @Override - public Map pullPostDoctoral() { - String appId = "1926887238437818370"; - String secret = "32df31fcf4fc43f9a647ee1f47134f36"; - - List allUsers = new ArrayList<>(); - int skip = 0; - int totalCount = 0; - boolean firstRequest = true; - - do { - long ts = System.currentTimeMillis(); - String sign = Base64.encode(DigestUtil.md5Hex(appId + secret + ts, CharsetUtil.CHARSET_UTF_8)); - - HttpRequest httpRequest = HttpUtil.createPost("https://sjzcpt.nnu.edu.cn/cdsp/data-api/v2/DS0040"); - httpRequest.header("Content-Type", "application/json"); - httpRequest.header("appId", appId); - httpRequest.header("timestamp", String.valueOf(ts)); - httpRequest.header("sign", sign); - - HashMap reqBody = new HashMap<>(); - reqBody.put("$count", true); - reqBody.put("$skip", skip); - reqBody.put("$top", 1000); - - httpRequest.body(JSONUtil.toJsonStr(reqBody)); - String resBody = httpRequest.execute().body(); - - JSONObject entries = JSONUtil.parseObj(resBody); - - if (firstRequest) { - totalCount = entries.getInt("@odata.count", 0); - firstRequest = false; - } - - JSONArray value = entries.getJSONArray("value"); - - if (!value.isEmpty()) { - List list = value.stream().map(item -> (JSONObject) item).toList(); - allUsers.addAll(list); - skip += list.size(); - } else { - break; - } - - } while (skip < totalCount); - - Map data = allUsers.stream().filter(v -> v.getStr("进站日期") != null).collect(Collectors.toMap(v -> v.getStr("教职工号"), v -> v.getDate("进站日期"))); - - return data; - - } - @Override public NutMap searchOptions() { Sql sql = Sqls.create(""" @@ -385,73 +317,8 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl return listMap(sql); } - @Override public Map pullFinance() { - // 获取配置文件 接口地址 和 token - DataCenterProperties.Credential credential = dataCenterProperties.credential("finance"); - String url = credential.getUrl(); - String token = credential.getToken(); - - List rawDataList = new ArrayList<>(); - String year = String.valueOf(DateUtil.thisYear()); - String month = String.valueOf(DateUtil.thisMonth() + 1); - - int pageNum = 1; - int pageSize = 1000; - int totalPages = 1; - int totalCount = 0; - do { - HttpRequest httpRequest = HttpUtil.createPost(url); - httpRequest.header(Header.CONTENT_TYPE, "application/json"); - httpRequest.header("Accept-Encoding", "gzip, deflate, br"); - httpRequest.header("X-H3C-TOKEN", token); - Map reqBody = Map.of( - "nf", year, - "yf", month, - "pageNum", pageNum, - "pageSize", pageSize - ); - httpRequest.body(JSONUtil.toJsonStr(reqBody)); - - JSONObject resp = JSONUtil.parseObj(httpRequest.execute().body()); - if (resp.getInt("code") != 0) { - throw new BaseException("获取财务数据失败，错误码: " + resp.getInt("code") + "；错误原因：" + resp.getStr("msg")); - } - - JSONObject data = resp.getJSONObject("data"); - if (pageNum == 1) { - totalCount = data.getInt("total", 0); -// totalPages = data.getInt("pages", 0); - totalPages = (int) Math.ceil((double) totalCount / pageSize); - } - - JSONArray records = data.getJSONArray("records"); - if (CollUtil.isNotEmpty(records)) { - rawDataList.addAll(records.stream().map(v -> (JSONObject) v).toList()); - log.info("数据拉取进度: {}/{}", rawDataList.size(), totalCount); - } - pageNum++; // 只加页码 - } while (pageNum <= totalPages); - - // 接口数据输出 - log.info("数据拉取结果: {}", rawDataList); - - // 最后结果数据 - Map result = new HashMap<>(); - - Map api2db = Arrays.stream(Api2FinanceFiledMap.values()) - .collect(Collectors.toMap(e -> e.apiField, e -> e.dbColumn)); - - for (JSONObject row : rawDataList) { - NutMap nutMap = DataCenterUtil.mapJsonToBean(row, NutMap.class, api2db); - result.put(nutMap.getString("loginname"), nutMap); - } - - // 输出结果 - log.info("财务数据拉取转换结果: {}", result); - - log.info("数据拉取完成，共拉取 {} 条数据", result.size()); - return result; + return null; } }