fix: 人员数据治理迁移5

This commit is contained in:
2026-11-10 15:11:46 +08:00
parent 0da369366a
commit 81b0a4c50e
3 changed files with 108 additions and 235 deletions
+2 -1
View File
@@ -29,4 +29,5 @@ src/main/resources/application.yaml
src/main/resources/application-dev.yaml
.vscode
bin
node-data-center-mock/
node-data-center-mock/
.workbuddy
@@ -1,9 +1,9 @@
package com.budwk.app.sys.services.impl;
import cn.hutool.core.util.StrUtil;
import cn.hutool.core.util.URLUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.budwk.app.base.config.DataCenterProperties;
@@ -13,16 +13,19 @@ import com.budwk.app.sys.enums.Api2UnitFiledMap;
import com.budwk.app.sys.models.Sys_unit;
import com.budwk.app.sys.services.SysDataUnitPullService;
import com.budwk.app.sys.utils.DataCenterUtil;
import com.budwk.app.web.commons.base.Globals;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.nutz.aop.interceptor.ioc.TransAop;
import org.nutz.dao.Cnd;
import org.nutz.dao.Dao;
import org.nutz.dao.Sqls;
import org.nutz.dao.entity.Record;
import org.nutz.dao.sql.Sql;
import org.nutz.ioc.aop.Aop;
import org.nutz.ioc.loader.annotation.Inject;
import org.nutz.ioc.loader.annotation.IocBean;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -40,6 +43,10 @@ import java.util.stream.Collectors;
@IocBean(args = {"refer:dao"})
public class SysDataUnitPullServiceImpl extends BaseServiceImpl<Sys_unit> implements SysDataUnitPullService {
private static final String URL = "http://api.jsahvc.edu.cn/getREST_XXZZJG_c515102514c74b4e913f01e05ecb0286";
private static final String APPLY_ID = "2001030048_1778744549088";
private static final String SECRET_KEY = "9ec51871-ee77-47ce-b981-d1c29d3477fa";
@Inject
private DataCenterProperties dcPro;
@@ -64,54 +71,52 @@ public class SysDataUnitPullServiceImpl extends BaseServiceImpl<Sys_unit> implem
@Override
@Aop(TransAop.READ_COMMITTED)
public void updateUnits(List<String> unitIds) {
// token url参数
Map<String, String> tokenParams = Map.of(
"grant_type", "password",
"scope", "read",
"username", "3848A3C738EFBB2C2E76E0D59B419DACA16A80F3AB0354CD",
"password", "3848A3C738EFBB2C2E76E0D59B419DACA16A80F3AB0354CD"
);
String params = URLUtil.buildQuery(tokenParams, StandardCharsets.UTF_8);
String tokenUrl = "https://bd.whcp.edu.cn/sjjkfwpt/api/bd-api/oauth/token" + (StrUtil.isBlank(params) ? "" : "?" + params);
HttpRequest tokenHttpRequest = HttpUtil.createPost(tokenUrl);
tokenHttpRequest.header("Content-Type", "application/json");
tokenHttpRequest.header("Authorization", "Basic MTE0OjU5M2UwNGI4YTE4NWRlMzkyNjY4ZmY0ZDJhZGMyZTEw");
log.info("请求token: {}", tokenHttpRequest);
String tokenResBody = tokenHttpRequest.execute().body();
log.info("请求token结果: {}", tokenResBody);
JSONObject tokenJsonBody = JSONUtil.parseObj(tokenResBody);
// 请求数据
HttpRequest httpRequest = HttpUtil.createGet("https://bd.whcp.edu.cn/sjjkfwpt/api/bd-api/get/YXSDWJBSJZLB");
httpRequest.form(Map.of("access_token", tokenJsonBody.getStr("access_token"), "pageSize", 100, "page", 1));
log.info("请求单位数据: {}", httpRequest);
String resBody = httpRequest.execute().body();
log.info("请求单位数据结果: {}", resBody);
JSONObject jsonBody = JSONUtil.parseObj(resBody);
HttpRequest httpRequest = HttpUtil.createPost(URL);
httpRequest.header("Content-Type", "application/json");
httpRequest.header("applyId", APPLY_ID);
httpRequest.header("secretKey", SECRET_KEY);
httpRequest.body(JSONUtil.toJsonStr(Map.of("page", 1, "pagesize", 1000)));
if (jsonBody.getInt("code") != 200) {
throw new BaseException("获取单位数据失败,错误码: " + jsonBody.getInt("code") + ";错误原因:" + jsonBody.getStr("msg"));
String resBody = httpRequest.execute().body();
log.info("请求数据: {}", resBody);
JSONObject result = JSONUtil.parseObj(resBody);
if (!"successful".equals(result.getStr("msg"))) {
throw new BaseException("获取人员数据失败,错误码: " + result.getInt("status") + ";错误原因:" + result.getStr("msg"));
}
// 需要新增的单位
List<Sys_unit> insertList = new ArrayList<>();
// 需要更新的单位
List<Sys_unit> updateList = new ArrayList<>();
// 拉取过来的数据,全部的单位,一万四千多条,这里只保留 “部门类”
List<JSONObject> data = jsonBody.getJSONArray("data").stream()
.map(o -> (JSONObject) o)
.toList();
JSONObject resultData = result.getJSONObject("data");
JSONArray dataArray = resultData.getJSONArray("Rows");
List<JSONObject> data = dataArray.stream().map(v -> JSONUtil.parseObj(JSONUtil.toJsonStr(v))).toList();
Map<String, String> api2db = Arrays.stream(Api2UnitFiledMap.values())
.collect(Collectors.toMap(e -> e.apiField, e -> e.dbColumn));
for (JSONObject row : data) {
Sys_unit unit = DataCenterUtil.mapJsonToBean(row, Sys_unit.class, api2db);
JSONObject jsonRow = JSONUtil.parseObj(JSONUtil.toJsonStr(row));
Sys_unit unit = DataCenterUtil.mapJsonToBean(jsonRow, Sys_unit.class, api2db);
if("100".equals(unit.getParentId())) {
unit.setUnitcode("0");
unit.setParentId("0");
unit.setUnitLevel(1);
unit.setUnitType("根部门");
unit.setUnitTypeCode(0);
} else if(StringUtils.isBlank(unit.getParentId())) {
unit.setParentId("0");
unit.setUnitLevel(2);
unit.setUnitType("子部门");
unit.setUnitTypeCode(0);
} else {
unit.setUnitLevel(3);
unit.setUnitType("子部门");
unit.setUnitTypeCode(1);
}
unit.setUnitcode(unit.getId());
unit.setUnitTypeCode(1);
if (unitIds.contains(unit.getId())) {
// 存在则更新
@@ -6,7 +6,6 @@ import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.CharsetUtil;
import cn.hutool.core.util.IdcardUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.core.util.URLUtil;
import cn.hutool.crypto.digest.DigestUtil;
import cn.hutool.http.Header;
import cn.hutool.http.HttpRequest;
@@ -33,6 +32,7 @@ import org.nutz.aop.interceptor.ioc.TransAop;
import org.nutz.dao.Cnd;
import org.nutz.dao.Dao;
import org.nutz.dao.Sqls;
import org.nutz.dao.entity.Record;
import org.nutz.dao.sql.Sql;
import org.nutz.ioc.aop.Aop;
import org.nutz.ioc.loader.annotation.Inject;
@@ -42,7 +42,6 @@ import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.lang.reflect.Field;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.stream.Collectors;
@@ -50,6 +49,10 @@ import java.util.stream.Collectors;
@Slf4j
public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source> implements SysDataUserPullService {
private static final String URL = "http://api.jsahvc.edu.cn/getREST_JZGJBXXOABG_529a0bf78f574e10a4de5c6acbb48410";
private static final String APPLY_ID = "2001030048_1778744549088";
private static final String SECRET_KEY = "9ec51871-ee77-47ce-b981-d1c29d3477fa";
@Inject
private SysDictService sysDictService;
@Inject
@@ -110,59 +113,47 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
@Aop(TransAop.READ_COMMITTED)
public Date pull() {
try {
// 获取数据中心的字典
List<Sys_data_dict> dataDictList = dao().query(Sys_data_dict.class, Cnd.NEW());
// 字典码表Map,key为父级编码,value为子级字典列表
Map<String, List<Sys_data_dict>> dataDictMap = dataDictList.stream().collect(Collectors.groupingBy(Sys_data_dict::getParentCode));
// token url参数
Map<String, String> tokenParams = Map.of(
"grant_type", "password",
"scope", "read",
"username", "3848A3C738EFBB2C2E76E0D59B419DACA16A80F3AB0354CD",
"password", "3848A3C738EFBB2C2E76E0D59B419DACA16A80F3AB0354CD"
);
String params = URLUtil.buildQuery(tokenParams, StandardCharsets.UTF_8);
String tokenUrl = "https://bd.whcp.edu.cn/sjjkfwpt/api/bd-api/oauth/token" + (StrUtil.isBlank(params) ? "" : "?" + params);
HttpRequest tokenHttpRequest = HttpUtil.createPost(tokenUrl);
tokenHttpRequest.header("Content-Type", "application/json");
tokenHttpRequest.header("Authorization", "Basic MTE0OjU5M2UwNGI4YTE4NWRlMzkyNjY4ZmY0ZDJhZGMyZTEw");
log.info("请求token: {}", tokenHttpRequest);
String tokenResBody = tokenHttpRequest.execute().body();
log.info("请求token结果: {}", tokenResBody);
JSONObject tokenJsonBody = JSONUtil.parseObj(tokenResBody);
// 请求数据
HttpRequest httpRequest = HttpUtil.createGet("https://bd.whcp.edu.cn/sjjkfwpt/api/bd-api/get/JZGJCSJZLB");
httpRequest.form(Map.of("access_token", tokenJsonBody.getStr("access_token"), "pageSize", 5000, "page", 1));
log.info("请求人员数据: {}", httpRequest);
String resBody = httpRequest.execute().body();
log.info("请求人员数据结果: {}", resBody);
JSONObject jsonBody = JSONUtil.parseObj(resBody);
if (jsonBody.getInt("code") != 200) {
throw new BaseException("获取人员数据失败,错误码: " + jsonBody.getInt("code") + ";错误原因:" + jsonBody.getStr("msg"));
}
List<JSONObject> rawDataList = new ArrayList<>();
JSONArray data = jsonBody.getJSONArray("data");
int page = 1;
int total;
final Integer pageSize = 1000;
do {
// 请求数据
HttpRequest httpRequest = HttpUtil.createPost(URL);
httpRequest.header("Content-Type", "application/json");
httpRequest.header("applyId", APPLY_ID);
httpRequest.header("secretKey", SECRET_KEY);
String jsonStr = JSONUtil.toJsonStr(Map.of("page", page, "pagesize", pageSize));
if (!data.isEmpty()) {
// 先收集所有原始数据
rawDataList = data.stream().map(v -> (JSONObject) v).toList();
log.info("数据拉取进度: {}/{}", rawDataList.size(), jsonBody.getInt("total"));
} else {
log.warn("当前未获取到数据");
}
httpRequest.body(jsonStr);
String resBody = httpRequest.execute().body();
log.info("请求数据: {}", resBody);
JSONObject result = JSONUtil.parseObj(resBody);
if (!"successful".equals(result.getStr("msg"))) {
throw new BaseException("获取人员数据失败,错误码: " + result.getInt("status") + ";错误原因:" + result.getStr("msg"));
}
JSONObject resultData = result.getJSONObject("data");
total = result.getInt("total");
JSONArray data = resultData.getJSONArray("Rows");
if (!data.isEmpty()) {
// 先收集所有原始数据
rawDataList.addAll(data.stream().map(v -> JSONUtil.parseObj(JSONUtil.toJsonStr(v))).toList());
log.info("数据拉取进度: 总数据量:{},已经拉取{},本次拉取:{}", total, rawDataList.size(), data.size());
} else {
log.warn("当前未获取到数据");
}
page ++;
} while (page <= Math.ceil(total / (double) pageSize));
Date nowDate = new Date();
// 赋值
List<Sys_user_source> latestSourceList = rawDataList.stream().map(raw -> {
List<Sys_user_source> latestSourceList = rawDataList.stream().filter(e -> e.getStr("BH").length() == 10).map(raw -> {
Sys_user_source sysUser = new Sys_user_source();
// 使用缓存的字段映射
@@ -170,25 +161,9 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
try {
// 根据字段类型设置值
if (mapping.field.getType() == String.class) {
if (StrUtil.isNotBlank(mapping.dict)) {
Sys_data_dict sysDataDict = dataDictMap.get(mapping.dict).stream()
.filter(v -> raw.getStr(mapping.key)
.equals(v.getCode())).findFirst().orElse(new Sys_data_dict());
mapping.field.set(sysUser, sysDataDict.getName());
} else {
// 特殊处理 根据身份证号获取性别和出生年月
if ("SFZJH".equals(mapping.field.getName())) {
String idCard = raw.getStr(mapping.key);
// 设置出生年月
try {
sysUser.setBirthday(IdcardUtil.getBirthDate(idCard));
} catch (Exception e) {
sysUser.setBirthday(null);
}
} else {
mapping.field.set(sysUser, raw.getStr(mapping.key));
}
mapping.field.set(sysUser, raw.getStr(mapping.key));
if("XB".equals(mapping.key) && StrUtil.isNotBlank(raw.getStr(mapping.key))) {
mapping.field.set(sysUser, raw.getStr(mapping.key).replace("性", ""));
}
} else if (mapping.field.getType() == Date.class) {
mapping.field.set(sysUser, raw.getDate(mapping.key));
@@ -198,15 +173,15 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
}
}
// 设置拉取时间
sysUser.setPullTime(nowDate);
return sysUser;
}).toList();
log.info("用户数据初始化完成,等待插入,当前数据条数{}", latestSourceList.size());
// 插入到数据库
dao().insert(latestSourceList);
// 更新字典数据
this.updateDict(latestSourceList);
// 人员的单位数据和数据库的单位数据比较,如果人员里面有单位不存在,去更新单位数据
List<String> sourceUnitIds = latestSourceList.stream().map(Sys_user_source::getUnitId).distinct().toList();
@@ -221,10 +196,16 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
}
return nowDate;
} catch (Exception e) {
throw new BaseException("数据拉取失败,{}", e.getMessage());
log.error("数据拉取失败,{}", e.getMessage());
throw new BaseException("数据拉取失败");
}
}
@Override
public Map<String, Date> pullPostDoctoral() {
return null;
}
@Async
private void updateDict(List<Sys_user_source> userSources) {
//判断是否要更新在职状态字典
@@ -234,9 +215,7 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
//判断是否要更新学位字典
List<String> sourceAcademicDegrees = userSources.stream().map(Sys_user::getAcademicDegree).filter(StrUtil::isNotBlank).distinct().toList();
//判断是否要更新教职工类别字典
List<String> personTypes = userSources.stream().map(Sys_user::getPersonType).filter(StrUtil::isNotBlank).distinct().toList();
//判断是否要更新身份类型字典
List<String> sourceIdentityTypes = userSources.stream().map(Sys_user::getIdentityType).filter(StrUtil::isNotBlank).distinct().toList();
List<String> sourcePersonTypes = userSources.stream().map(Sys_user::getPersonType).filter(StrUtil::isNotBlank).distinct().toList();
//系统在职状态字典
List<Sys_dict> sysUserStates = sysDictService.getSubListByCode("USER_STATE");
@@ -244,10 +223,7 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
List<Sys_dict> sysEducations = sysDictService.getSubListByCode("USER_EDUCATION");
//系统学位字典
List<Sys_dict> sysAcademicDegrees = sysDictService.getSubListByCode("USER_ACADEMIC_DEGREE");
//系统教职工类别字典
List<Sys_dict> sysStaffTypes = sysDictService.getSubListByCode("USER_STAFF_TYPE");
//系统身份类型字典
List<Sys_dict> sysIdentityTypes = sysDictService.getSubListByCode("USER_IDENTITY_TYPE");
List<Sys_dict> sysPersonTypes = sysDictService.getSubListByCode("USER_PERSON_TYPE");
//系统中不存在的就插入
List<Sys_dict> insertUserStates = sourceUserStates.stream().filter(state -> sysUserStates.stream().noneMatch(dict -> dict.getCode().equals(state))).map(state -> {
@@ -271,6 +247,13 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
return dict;
}).toList();
List<Sys_dict> insertPersonTypes = sourcePersonTypes.stream().filter(personType -> sysPersonTypes.stream().noneMatch(dict -> dict.getCode().equals(personType))).map(personType -> {
Sys_dict dict = new Sys_dict();
dict.setCode(personType);
dict.setName(personType);
return dict;
}).toList();
for (Sys_dict dict : insertUserStates) {
sysDictService.saveByParentCode(dict, "USER_STATE");
}
@@ -280,63 +263,12 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
for (Sys_dict dict : insertAcademicDegrees) {
sysDictService.saveByParentCode(dict, "USER_ACADEMIC_DEGREE");
}
for (Sys_dict dict : insertPersonTypes) {
sysDictService.saveByParentCode(dict, "USER_PERSON_TYPE");
}
sysDictService.clearCache();
}
@Override
public Map<String, Date> pullPostDoctoral() {
String appId = "1926887238437818370";
String secret = "32df31fcf4fc43f9a647ee1f47134f36";
List<JSONObject> allUsers = new ArrayList<>();
int skip = 0;
int totalCount = 0;
boolean firstRequest = true;
do {
long ts = System.currentTimeMillis();
String sign = Base64.encode(DigestUtil.md5Hex(appId + secret + ts, CharsetUtil.CHARSET_UTF_8));
HttpRequest httpRequest = HttpUtil.createPost("https://sjzcpt.nnu.edu.cn/cdsp/data-api/v2/DS0040");
httpRequest.header("Content-Type", "application/json");
httpRequest.header("appId", appId);
httpRequest.header("timestamp", String.valueOf(ts));
httpRequest.header("sign", sign);
HashMap<String, Object> reqBody = new HashMap<>();
reqBody.put("$count", true);
reqBody.put("$skip", skip);
reqBody.put("$top", 1000);
httpRequest.body(JSONUtil.toJsonStr(reqBody));
String resBody = httpRequest.execute().body();
JSONObject entries = JSONUtil.parseObj(resBody);
if (firstRequest) {
totalCount = entries.getInt("@odata.count", 0);
firstRequest = false;
}
JSONArray value = entries.getJSONArray("value");
if (!value.isEmpty()) {
List<JSONObject> list = value.stream().map(item -> (JSONObject) item).toList();
allUsers.addAll(list);
skip += list.size();
} else {
break;
}
} while (skip < totalCount);
Map<String, Date> data = allUsers.stream().filter(v -> v.getStr("进站日期") != null).collect(Collectors.toMap(v -> v.getStr("教职工号"), v -> v.getDate("进站日期")));
return data;
}
@Override
public NutMap searchOptions() {
Sql sql = Sqls.create("""
@@ -385,73 +317,8 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
return listMap(sql);
}
@Override
public Map<String, NutMap> pullFinance() {
// 获取配置文件 接口地址 和 token
DataCenterProperties.Credential credential = dataCenterProperties.credential("finance");
String url = credential.getUrl();
String token = credential.getToken();
List<JSONObject> rawDataList = new ArrayList<>();
String year = String.valueOf(DateUtil.thisYear());
String month = String.valueOf(DateUtil.thisMonth() + 1);
int pageNum = 1;
int pageSize = 1000;
int totalPages = 1;
int totalCount = 0;
do {
HttpRequest httpRequest = HttpUtil.createPost(url);
httpRequest.header(Header.CONTENT_TYPE, "application/json");
httpRequest.header("Accept-Encoding", "gzip, deflate, br");
httpRequest.header("X-H3C-TOKEN", token);
Map<String, Object> reqBody = Map.of(
"nf", year,
"yf", month,
"pageNum", pageNum,
"pageSize", pageSize
);
httpRequest.body(JSONUtil.toJsonStr(reqBody));
JSONObject resp = JSONUtil.parseObj(httpRequest.execute().body());
if (resp.getInt("code") != 0) {
throw new BaseException("获取财务数据失败,错误码: " + resp.getInt("code") + ";错误原因:" + resp.getStr("msg"));
}
JSONObject data = resp.getJSONObject("data");
if (pageNum == 1) {
totalCount = data.getInt("total", 0);
// totalPages = data.getInt("pages", 0);
totalPages = (int) Math.ceil((double) totalCount / pageSize);
}
JSONArray records = data.getJSONArray("records");
if (CollUtil.isNotEmpty(records)) {
rawDataList.addAll(records.stream().map(v -> (JSONObject) v).toList());
log.info("数据拉取进度: {}/{}", rawDataList.size(), totalCount);
}
pageNum++; // 只加页码
} while (pageNum <= totalPages);
// 接口数据输出
log.info("数据拉取结果: {}", rawDataList);
// 最后结果数据
Map<String, NutMap> result = new HashMap<>();
Map<String, String> api2db = Arrays.stream(Api2FinanceFiledMap.values())
.collect(Collectors.toMap(e -> e.apiField, e -> e.dbColumn));
for (JSONObject row : rawDataList) {
NutMap nutMap = DataCenterUtil.mapJsonToBean(row, NutMap.class, api2db);
result.put(nutMap.getString("loginname"), nutMap);
}
// 输出结果
log.info("财务数据拉取转换结果: {}", result);
log.info("数据拉取完成,共拉取 {} 条数据", result.size());
return result;
return null;
}
}