fix: 人员数据治理迁移5
This commit is contained in:
@@ -30,3 +30,4 @@ src/main/resources/application-dev.yaml
|
|||||||
.vscode
|
.vscode
|
||||||
bin
|
bin
|
||||||
node-data-center-mock/
|
node-data-center-mock/
|
||||||
|
.workbuddy
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
package com.budwk.app.sys.services.impl;
|
package com.budwk.app.sys.services.impl;
|
||||||
|
|
||||||
import cn.hutool.core.util.StrUtil;
|
import cn.hutool.core.util.StrUtil;
|
||||||
import cn.hutool.core.util.URLUtil;
|
|
||||||
import cn.hutool.http.HttpRequest;
|
import cn.hutool.http.HttpRequest;
|
||||||
import cn.hutool.http.HttpUtil;
|
import cn.hutool.http.HttpUtil;
|
||||||
|
import cn.hutool.json.JSONArray;
|
||||||
import cn.hutool.json.JSONObject;
|
import cn.hutool.json.JSONObject;
|
||||||
import cn.hutool.json.JSONUtil;
|
import cn.hutool.json.JSONUtil;
|
||||||
import com.budwk.app.base.config.DataCenterProperties;
|
import com.budwk.app.base.config.DataCenterProperties;
|
||||||
@@ -13,16 +13,19 @@ import com.budwk.app.sys.enums.Api2UnitFiledMap;
|
|||||||
import com.budwk.app.sys.models.Sys_unit;
|
import com.budwk.app.sys.models.Sys_unit;
|
||||||
import com.budwk.app.sys.services.SysDataUnitPullService;
|
import com.budwk.app.sys.services.SysDataUnitPullService;
|
||||||
import com.budwk.app.sys.utils.DataCenterUtil;
|
import com.budwk.app.sys.utils.DataCenterUtil;
|
||||||
|
import com.budwk.app.web.commons.base.Globals;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.nutz.aop.interceptor.ioc.TransAop;
|
import org.nutz.aop.interceptor.ioc.TransAop;
|
||||||
|
import org.nutz.dao.Cnd;
|
||||||
import org.nutz.dao.Dao;
|
import org.nutz.dao.Dao;
|
||||||
import org.nutz.dao.Sqls;
|
import org.nutz.dao.Sqls;
|
||||||
|
import org.nutz.dao.entity.Record;
|
||||||
import org.nutz.dao.sql.Sql;
|
import org.nutz.dao.sql.Sql;
|
||||||
import org.nutz.ioc.aop.Aop;
|
import org.nutz.ioc.aop.Aop;
|
||||||
import org.nutz.ioc.loader.annotation.Inject;
|
import org.nutz.ioc.loader.annotation.Inject;
|
||||||
import org.nutz.ioc.loader.annotation.IocBean;
|
import org.nutz.ioc.loader.annotation.IocBean;
|
||||||
|
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -40,6 +43,10 @@ import java.util.stream.Collectors;
|
|||||||
@IocBean(args = {"refer:dao"})
|
@IocBean(args = {"refer:dao"})
|
||||||
public class SysDataUnitPullServiceImpl extends BaseServiceImpl<Sys_unit> implements SysDataUnitPullService {
|
public class SysDataUnitPullServiceImpl extends BaseServiceImpl<Sys_unit> implements SysDataUnitPullService {
|
||||||
|
|
||||||
|
private static final String URL = "http://api.jsahvc.edu.cn/getREST_XXZZJG_c515102514c74b4e913f01e05ecb0286";
|
||||||
|
private static final String APPLY_ID = "2001030048_1778744549088";
|
||||||
|
private static final String SECRET_KEY = "9ec51871-ee77-47ce-b981-d1c29d3477fa";
|
||||||
|
|
||||||
@Inject
|
@Inject
|
||||||
private DataCenterProperties dcPro;
|
private DataCenterProperties dcPro;
|
||||||
|
|
||||||
@@ -64,54 +71,52 @@ public class SysDataUnitPullServiceImpl extends BaseServiceImpl<Sys_unit> implem
|
|||||||
@Override
|
@Override
|
||||||
@Aop(TransAop.READ_COMMITTED)
|
@Aop(TransAop.READ_COMMITTED)
|
||||||
public void updateUnits(List<String> unitIds) {
|
public void updateUnits(List<String> unitIds) {
|
||||||
// token url参数
|
|
||||||
Map<String, String> tokenParams = Map.of(
|
|
||||||
"grant_type", "password",
|
|
||||||
"scope", "read",
|
|
||||||
"username", "3848A3C738EFBB2C2E76E0D59B419DACA16A80F3AB0354CD",
|
|
||||||
"password", "3848A3C738EFBB2C2E76E0D59B419DACA16A80F3AB0354CD"
|
|
||||||
);
|
|
||||||
|
|
||||||
String params = URLUtil.buildQuery(tokenParams, StandardCharsets.UTF_8);
|
|
||||||
String tokenUrl = "https://bd.whcp.edu.cn/sjjkfwpt/api/bd-api/oauth/token" + (StrUtil.isBlank(params) ? "" : "?" + params);
|
|
||||||
HttpRequest tokenHttpRequest = HttpUtil.createPost(tokenUrl);
|
|
||||||
tokenHttpRequest.header("Content-Type", "application/json");
|
|
||||||
tokenHttpRequest.header("Authorization", "Basic MTE0OjU5M2UwNGI4YTE4NWRlMzkyNjY4ZmY0ZDJhZGMyZTEw");
|
|
||||||
|
|
||||||
log.info("请求token: {}", tokenHttpRequest);
|
|
||||||
String tokenResBody = tokenHttpRequest.execute().body();
|
|
||||||
|
|
||||||
log.info("请求token结果: {}", tokenResBody);
|
|
||||||
JSONObject tokenJsonBody = JSONUtil.parseObj(tokenResBody);
|
|
||||||
|
|
||||||
// 请求数据
|
// 请求数据
|
||||||
HttpRequest httpRequest = HttpUtil.createGet("https://bd.whcp.edu.cn/sjjkfwpt/api/bd-api/get/YXSDWJBSJZLB");
|
HttpRequest httpRequest = HttpUtil.createPost(URL);
|
||||||
httpRequest.form(Map.of("access_token", tokenJsonBody.getStr("access_token"), "pageSize", 100, "page", 1));
|
httpRequest.header("Content-Type", "application/json");
|
||||||
log.info("请求单位数据: {}", httpRequest);
|
httpRequest.header("applyId", APPLY_ID);
|
||||||
String resBody = httpRequest.execute().body();
|
httpRequest.header("secretKey", SECRET_KEY);
|
||||||
log.info("请求单位数据结果: {}", resBody);
|
httpRequest.body(JSONUtil.toJsonStr(Map.of("page", 1, "pagesize", 1000)));
|
||||||
JSONObject jsonBody = JSONUtil.parseObj(resBody);
|
|
||||||
|
|
||||||
if (jsonBody.getInt("code") != 200) {
|
String resBody = httpRequest.execute().body();
|
||||||
throw new BaseException("获取单位数据失败,错误码: " + jsonBody.getInt("code") + ";错误原因:" + jsonBody.getStr("msg"));
|
log.info("请求数据: {}", resBody);
|
||||||
|
JSONObject result = JSONUtil.parseObj(resBody);
|
||||||
|
|
||||||
|
if (!"successful".equals(result.getStr("msg"))) {
|
||||||
|
throw new BaseException("获取人员数据失败,错误码: " + result.getInt("status") + ";错误原因:" + result.getStr("msg"));
|
||||||
}
|
}
|
||||||
|
|
||||||
// 需要新增的单位
|
// 需要新增的单位
|
||||||
List<Sys_unit> insertList = new ArrayList<>();
|
List<Sys_unit> insertList = new ArrayList<>();
|
||||||
// 需要更新的单位
|
// 需要更新的单位
|
||||||
List<Sys_unit> updateList = new ArrayList<>();
|
List<Sys_unit> updateList = new ArrayList<>();
|
||||||
|
JSONObject resultData = result.getJSONObject("data");
|
||||||
// 拉取过来的数据,全部的单位,一万四千多条,这里只保留 “部门类”
|
JSONArray dataArray = resultData.getJSONArray("Rows");
|
||||||
List<JSONObject> data = jsonBody.getJSONArray("data").stream()
|
List<JSONObject> data = dataArray.stream().map(v -> JSONUtil.parseObj(JSONUtil.toJsonStr(v))).toList();
|
||||||
.map(o -> (JSONObject) o)
|
|
||||||
.toList();
|
|
||||||
|
|
||||||
Map<String, String> api2db = Arrays.stream(Api2UnitFiledMap.values())
|
Map<String, String> api2db = Arrays.stream(Api2UnitFiledMap.values())
|
||||||
.collect(Collectors.toMap(e -> e.apiField, e -> e.dbColumn));
|
.collect(Collectors.toMap(e -> e.apiField, e -> e.dbColumn));
|
||||||
for (JSONObject row : data) {
|
for (JSONObject row : data) {
|
||||||
Sys_unit unit = DataCenterUtil.mapJsonToBean(row, Sys_unit.class, api2db);
|
JSONObject jsonRow = JSONUtil.parseObj(JSONUtil.toJsonStr(row));
|
||||||
|
Sys_unit unit = DataCenterUtil.mapJsonToBean(jsonRow, Sys_unit.class, api2db);
|
||||||
|
if("100".equals(unit.getParentId())) {
|
||||||
|
unit.setUnitcode("0");
|
||||||
|
unit.setParentId("0");
|
||||||
|
unit.setUnitLevel(1);
|
||||||
|
unit.setUnitType("根部门");
|
||||||
|
unit.setUnitTypeCode(0);
|
||||||
|
} else if(StringUtils.isBlank(unit.getParentId())) {
|
||||||
|
unit.setParentId("0");
|
||||||
|
unit.setUnitLevel(2);
|
||||||
|
unit.setUnitType("子部门");
|
||||||
|
unit.setUnitTypeCode(0);
|
||||||
|
} else {
|
||||||
|
unit.setUnitLevel(3);
|
||||||
|
unit.setUnitType("子部门");
|
||||||
|
unit.setUnitTypeCode(1);
|
||||||
|
}
|
||||||
unit.setUnitcode(unit.getId());
|
unit.setUnitcode(unit.getId());
|
||||||
unit.setUnitTypeCode(1);
|
|
||||||
|
|
||||||
if (unitIds.contains(unit.getId())) {
|
if (unitIds.contains(unit.getId())) {
|
||||||
// 存在则更新
|
// 存在则更新
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import cn.hutool.core.date.DateUtil;
|
|||||||
import cn.hutool.core.util.CharsetUtil;
|
import cn.hutool.core.util.CharsetUtil;
|
||||||
import cn.hutool.core.util.IdcardUtil;
|
import cn.hutool.core.util.IdcardUtil;
|
||||||
import cn.hutool.core.util.StrUtil;
|
import cn.hutool.core.util.StrUtil;
|
||||||
import cn.hutool.core.util.URLUtil;
|
|
||||||
import cn.hutool.crypto.digest.DigestUtil;
|
import cn.hutool.crypto.digest.DigestUtil;
|
||||||
import cn.hutool.http.Header;
|
import cn.hutool.http.Header;
|
||||||
import cn.hutool.http.HttpRequest;
|
import cn.hutool.http.HttpRequest;
|
||||||
@@ -33,6 +32,7 @@ import org.nutz.aop.interceptor.ioc.TransAop;
|
|||||||
import org.nutz.dao.Cnd;
|
import org.nutz.dao.Cnd;
|
||||||
import org.nutz.dao.Dao;
|
import org.nutz.dao.Dao;
|
||||||
import org.nutz.dao.Sqls;
|
import org.nutz.dao.Sqls;
|
||||||
|
import org.nutz.dao.entity.Record;
|
||||||
import org.nutz.dao.sql.Sql;
|
import org.nutz.dao.sql.Sql;
|
||||||
import org.nutz.ioc.aop.Aop;
|
import org.nutz.ioc.aop.Aop;
|
||||||
import org.nutz.ioc.loader.annotation.Inject;
|
import org.nutz.ioc.loader.annotation.Inject;
|
||||||
@@ -42,7 +42,6 @@ import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
|||||||
|
|
||||||
import java.lang.reflect.Field;
|
import java.lang.reflect.Field;
|
||||||
import java.math.BigDecimal;
|
import java.math.BigDecimal;
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@@ -50,6 +49,10 @@ import java.util.stream.Collectors;
|
|||||||
@Slf4j
|
@Slf4j
|
||||||
public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source> implements SysDataUserPullService {
|
public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source> implements SysDataUserPullService {
|
||||||
|
|
||||||
|
private static final String URL = "http://api.jsahvc.edu.cn/getREST_JZGJBXXOABG_529a0bf78f574e10a4de5c6acbb48410";
|
||||||
|
private static final String APPLY_ID = "2001030048_1778744549088";
|
||||||
|
private static final String SECRET_KEY = "9ec51871-ee77-47ce-b981-d1c29d3477fa";
|
||||||
|
|
||||||
@Inject
|
@Inject
|
||||||
private SysDictService sysDictService;
|
private SysDictService sysDictService;
|
||||||
@Inject
|
@Inject
|
||||||
@@ -110,59 +113,47 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
|
|||||||
@Aop(TransAop.READ_COMMITTED)
|
@Aop(TransAop.READ_COMMITTED)
|
||||||
public Date pull() {
|
public Date pull() {
|
||||||
try {
|
try {
|
||||||
// 获取数据中心的字典
|
|
||||||
List<Sys_data_dict> dataDictList = dao().query(Sys_data_dict.class, Cnd.NEW());
|
|
||||||
// 字典码表Map,key为父级编码,value为子级字典列表
|
|
||||||
Map<String, List<Sys_data_dict>> dataDictMap = dataDictList.stream().collect(Collectors.groupingBy(Sys_data_dict::getParentCode));
|
|
||||||
|
|
||||||
// token url参数
|
|
||||||
Map<String, String> tokenParams = Map.of(
|
|
||||||
"grant_type", "password",
|
|
||||||
"scope", "read",
|
|
||||||
"username", "3848A3C738EFBB2C2E76E0D59B419DACA16A80F3AB0354CD",
|
|
||||||
"password", "3848A3C738EFBB2C2E76E0D59B419DACA16A80F3AB0354CD"
|
|
||||||
);
|
|
||||||
|
|
||||||
String params = URLUtil.buildQuery(tokenParams, StandardCharsets.UTF_8);
|
|
||||||
String tokenUrl = "https://bd.whcp.edu.cn/sjjkfwpt/api/bd-api/oauth/token" + (StrUtil.isBlank(params) ? "" : "?" + params);
|
|
||||||
HttpRequest tokenHttpRequest = HttpUtil.createPost(tokenUrl);
|
|
||||||
tokenHttpRequest.header("Content-Type", "application/json");
|
|
||||||
tokenHttpRequest.header("Authorization", "Basic MTE0OjU5M2UwNGI4YTE4NWRlMzkyNjY4ZmY0ZDJhZGMyZTEw");
|
|
||||||
|
|
||||||
log.info("请求token: {}", tokenHttpRequest);
|
|
||||||
String tokenResBody = tokenHttpRequest.execute().body();
|
|
||||||
log.info("请求token结果: {}", tokenResBody);
|
|
||||||
|
|
||||||
JSONObject tokenJsonBody = JSONUtil.parseObj(tokenResBody);
|
|
||||||
|
|
||||||
// 请求数据
|
|
||||||
HttpRequest httpRequest = HttpUtil.createGet("https://bd.whcp.edu.cn/sjjkfwpt/api/bd-api/get/JZGJCSJZLB");
|
|
||||||
httpRequest.form(Map.of("access_token", tokenJsonBody.getStr("access_token"), "pageSize", 5000, "page", 1));
|
|
||||||
|
|
||||||
log.info("请求人员数据: {}", httpRequest);
|
|
||||||
String resBody = httpRequest.execute().body();
|
|
||||||
log.info("请求人员数据结果: {}", resBody);
|
|
||||||
JSONObject jsonBody = JSONUtil.parseObj(resBody);
|
|
||||||
|
|
||||||
if (jsonBody.getInt("code") != 200) {
|
|
||||||
throw new BaseException("获取人员数据失败,错误码: " + jsonBody.getInt("code") + ";错误原因:" + jsonBody.getStr("msg"));
|
|
||||||
}
|
|
||||||
|
|
||||||
List<JSONObject> rawDataList = new ArrayList<>();
|
List<JSONObject> rawDataList = new ArrayList<>();
|
||||||
JSONArray data = jsonBody.getJSONArray("data");
|
int page = 1;
|
||||||
|
int total;
|
||||||
|
final Integer pageSize = 1000;
|
||||||
|
do {
|
||||||
|
// 请求数据
|
||||||
|
HttpRequest httpRequest = HttpUtil.createPost(URL);
|
||||||
|
httpRequest.header("Content-Type", "application/json");
|
||||||
|
httpRequest.header("applyId", APPLY_ID);
|
||||||
|
httpRequest.header("secretKey", SECRET_KEY);
|
||||||
|
String jsonStr = JSONUtil.toJsonStr(Map.of("page", page, "pagesize", pageSize));
|
||||||
|
|
||||||
if (!data.isEmpty()) {
|
httpRequest.body(jsonStr);
|
||||||
// 先收集所有原始数据
|
|
||||||
rawDataList = data.stream().map(v -> (JSONObject) v).toList();
|
String resBody = httpRequest.execute().body();
|
||||||
log.info("数据拉取进度: {}/{}", rawDataList.size(), jsonBody.getInt("total"));
|
log.info("请求数据: {}", resBody);
|
||||||
} else {
|
JSONObject result = JSONUtil.parseObj(resBody);
|
||||||
log.warn("当前未获取到数据");
|
|
||||||
}
|
if (!"successful".equals(result.getStr("msg"))) {
|
||||||
|
throw new BaseException("获取人员数据失败,错误码: " + result.getInt("status") + ";错误原因:" + result.getStr("msg"));
|
||||||
|
}
|
||||||
|
|
||||||
|
JSONObject resultData = result.getJSONObject("data");
|
||||||
|
total = result.getInt("total");
|
||||||
|
JSONArray data = resultData.getJSONArray("Rows");
|
||||||
|
|
||||||
|
if (!data.isEmpty()) {
|
||||||
|
// 先收集所有原始数据
|
||||||
|
rawDataList.addAll(data.stream().map(v -> JSONUtil.parseObj(JSONUtil.toJsonStr(v))).toList());
|
||||||
|
log.info("数据拉取进度: 总数据量:{},已经拉取{},本次拉取:{}", total, rawDataList.size(), data.size());
|
||||||
|
} else {
|
||||||
|
log.warn("当前未获取到数据");
|
||||||
|
}
|
||||||
|
|
||||||
|
page ++;
|
||||||
|
} while (page <= Math.ceil(total / (double) pageSize));
|
||||||
|
|
||||||
Date nowDate = new Date();
|
Date nowDate = new Date();
|
||||||
|
|
||||||
// 赋值
|
// 赋值
|
||||||
List<Sys_user_source> latestSourceList = rawDataList.stream().map(raw -> {
|
List<Sys_user_source> latestSourceList = rawDataList.stream().filter(e -> e.getStr("BH").length() == 10).map(raw -> {
|
||||||
Sys_user_source sysUser = new Sys_user_source();
|
Sys_user_source sysUser = new Sys_user_source();
|
||||||
|
|
||||||
// 使用缓存的字段映射
|
// 使用缓存的字段映射
|
||||||
@@ -170,25 +161,9 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
|
|||||||
try {
|
try {
|
||||||
// 根据字段类型设置值
|
// 根据字段类型设置值
|
||||||
if (mapping.field.getType() == String.class) {
|
if (mapping.field.getType() == String.class) {
|
||||||
|
mapping.field.set(sysUser, raw.getStr(mapping.key));
|
||||||
if (StrUtil.isNotBlank(mapping.dict)) {
|
if("XB".equals(mapping.key) && StrUtil.isNotBlank(raw.getStr(mapping.key))) {
|
||||||
Sys_data_dict sysDataDict = dataDictMap.get(mapping.dict).stream()
|
mapping.field.set(sysUser, raw.getStr(mapping.key).replace("性", ""));
|
||||||
.filter(v -> raw.getStr(mapping.key)
|
|
||||||
.equals(v.getCode())).findFirst().orElse(new Sys_data_dict());
|
|
||||||
mapping.field.set(sysUser, sysDataDict.getName());
|
|
||||||
} else {
|
|
||||||
// 特殊处理 根据身份证号获取性别和出生年月
|
|
||||||
if ("SFZJH".equals(mapping.field.getName())) {
|
|
||||||
String idCard = raw.getStr(mapping.key);
|
|
||||||
// 设置出生年月
|
|
||||||
try {
|
|
||||||
sysUser.setBirthday(IdcardUtil.getBirthDate(idCard));
|
|
||||||
} catch (Exception e) {
|
|
||||||
sysUser.setBirthday(null);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
mapping.field.set(sysUser, raw.getStr(mapping.key));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} else if (mapping.field.getType() == Date.class) {
|
} else if (mapping.field.getType() == Date.class) {
|
||||||
mapping.field.set(sysUser, raw.getDate(mapping.key));
|
mapping.field.set(sysUser, raw.getDate(mapping.key));
|
||||||
@@ -198,15 +173,15 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 设置拉取时间
|
|
||||||
sysUser.setPullTime(nowDate);
|
sysUser.setPullTime(nowDate);
|
||||||
|
|
||||||
return sysUser;
|
return sysUser;
|
||||||
}).toList();
|
}).toList();
|
||||||
|
|
||||||
log.info("用户数据初始化完成,等待插入,当前数据条数{}", latestSourceList.size());
|
log.info("用户数据初始化完成,等待插入,当前数据条数{}", latestSourceList.size());
|
||||||
// 插入到数据库
|
// 插入到数据库
|
||||||
dao().insert(latestSourceList);
|
dao().insert(latestSourceList);
|
||||||
|
// 更新字典数据
|
||||||
|
this.updateDict(latestSourceList);
|
||||||
|
|
||||||
// 人员的单位数据和数据库的单位数据比较,如果人员里面有单位不存在,去更新单位数据
|
// 人员的单位数据和数据库的单位数据比较,如果人员里面有单位不存在,去更新单位数据
|
||||||
List<String> sourceUnitIds = latestSourceList.stream().map(Sys_user_source::getUnitId).distinct().toList();
|
List<String> sourceUnitIds = latestSourceList.stream().map(Sys_user_source::getUnitId).distinct().toList();
|
||||||
@@ -221,10 +196,16 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
|
|||||||
}
|
}
|
||||||
return nowDate;
|
return nowDate;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new BaseException("数据拉取失败,{}", e.getMessage());
|
log.error("数据拉取失败,{}", e.getMessage());
|
||||||
|
throw new BaseException("数据拉取失败");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String, Date> pullPostDoctoral() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
@Async
|
@Async
|
||||||
private void updateDict(List<Sys_user_source> userSources) {
|
private void updateDict(List<Sys_user_source> userSources) {
|
||||||
//判断是否要更新在职状态字典
|
//判断是否要更新在职状态字典
|
||||||
@@ -234,9 +215,7 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
|
|||||||
//判断是否要更新学位字典
|
//判断是否要更新学位字典
|
||||||
List<String> sourceAcademicDegrees = userSources.stream().map(Sys_user::getAcademicDegree).filter(StrUtil::isNotBlank).distinct().toList();
|
List<String> sourceAcademicDegrees = userSources.stream().map(Sys_user::getAcademicDegree).filter(StrUtil::isNotBlank).distinct().toList();
|
||||||
//判断是否要更新教职工类别字典
|
//判断是否要更新教职工类别字典
|
||||||
List<String> personTypes = userSources.stream().map(Sys_user::getPersonType).filter(StrUtil::isNotBlank).distinct().toList();
|
List<String> sourcePersonTypes = userSources.stream().map(Sys_user::getPersonType).filter(StrUtil::isNotBlank).distinct().toList();
|
||||||
//判断是否要更新身份类型字典
|
|
||||||
List<String> sourceIdentityTypes = userSources.stream().map(Sys_user::getIdentityType).filter(StrUtil::isNotBlank).distinct().toList();
|
|
||||||
|
|
||||||
//系统在职状态字典
|
//系统在职状态字典
|
||||||
List<Sys_dict> sysUserStates = sysDictService.getSubListByCode("USER_STATE");
|
List<Sys_dict> sysUserStates = sysDictService.getSubListByCode("USER_STATE");
|
||||||
@@ -244,10 +223,7 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
|
|||||||
List<Sys_dict> sysEducations = sysDictService.getSubListByCode("USER_EDUCATION");
|
List<Sys_dict> sysEducations = sysDictService.getSubListByCode("USER_EDUCATION");
|
||||||
//系统学位字典
|
//系统学位字典
|
||||||
List<Sys_dict> sysAcademicDegrees = sysDictService.getSubListByCode("USER_ACADEMIC_DEGREE");
|
List<Sys_dict> sysAcademicDegrees = sysDictService.getSubListByCode("USER_ACADEMIC_DEGREE");
|
||||||
//系统教职工类别字典
|
List<Sys_dict> sysPersonTypes = sysDictService.getSubListByCode("USER_PERSON_TYPE");
|
||||||
List<Sys_dict> sysStaffTypes = sysDictService.getSubListByCode("USER_STAFF_TYPE");
|
|
||||||
//系统身份类型字典
|
|
||||||
List<Sys_dict> sysIdentityTypes = sysDictService.getSubListByCode("USER_IDENTITY_TYPE");
|
|
||||||
|
|
||||||
//系统中不存在的就插入
|
//系统中不存在的就插入
|
||||||
List<Sys_dict> insertUserStates = sourceUserStates.stream().filter(state -> sysUserStates.stream().noneMatch(dict -> dict.getCode().equals(state))).map(state -> {
|
List<Sys_dict> insertUserStates = sourceUserStates.stream().filter(state -> sysUserStates.stream().noneMatch(dict -> dict.getCode().equals(state))).map(state -> {
|
||||||
@@ -271,6 +247,13 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
|
|||||||
return dict;
|
return dict;
|
||||||
}).toList();
|
}).toList();
|
||||||
|
|
||||||
|
List<Sys_dict> insertPersonTypes = sourcePersonTypes.stream().filter(personType -> sysPersonTypes.stream().noneMatch(dict -> dict.getCode().equals(personType))).map(personType -> {
|
||||||
|
Sys_dict dict = new Sys_dict();
|
||||||
|
dict.setCode(personType);
|
||||||
|
dict.setName(personType);
|
||||||
|
return dict;
|
||||||
|
}).toList();
|
||||||
|
|
||||||
for (Sys_dict dict : insertUserStates) {
|
for (Sys_dict dict : insertUserStates) {
|
||||||
sysDictService.saveByParentCode(dict, "USER_STATE");
|
sysDictService.saveByParentCode(dict, "USER_STATE");
|
||||||
}
|
}
|
||||||
@@ -280,63 +263,12 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
|
|||||||
for (Sys_dict dict : insertAcademicDegrees) {
|
for (Sys_dict dict : insertAcademicDegrees) {
|
||||||
sysDictService.saveByParentCode(dict, "USER_ACADEMIC_DEGREE");
|
sysDictService.saveByParentCode(dict, "USER_ACADEMIC_DEGREE");
|
||||||
}
|
}
|
||||||
|
for (Sys_dict dict : insertPersonTypes) {
|
||||||
|
sysDictService.saveByParentCode(dict, "USER_PERSON_TYPE");
|
||||||
|
}
|
||||||
sysDictService.clearCache();
|
sysDictService.clearCache();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Map<String, Date> pullPostDoctoral() {
|
|
||||||
String appId = "1926887238437818370";
|
|
||||||
String secret = "32df31fcf4fc43f9a647ee1f47134f36";
|
|
||||||
|
|
||||||
List<JSONObject> allUsers = new ArrayList<>();
|
|
||||||
int skip = 0;
|
|
||||||
int totalCount = 0;
|
|
||||||
boolean firstRequest = true;
|
|
||||||
|
|
||||||
do {
|
|
||||||
long ts = System.currentTimeMillis();
|
|
||||||
String sign = Base64.encode(DigestUtil.md5Hex(appId + secret + ts, CharsetUtil.CHARSET_UTF_8));
|
|
||||||
|
|
||||||
HttpRequest httpRequest = HttpUtil.createPost("https://sjzcpt.nnu.edu.cn/cdsp/data-api/v2/DS0040");
|
|
||||||
httpRequest.header("Content-Type", "application/json");
|
|
||||||
httpRequest.header("appId", appId);
|
|
||||||
httpRequest.header("timestamp", String.valueOf(ts));
|
|
||||||
httpRequest.header("sign", sign);
|
|
||||||
|
|
||||||
HashMap<String, Object> reqBody = new HashMap<>();
|
|
||||||
reqBody.put("$count", true);
|
|
||||||
reqBody.put("$skip", skip);
|
|
||||||
reqBody.put("$top", 1000);
|
|
||||||
|
|
||||||
httpRequest.body(JSONUtil.toJsonStr(reqBody));
|
|
||||||
String resBody = httpRequest.execute().body();
|
|
||||||
|
|
||||||
JSONObject entries = JSONUtil.parseObj(resBody);
|
|
||||||
|
|
||||||
if (firstRequest) {
|
|
||||||
totalCount = entries.getInt("@odata.count", 0);
|
|
||||||
firstRequest = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
JSONArray value = entries.getJSONArray("value");
|
|
||||||
|
|
||||||
if (!value.isEmpty()) {
|
|
||||||
List<JSONObject> list = value.stream().map(item -> (JSONObject) item).toList();
|
|
||||||
allUsers.addAll(list);
|
|
||||||
skip += list.size();
|
|
||||||
} else {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
} while (skip < totalCount);
|
|
||||||
|
|
||||||
Map<String, Date> data = allUsers.stream().filter(v -> v.getStr("进站日期") != null).collect(Collectors.toMap(v -> v.getStr("教职工号"), v -> v.getDate("进站日期")));
|
|
||||||
|
|
||||||
return data;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public NutMap searchOptions() {
|
public NutMap searchOptions() {
|
||||||
Sql sql = Sqls.create("""
|
Sql sql = Sqls.create("""
|
||||||
@@ -385,73 +317,8 @@ public class SysDataUserPullServiceImpl extends BaseServiceImpl<Sys_user_source>
|
|||||||
return listMap(sql);
|
return listMap(sql);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Map<String, NutMap> pullFinance() {
|
public Map<String, NutMap> pullFinance() {
|
||||||
// 获取配置文件 接口地址 和 token
|
return null;
|
||||||
DataCenterProperties.Credential credential = dataCenterProperties.credential("finance");
|
|
||||||
String url = credential.getUrl();
|
|
||||||
String token = credential.getToken();
|
|
||||||
|
|
||||||
List<JSONObject> rawDataList = new ArrayList<>();
|
|
||||||
String year = String.valueOf(DateUtil.thisYear());
|
|
||||||
String month = String.valueOf(DateUtil.thisMonth() + 1);
|
|
||||||
|
|
||||||
int pageNum = 1;
|
|
||||||
int pageSize = 1000;
|
|
||||||
int totalPages = 1;
|
|
||||||
int totalCount = 0;
|
|
||||||
do {
|
|
||||||
HttpRequest httpRequest = HttpUtil.createPost(url);
|
|
||||||
httpRequest.header(Header.CONTENT_TYPE, "application/json");
|
|
||||||
httpRequest.header("Accept-Encoding", "gzip, deflate, br");
|
|
||||||
httpRequest.header("X-H3C-TOKEN", token);
|
|
||||||
Map<String, Object> reqBody = Map.of(
|
|
||||||
"nf", year,
|
|
||||||
"yf", month,
|
|
||||||
"pageNum", pageNum,
|
|
||||||
"pageSize", pageSize
|
|
||||||
);
|
|
||||||
httpRequest.body(JSONUtil.toJsonStr(reqBody));
|
|
||||||
|
|
||||||
JSONObject resp = JSONUtil.parseObj(httpRequest.execute().body());
|
|
||||||
if (resp.getInt("code") != 0) {
|
|
||||||
throw new BaseException("获取财务数据失败,错误码: " + resp.getInt("code") + ";错误原因:" + resp.getStr("msg"));
|
|
||||||
}
|
|
||||||
|
|
||||||
JSONObject data = resp.getJSONObject("data");
|
|
||||||
if (pageNum == 1) {
|
|
||||||
totalCount = data.getInt("total", 0);
|
|
||||||
// totalPages = data.getInt("pages", 0);
|
|
||||||
totalPages = (int) Math.ceil((double) totalCount / pageSize);
|
|
||||||
}
|
|
||||||
|
|
||||||
JSONArray records = data.getJSONArray("records");
|
|
||||||
if (CollUtil.isNotEmpty(records)) {
|
|
||||||
rawDataList.addAll(records.stream().map(v -> (JSONObject) v).toList());
|
|
||||||
log.info("数据拉取进度: {}/{}", rawDataList.size(), totalCount);
|
|
||||||
}
|
|
||||||
pageNum++; // 只加页码
|
|
||||||
} while (pageNum <= totalPages);
|
|
||||||
|
|
||||||
// 接口数据输出
|
|
||||||
log.info("数据拉取结果: {}", rawDataList);
|
|
||||||
|
|
||||||
// 最后结果数据
|
|
||||||
Map<String, NutMap> result = new HashMap<>();
|
|
||||||
|
|
||||||
Map<String, String> api2db = Arrays.stream(Api2FinanceFiledMap.values())
|
|
||||||
.collect(Collectors.toMap(e -> e.apiField, e -> e.dbColumn));
|
|
||||||
|
|
||||||
for (JSONObject row : rawDataList) {
|
|
||||||
NutMap nutMap = DataCenterUtil.mapJsonToBean(row, NutMap.class, api2db);
|
|
||||||
result.put(nutMap.getString("loginname"), nutMap);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 输出结果
|
|
||||||
log.info("财务数据拉取转换结果: {}", result);
|
|
||||||
|
|
||||||
log.info("数据拉取完成,共拉取 {} 条数据", result.size());
|
|
||||||
return result;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user