This commit is contained in:
Paidax
2024-12-13 09:37:20 +08:00
parent 014beea7d4
commit c418e37618
20 changed files with 2043 additions and 230 deletions
@@ -0,0 +1,125 @@
package io.v.nutz.base.config;
import lombok.extern.slf4j.Slf4j;
import org.nutz.dao.Dao;
import org.nutz.ioc.loader.annotation.Inject;
import org.nutz.ioc.loader.annotation.IocBean;
import org.nutz.lang.Lang;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.List;
import java.util.concurrent.ThreadPoolExecutor;
/**
* @version 1.0
* @Author zzr
* @nameThreadPoolConfig
* @Date 2024/12/6 14:35
* @注释
*/
@Slf4j
@IocBean
public class ThreadPoolConfig {
@Inject
private Dao dao;
/**
* 核心线程数(默认线程数)
*/
private static final int CORE_POOL_SIZE = 2 * Runtime.getRuntime().availableProcessors() + 1;
/**
* 最大线程数
*/
private static final int MAX_POOL_SIZE = 128;
/**
* 允许线程空闲时间(单位:默认为秒)
*/
private static final int KEEP_ALIVE_TIME = 5;
/**
* 任务的等待时间
*/
private static final int AWAIT_TERMINATION_TIME = 30;
/**
* 缓冲队列数
*/
private static final int QUEUE_CAPACITY = 1200;
/**
* 线程池名前缀
*/
private static final String THREAD_NAME_PREFIX = "dd3s-thread-pool";
/**
* bean的名称,默认为首字母小写的方法名
* spring管理的线程池,顶级父类也是Executor
*/
@IocBean(name = "executorService")
public ThreadPoolTaskExecutor executorService() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(CORE_POOL_SIZE);
executor.setMaxPoolSize(MAX_POOL_SIZE);
executor.setQueueCapacity(QUEUE_CAPACITY);
executor.setKeepAliveSeconds(KEEP_ALIVE_TIME);
executor.setThreadNamePrefix(THREAD_NAME_PREFIX);
executor.setAwaitTerminationSeconds(AWAIT_TERMINATION_TIME);
executor.setWaitForTasksToCompleteOnShutdown(true);
// 线程池对拒绝任务的处理策略
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
executor.initialize();
return executor;
}
/**
* 异步执行
*
* @param list 泛型list,任意实体类集合
* @param batchSize 期望单次操作的数量,默认200
* @param mode insert插入 update更新 insertOrUpdate插入或更新
* @param isFastInsertOrUpdateIgnoreNull mode为新增此值为true代表fastInsertmode为修改时此值为true代表updateIgnoreNull
* @param <T>
*/
public <T> void asyncExecute(List<T> list, Integer batchSize, String mode, boolean isFastInsertOrUpdateIgnoreNull) {
if (Lang.isEmpty(list)) {
return;
}
if (batchSize == null || batchSize <= 0) {
batchSize = 200;
}
for (int i = 0; i < list.size(); i += batchSize) {
final int end = Math.min(i + batchSize, list.size());
List<T> batch = list.subList(i, end);
executorService().execute(() -> {
try {
switch(mode) {
case "insert" -> {
if (isFastInsertOrUpdateIgnoreNull) {
dao.fastInsert(batch);
} else {
dao.insert(batch);
}
}
case "update" -> {
if (isFastInsertOrUpdateIgnoreNull) {
dao.updateIgnoreNull(batch);
} else {
dao.update(batch);
}
}
case "insertOrUpdate" -> dao.insertOrUpdate(batch);
}
} catch (Exception e) {
// 日志记录异常信息
System.err.println("Error occurred while inserting batch: " + e.getMessage());
e.printStackTrace();
}
});
}
}
}
@@ -0,0 +1,82 @@
package io.v.nutz.base.utils;
import io.v.nutz.base.config.ThreadPoolConfig;
import org.nutz.ioc.loader.annotation.Inject;
import org.nutz.ioc.loader.annotation.IocBean;
import java.util.List;
/**
* @version 1.0
* @Author zzr
* @nameManyAddOrRenewUtil
* @Date 2024/7/24 16:16
* @注释
*/
@IocBean
public class ManyAddOrRenewUtil {
@Inject
private ThreadPoolConfig threadPoolConfig;
/**
* 批量快速插入,异步执行
*
* @param list 泛型list,任意实体类集合
* @param batchSize 期望单次操作的数量,默认200
* @param <T>
*/
public <T> void asyncExecuteFastInsert(List<T> list, Integer batchSize) {
threadPoolConfig.asyncExecute(list, batchSize, "insert", true);
}
/**
* 批量插入(非快速),异步执行
*
* @param list 泛型list,任意实体类集合
* @param batchSize 期望单次操作的数量,默认200
* @param <T>
*/
public <T> void asyncExecuteInsert(List<T> list, Integer batchSize) {
threadPoolConfig.asyncExecute(list, batchSize, "insert", true);
}
/**
* 批量修改(忽略空值),异步执行
*
* @param list 泛型list,任意实体类集合
* @param batchSize 期望单次操作的数量,默认200
* @param <T>
*/
public <T> void asyncExecuteUpdateIgnoreNull(List<T> list, Integer batchSize) {
threadPoolConfig.asyncExecute(list, batchSize, "update", true);
}
/**
* 批量修改(不忽略空值),异步执行
*
* @param list 泛型list,任意实体类集合
* @param batchSize 期望单次操作的数量,默认200
* @param <T>
*/
public <T> void asyncExecuteUpdate(List<T> list, Integer batchSize) {
threadPoolConfig.asyncExecute(list, batchSize, "update", false);
}
/**
* 新增或修改,异步执行
*
* @param list 泛型list,任意实体类集合
* @param batchSize 期望单次操作的数量,默认200
* @param <T>
*/
public <T> void asyncExecuteInsertOrUpdate(List<T> list, Integer batchSize) {
threadPoolConfig.asyncExecute(list, batchSize, "insertOrUpdate", false);
}
}
@@ -110,6 +110,7 @@ public class SysUnitMangeController {
public Object pageData(@Param(value = "parentId", required = false) String parentId, PageForm pageForm) {
CndPlus cnd = CndPlus.create();
cnd.andEx("parentId", "=", parentId);
cnd.asc("location");
return sysUnitService.list(pageForm, cnd);
}
@@ -211,7 +212,7 @@ public class SysUnitMangeController {
private AsyncService asyncService;
private List<Sys_unit> child(String parentId) {
List<Sys_unit> units = sysUnitService.query(Cnd.where("parentId", "=", parentId));
List<Sys_unit> units = sysUnitService.query(Cnd.where("parentId", "=", parentId).asc("location"));
units.forEach(unit -> {
unit.setChild(child(unit.getId()));
@@ -13,4 +13,6 @@ public interface UserPatUpService extends ViService<UserPartUp> {
void largeDataInsert(List<UserPartUp> list);
void renewUserState();
void deleteNotInSourceUser();
}
@@ -3,12 +3,18 @@ package io.v.nutz.web.commons.controller.userpartupdate.service.impl;
import cn.hutool.core.collection.CollectionUtil;
import io.v.nutz.base.service.impl.ViServiceImpl;
import io.v.nutz.base.utils.DateUtil;
import io.v.nutz.base.utils.Roles;
import io.v.nutz.sys.models.Sys_user;
import io.v.nutz.sys.models.Sys_user_role;
import io.v.nutz.web.commons.controller.userpartupdate.models.UserPartUp;
import io.v.nutz.web.commons.controller.userpartupdate.service.UserPatUpService;
import io.v.nutz.sys.models.Sys_dict;
import io.v.nutz.sys.services.SysDictService;
import io.v.nutz.zhgh.data.model.UserSource;
import lombok.extern.slf4j.Slf4j;
import org.nutz.aop.interceptor.ioc.TransAop;
import org.nutz.dao.Chain;
import org.nutz.dao.Cnd;
import org.nutz.dao.Dao;
import org.nutz.dao.Sqls;
import org.nutz.dao.impl.NutTxDao;
@@ -17,6 +23,7 @@ import org.nutz.ioc.aop.Aop;
import org.nutz.ioc.loader.annotation.Inject;
import org.nutz.ioc.loader.annotation.IocBean;
import org.nutz.lang.Lang;
import org.nutz.lang.util.NutMap;
import java.util.ArrayList;
import java.util.List;
@@ -143,4 +150,36 @@ public class UserPartUpServiceImpl extends ViServiceImpl<UserPartUp> implements
sysDictService.clearCache();
}
@Override
@Aop(TransAop.READ_COMMITTED)
public void deleteNotInSourceUser() {
Sql maxPullTimeSql = Sqls.create("select pullTime from user_source GROUP BY pullTime ORDER BY pullTime desc LIMIT 1");
maxPullTimeSql.setCallback(Sqls.callback.map());
dao().execute(maxPullTimeSql);
NutMap map = (NutMap) maxPullTimeSql.getResult();
// 修改不在人事库的人员状态
Sql querySql = Sqls.create("select id,loginname from sys_user where loginname not in (select loginname from user_source where pullTime = @pullTime) and loginname != 'superadmin'")
.setParam("pullTime", map.getString("pullTime"));
querySql.setCallback(Sqls.callback.maps());
dao().execute(querySql);
List<NutMap> list = (List<NutMap>) querySql.getResult();
if (Lang.isEmpty(list)) {
return;
}
List<String> userIdList = list.stream().map(v -> v.getString("id")).toList();
List<String> userLoginNameList = list.stream().map(v -> v.getString("loginname")).toList();
// 修改不在人事库人员的状态
dao().update(Sys_user.class, Chain.make("userState", "非人事库人员").add("personType", "非人事库人员")
.add("member", 0).add("welfareMember", 0),
Cnd.where("loginname", "in", userLoginNameList));
// 去除不在人事库的人员角色,只保留公共角色
dao().clear(Sys_user_role.class, Cnd.where("userId", "in", userIdList).and("roleId", "!=", Roles.PUBLIC));
}
}
@@ -0,0 +1,78 @@
package io.v.nutz.web.commons.ext.beetl;
import org.beetl.core.Resource;
import org.beetl.core.misc.BeetlUtil;
import org.beetl.core.resource.ClasspathResource;
import org.beetl.core.resource.ClasspathResourceLoader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
public class BeetlCustomResourceLoader extends ClasspathResourceLoader {
public Resource getResource(String key) {
Resource resource = new ClasspathResource(key, this.getChildPath(super.getRoot(), key), this);
return resource;
}
public String getResourceId(Resource resource, String id) {
// return resource == null ? id : BeetlUtil.getRelPath(resource.getId(), id);
return getRelPath(resource.getId(), id);
}
/**
* 增加判断相对路径
* 光见鬼
*/
private String getRelPath(String siblings, String resourceId) {
// 参数校验
if (resourceId == null || resourceId.isEmpty()) {
throw new RuntimeException("资源ID为空,参数错");
}
// 如果是绝对路径,直接返回(以 / 或 \ 开头)
if (resourceId.charAt(0) == '\\' || resourceId.charAt(0) == '/') {
if (BeetlUtil.isOutsideOfRoot(resourceId)) {
throw new RuntimeException("不能访问外部文件或者模板");
}
return resourceId;
}
// 处理相对路径
String baseDir = siblings;
// 如果siblings是文件路径,获取其所在目录
int lastSeparatorIndex = Math.max(siblings.lastIndexOf('/'), siblings.lastIndexOf('\\'));
if (lastSeparatorIndex > 0) {
baseDir = siblings.substring(0, lastSeparatorIndex + 1);
}
// 分割路径
String[] parts = resourceId.replace('\\', '/').split("/");
List<String> baseParts = new ArrayList<>(
Arrays.asList(baseDir.replace('\\', '/').split("/"))
);
baseParts.removeIf(String::isEmpty);
// 处理 ../ 和 ./
for (String part : parts) {
if ("..".equals(part)) {
if (!baseParts.isEmpty()) {
baseParts.remove(baseParts.size() - 1);
}
} else if (!".".equals(part) && !part.isEmpty()) {
baseParts.add(part);
}
}
// 组合最终路径
String result = "/" + String.join("/", baseParts);
// 检查是否访问外部文件
if (BeetlUtil.isOutsideOfRoot(result)) {
throw new RuntimeException("不能访问外部文件或者模板");
}
return result;
}
}
@@ -308,7 +308,7 @@ public class ActivityBasicScopeController {
COLUMN_COMMENT
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = 'sys_user'
AND TABLE_SCHEMA = 'zhgh_jiangnan'
AND TABLE_SCHEMA = 'zhgh_hmc'
""");
List<NutMap> sqlDataList = activityBasicScopeService.listMap(sql);
// sqlDataList.forEach(v -> v.put("DATA_TYPE", new String((byte[]) v.get("DATA_TYPE"))));
@@ -62,10 +62,6 @@ public class HMCFieldCode {
* 学位码
*/
static Map<String, String> ACADEMIC_DEGREE_CODE = new HashMap<>() {{
put("260", "医学博士专业学位");
put("270", "理学博士专业学位");
put("390", "医学硕士专业学位");
put("391", "理学硕士专业学位");
put("2", "博士");
put("201", "哲学博士学位");
put("202", "经济学博士学位");
@@ -80,9 +76,10 @@ public class HMCFieldCode {
put("211", "军事学博士学位");
put("212", "管理学博士学位");
put("245", "临床医学博士专业学位");
put("380", "农业硕士专业学位");
put("248", "兽医博士专业学位");
put("250", "口腔医学博士专业学位");
put("260", "医学博士专业学位");
put("270", "理学博士专业学位");
put("3", "硕士");
put("301", "哲学硕士学位");
put("302", "经济学硕士学位");
@@ -108,6 +105,9 @@ public class HMCFieldCode {
put("350", "口腔医学硕士专业学位");
put("351", "公共卫生硕士专业学位");
put("352", "军事硕士专业学位");
put("380", "农业硕士专业学位");
put("390", "医学硕士专业学位");
put("391", "理学硕士专业学位");
put("4", "学士");
put("401", "哲学学士学位");
put("402", "经济学学士学位");
@@ -27,7 +27,7 @@ import java.util.Map;
*
* @author 1V
* @date 2021/3/4
* @since 浙江财经
* @since 杭州医学院
*/
public interface SourceData {
@@ -45,13 +45,11 @@ public interface SourceData {
// Assert.isTrue(jsonResult.getBoolean("success"), jsonResult.getString("data"));
}
/* *//**
* 在职状态码
*//*
Map<String, String> USER_STATE_CODE = new HashMap<>() {{
put("100", "在职状态");
put("200", "不在职状态");
}};*/
/**
* 不进系统人员
*/
List<String> NOT_ENTERING_USER = List.of("退休人员", "兼职教师", "非全日用工(项目临聘)", "服务外包和其他人员",
"教学编制2", "教学教师(附属医院)", "教学编制", "离休人员", "临时人员", "非全日制工", "博士后(在职)");
/**
* 接口中的数据跟SourceUser的对应关系
@@ -116,6 +114,9 @@ public interface SourceData {
// checkSuccess(map);
List<NutMap> data = map.getAsList("data", NutMap.class);
for (NutMap row : data) {
if (NOT_ENTERING_USER.contains(row.getString("RYFLMC"))) {
continue;
}
Map entity = new HashMap(500);
row.forEach((k, v) -> {
@@ -148,6 +149,7 @@ public interface SourceData {
put("DWH", new String[]{"id", "unitcode"});
put("DWMC", new String[]{"name"});
put("SJDWH", new String[]{"parentId"});
put("PX", new String[]{"location"});
}};
/**
@@ -3,6 +3,7 @@ package io.v.nutz.zhgh.data.service.impl;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.StrUtil;
import io.v.nutz.base.utils.ManyAddOrRenewUtil;
import io.v.nutz.base.utils.Roles;
import io.v.nutz.sys.models.Sys_user_role;
import io.v.nutz.sys.services.SysUserRoleService;
@@ -59,22 +60,23 @@ public class SourceUserServiceImpl extends ViServiceImpl<UserSource> implements
super(dao);
}
// 是会员的人员类型
private final List<String> MEMBER_PERSON_TYPE = List.of("事业编制(员额)", "事业编制", "校聘编外", "同工同酬", "学校编制", "博士后(统招)");
@Inject
private AsyncService asyncService;
@Inject
private SysUserService sysUserService;
@Inject
private HistoryUserService historyUserService;
@Inject
private UserPatUpService userPatUpService;
@Inject
private SysRoleService sysRoleService;
@Inject
private SysUserRoleService sysUserRoleService;
@Inject
private ManyAddOrRenewUtil manyAddOrRenewUtil;
@Override
@Aop(TransAop.READ_COMMITTED)
@@ -97,7 +99,7 @@ public class SourceUserServiceImpl extends ViServiceImpl<UserSource> implements
List<UserSource> sources = new ArrayList<>();
List<UserHistory> histories = new ArrayList<>();
List<UserHistory> histories = new CopyOnWriteArrayList<>();
Map<String, String> userPartMap = new HashMap<>();
if (Strings.isNotBlank(partGroupId)) {
@@ -117,16 +119,20 @@ public class SourceUserServiceImpl extends ViServiceImpl<UserSource> implements
AtomicReference<Dao> filterColumnDao = new AtomicReference<>(dao());
//全部更新时,要修改的用户表
List<Sys_user> needDoUpdateList = new ArrayList<>();
//新增的用户
List<Sys_user> needInitUserList = new ArrayList<>();
//添加角色的用户
List<Sys_user_role> userRoles = new ArrayList<>();
// 全部更新时,要修改的用户表
List<Sys_user> needDoUpdateList = new CopyOnWriteArrayList<>();
// 新增的用户
List<Sys_user> needInitUserList = new CopyOnWriteArrayList<>();
// 添加角色的用户
List<Sys_user_role> userRoles = new CopyOnWriteArrayList<>();
// 添加会员角色
List<String> addMemberUserIds = new CopyOnWriteArrayList<>();
// 移除会员角色
List<String> deleteMemberUserIds = new CopyOnWriteArrayList<>();
// 创建一个固定大小的线程池 可以根据服务器性能调整线程池大小
int numberOfThreads = Runtime.getRuntime().availableProcessors() * 2;
List<CompletableFuture<Void>> futures = new ArrayList<>();
List<CompletableFuture<Void>> futures = new CopyOnWriteArrayList<>();
// 每批处理的数据量,可以根据实际情况调整
int batchSize = 200;
@@ -142,11 +148,25 @@ public class SourceUserServiceImpl extends ViServiceImpl<UserSource> implements
source.setWelfareMember(null);
Sys_user user = userMap.get(source.getLoginname().toLowerCase());
if (MEMBER_PERSON_TYPE.contains(source.getPersonType()) && "在职".equals(source.getUserState())) {
source.setMember(1);
source.setWelfareMember(1);
if (Lang.isNotEmpty(user) && user.getMember() == 1) {
} else {
addMemberUserIds.add(Lang.isNotEmpty(user) ? user.getId() : source.getId());
}
} else {
source.setMember(0);
source.setWelfareMember(0);
deleteMemberUserIds.add(Lang.isNotEmpty(user) ? user.getId() : source.getId());
}
Sys_user u = new Sys_user();
BeanUtils.copyProperties(source, u);
switch (sourceType) {
case "all" -> {
//复制属性到一个新的user对象
// 复制属性到一个新的user对象
u.setId(user == null ? source.getId() : user.getId());
if (user != null) {
needDoUpdateList.add(u);
@@ -195,6 +215,22 @@ public class SourceUserServiceImpl extends ViServiceImpl<UserSource> implements
sysUserRoleService.insert(userRoles);
}
// 添加会员角色
if (Lang.isNotEmpty(addMemberUserIds)) {
String memberRoleId = Roles.MEMBER;
List<Sys_user_role> list = addMemberUserIds.stream().map(v -> {
Sys_user_role userRole = new Sys_user_role();
userRole.setUserId(v);
userRole.setRoleId(memberRoleId);
return userRole;
}).toList();
manyAddOrRenewUtil.asyncExecuteInsert(list, 200);
}
// 不是定义的personType的人员,去除会员角色
if (Lang.isNotEmpty(deleteMemberUserIds)) {
dao().clear(Sys_user_role.class, Cnd.where("userId", "in", deleteMemberUserIds).and("roleId", "=", Roles.MEMBER));
}
// Collection<String[]> updateColumns = SourceData.USER_FIELD_RELATION.values();
if ("part".equals(sourceType)) {
@@ -203,10 +239,12 @@ public class SourceUserServiceImpl extends ViServiceImpl<UserSource> implements
massUpdatesAsync(needDoUpdateList, false, null);
}
//历史记录表插入数据
// 历史记录表插入数据
historyUserService.dao().fastInsert(histories);
//人员状态及在职状态更新
// 人员状态及在职状态更新
userPatUpService.renewUserState();
// 修改不在人事库的人员,删除角色
userPatUpService.deleteNotInSourceUser();
//清除缓存
// sysUserService.clearCache();
// sysRoleService.clearCache();
@@ -17,7 +17,7 @@ import org.nutz.mvc.annotation.Ok;
public class FundsReimbursementViewController {
@At("")
@Ok("beetl:/platform/jf/funds/fundsReimbursementview.html")
@Ok("beetl:/platform/jf/Funds/fundsReimbursementview.html")
@RequiresPermissions("sys.jf.funds.reimbursementview")
public void index() {
}
@@ -455,59 +455,6 @@ public class MemberInquireIntegrateController {
@Param(value = "memberSearchKeyWord", required = false) String memberSearchKeyWord,
String props) {
try {
// int yyyy = Calendar.getInstance().get(Calendar.YEAR);
// CndPlus cnd = CndPlus.create();
// Sql sql = Sqls.create("""
// SELECT
// *,
// unitname,
// unionname,
// unitcode,
// unioncode
// FROM
// $table
// $condition
// """);
//
// if (year == yyyy) {
// sql.setVar("table", "user");
// } else {
// sql.setVar("table", "member_his");
// cnd.andEX("year", "=", year);
// }
//
// cnd.and("member", "=", 1);
// if (ShiroUtil.hasAnyRoles("sysadmin,A06,H03")) {
// cnd.andEX("unionid", "=", unionId);
// } else {
// cnd.andEX("unionid", "=", Vi.getUnionId());
// }
// cnd.andEX("unitid", "=", unitId);
// if (memberStatus != null) {
// if (Json.fromJsonAsArray(Integer.class, memberStatus).length > 0) {
// cnd.andEX("memberStatus", "in", Json.fromJsonAsArray(Integer.class, memberStatus));
// } else {
// List<Integer> status = new ArrayList<>();
// status.add(MemberStatus.NORMAL.getCode());
// status.add(MemberStatus.TURN_IN.getCode());
// status.add(MemberStatus.RESTORE.getCode());
// SqlExpressionGroup group = new SqlExpressionGroup();
// group.and("memberStatus", "in", status);
// group.or("memberStatus", "IS", null);
// cnd.and(group);
// }
// }
//
// if (personTypes != null && Json.fromJsonAsArray(String.class, personTypes).length > 0) {
// cnd.andEX("personType", "in", Json.fromJsonAsArray(String.class, personTypes));
// }
// if (userStates != null && Json.fromJsonAsArray(String.class, userStates).length > 0) {
// cnd.andEX("userState", "in", Json.fromJsonAsArray(String.class, userStates));
// }
// cnd.asc("unitcode");
// cnd.asc("unioncode");
// cnd.asc("memberJoinTime");
// sql.setCondition(cnd);
int yyyy = Calendar.getInstance().get(Calendar.YEAR);
Cnd cnd = MemberUtils.getCnd(null, startDate, endDate, unionId, unitId, personTypes, userStates, memberTypes, sexTypes, age, null, roleIds, null, null, memberStatus, threeUnitId, unionGroupId, reverseSelection, null, null, campus);