diff --git a/src/main/java/io/v/nutz/data/service/SourceUserService.java b/src/main/java/io/v/nutz/data/service/SourceUserService.java index 8100371..df555e5 100644 --- a/src/main/java/io/v/nutz/data/service/SourceUserService.java +++ b/src/main/java/io/v/nutz/data/service/SourceUserService.java @@ -25,4 +25,8 @@ public interface SourceUserService extends ViService { */ void updateSysUser(String pullTime, String sourceType,String[] columnNames,String isUpSet,String partGroupId,boolean isInvert); + /** + * 每日定时更新 + */ + void dailyScheduledUpdates(); } diff --git a/src/main/java/io/v/nutz/data/service/impl/SourceUserServiceImpl.java b/src/main/java/io/v/nutz/data/service/impl/SourceUserServiceImpl.java index eb7cdd8..5ce8c05 100644 --- a/src/main/java/io/v/nutz/data/service/impl/SourceUserServiceImpl.java +++ b/src/main/java/io/v/nutz/data/service/impl/SourceUserServiceImpl.java @@ -1,6 +1,8 @@ package io.v.nutz.data.service.impl; import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUtil; import cn.hutool.core.util.StrUtil; import io.v.nutz.base.utils.ChineseCharacterStrokeUtil; import io.v.nutz.base.utils.Roles; @@ -39,10 +41,10 @@ import org.nutz.plugins.wkcache.annotation.CacheDefaults; import org.springframework.beans.BeanUtils; import org.springframework.util.StringUtils; +import java.io.UnsupportedEncodingException; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.FutureTask; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -111,7 +113,7 @@ public class SourceUserServiceImpl extends ViServiceImpl implements sources = query(Cnd.where("pullTime", "=", pullTime).groupBy("loginname")); } - Dao filterColumnDao = dao(); + AtomicReference filterColumnDao = new AtomicReference<>(dao()); //全部更新时，要修改的用户表 List needDoUpdateList = new ArrayList<>(); @@ -122,58 +124,79 @@ public class SourceUserServiceImpl extends ViServiceImpl implements //添加角色的用户 List userRoles = new ArrayList<>(); - for (UserSource source : sources) { - source.setMember(null); - source.setWelfareMember(null); - source.setRetired(null); - Sys_user user = userMap.get(source.getLoginname().toLowerCase()); + // 创建一个固定大小的线程池 可以根据服务器性能调整线程池大小 + int numberOfThreads = Runtime.getRuntime().availableProcessors() * 2; + List> futures = new ArrayList<>(); - UserChangeType changeType = getChangeType(source, user); - Sys_user u = new Sys_user(); - BeanUtils.copyProperties(source, u); - switch (sourceType) { - case "all" -> { - //复制属性到一个新的user对象 - u.setId(user == null ? source.getId() : user.getId()); - if (user != null) { - needDoUpdateList.add(u); + // 每批处理的数据量，可以根据实际情况调整 + int batchSize = 200; + + for (int i = 0; i < sources.size(); i += batchSize) { + final int end = Math.min(i + batchSize, sources.size()); + List batch = sources.subList(i, end); + Map finalUserPartMap = userPartMap; + CompletableFuture future = CompletableFuture.runAsync(() -> { + for (UserSource source : batch) { + source.setMember(null); + source.setWelfareMember(null); + source.setRetired(null); + Sys_user user = userMap.get(source.getLoginname().toLowerCase()); + + UserChangeType changeType = getChangeType(source, user); + Sys_user u = new Sys_user(); + BeanUtils.copyProperties(source, u); + switch (sourceType) { + case "all" -> { + //复制属性到一个新的user对象 + u.setId(user == null ? source.getId() : user.getId()); + if (user != null) { + needDoUpdateList.add(u); + } + } + case "add" -> { + u.setId(source.getId()); + Integer strokeCount = 0; + try { + strokeCount = ChineseCharacterStrokeUtil.getStrokeCount(u.getUsername()); + } catch (UnsupportedEncodingException e) { + strokeCount = null; + } + u.setUserNameStroke(strokeCount); + } + case "part" -> { + u.setId(user == null ? source.getId() : user.getId()); + if (columnNames != null && columnNames.length > 0) { + String lockedColumn = String.join("|", columnNames); + FieldFilter fieldFilter = FieldFilter.create(Sys_user.class, null, "^" + lockedColumn + "$", true); + filterColumnDao.set(Daos.ext(dao(), fieldFilter)); + } + if (Strings.isNotBlank(finalUserPartMap.get(u.getLoginname())) && user != null) { + needDoUpdateList.add(u); + } + } + } + + if (changeType != null) { + if (changeType == UserChangeType.NEW) { + user = UserMode.initUser(u); + needInitUserList.add(user); + Sys_user_role userRole = new Sys_user_role(); + userRole.setUserId(user.getId()); + userRole.setRoleId(Roles.PUBLIC); + userRoles.add(userRole); + + histories.add(createUserHistory(user, changeType)); + } else { + histories.add(createUserHistory(user, changeType)); + } } } - case "add" -> { - u.setId(source.getId()); - int strokeCount = ChineseCharacterStrokeUtil.getStrokeCount(u.getUsername()); - u.setUserNameStroke(strokeCount); - } - case "part" -> { - u.setId(user == null ? source.getId() : user.getId()); - if (columnNames != null && columnNames.length > 0) { - String lockedColumn = String.join("|", columnNames); - FieldFilter fieldFilter = FieldFilter.create(Sys_user.class, null, "^" + lockedColumn + "$", true); - filterColumnDao = Daos.ext(dao(), fieldFilter); - } - if (Strings.isNotBlank(userPartMap.get(u.getLoginname())) && user != null) { - needDoUpdateList.add(u); - } - } - } - - if (changeType != null) { - if (changeType == UserChangeType.NEW) { - user = UserMode.initUser(u); - needInitUserList.add(user); - Sys_user_role userRole = new Sys_user_role(); - userRole.setUserId(user.getId()); - userRole.setRoleId(Roles.PUBLIC); - userRoles.add(userRole); - - histories.add(createUserHistory(user, changeType)); - } else { - histories.add(createUserHistory(user, changeType)); - } - } + }, Executors.newFixedThreadPool(numberOfThreads)); + futures.add(future); } -// if (Lang.isNotEmpty(needDoUpdateList)) filterColumnDao.updateIgnoreNull(needDoUpdateList); + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + //如果有新用户，增加到用户表同时增加角色 if (Lang.isNotEmpty(needInitUserList)) { sysUserService.fastInsert(needInitUserList); @@ -181,46 +204,13 @@ public class SourceUserServiceImpl extends ViServiceImpl implements sysUserRoleService.dao().clear(Sys_user_role.class, Cnd.where("userId", "in", userIdList)); sysUserRoleService.insert(userRoles); } - System.out.println("开始》》》》》》》》》》》》》》" + System.currentTimeMillis()); - //全部更新时，修改用户 - - List> updateUserFutures = new ArrayList<>(); - - for (List userList : CollectionUtil.split(needDoUpdateList, 50)) { -// CompletableFuture integerCompletableFuture = CompletableFuture.supplyAsync(() -> -// dao().updateIgnoreNull(userList)); -// updateUserFutures.add(integerCompletableFuture); - - Dao finalFilterColumnDao = filterColumnDao; - new Thread(() -> { - log.info(Thread.currentThread().getName() + "在更新数据"); - if ("part".equals(sourceType)){ - finalFilterColumnDao.updateIgnoreNull(userList); - } else { - dao().updateIgnoreNull(userList); - } - }).start(); + if ("part".equals(sourceType)) { + massUpdatesAsync(needDoUpdateList, true, filterColumnDao); + } else { + massUpdatesAsync(needDoUpdateList, false, null); } -// CompletableFuture.allOf(updateUserFutures.toArray(new CompletableFuture[0])).get(); -// int sum = updateUserFutures.stream().mapToInt(v -> { -// try { -// return v.get(); -// } catch (InterruptedException | ExecutionException e) { -// log.error(e.getMessage()); -// throw new RuntimeException(); -// } -// }).sum(); -// -// if (sum == allDoUpdateList.size()) { -// log.info("全部更新成功" + allDoUpdateList.size() + "条"); -// } else { -// log.error("更新失败"); -// throw new RuntimeException("人员更新数据与期望数据不符合"); -// } - - System.out.println("结束》》》》》》》》》》》》》》" + System.currentTimeMillis()); //历史记录表插入数据 historyUserService.dao().fastInsert(histories); //人员状态及在职状态更新 @@ -232,6 +222,20 @@ public class SourceUserServiceImpl extends ViServiceImpl implements } } + + @Override + public void dailyScheduledUpdates() { + pullSourceData(true); + Date pullTime = (Date) dao().func2(UserSource.class, "max", "pullTime"); + updateSysUser(DateUtil.formatDateTime(pullTime), "add",null,null,null,false); + + DateTime agoWeekDate = DateUtil.offsetWeek(new Date(), -1); + String formatAgoWeekDate = DateUtil.format(agoWeekDate, "yyyy-MM-dd"); + + dao().clear(UserSource.class, Cnd.where("DATE(pullTime)", "<", formatAgoWeekDate)); + } + + private UserChangeType getChangeType(Sys_user source, Sys_user user) { if (user == null) { return UserChangeType.NEW; @@ -261,4 +265,57 @@ public class SourceUserServiceImpl extends ViServiceImpl implements history.setChangeType(changeType.getCode()); return history; } + + + private void massUpdatesAsync(List needDoUpdateList, Boolean isPart, AtomicReference filterColumnDao) { + // 根据服务器性能和任务特性动态确定线程池大小;通常，线程池大小为CPU核心数的2倍 + int numberOfThreads = Runtime.getRuntime().availableProcessors() * 2; + ExecutorService executorService = Executors.newFixedThreadPool(numberOfThreads); + ExecutorCompletionService ecs = new ExecutorCompletionService<>(executorService); + + // 每批处理的数据量 + int batchSize = 200; + + try { + // 将sources分成若干批 + for (int i = 0; i < needDoUpdateList.size(); i += batchSize) { + final int end = Math.min(i + batchSize, needDoUpdateList.size()); + List batch = needDoUpdateList.subList(i, end); + + // 提交任务给线程池执行 + ecs.submit(() -> { + Dao daoToUse = isPart ? filterColumnDao.get() : dao(); + // 执行数据库更新操作 + synchronized (daoToUse) { + daoToUse.updateIgnoreNull(batch); + } + return null; // CompletionService需要一个返回值，这里返回null + }); + } + + // 等待所有任务完成 + for (int i = 0; i < needDoUpdateList.size(); i += batchSize) { + try { + Future future = ecs.take(); + future.get(); // 等待任务完成 + } catch (InterruptedException | ExecutionException e) { + Thread.currentThread().interrupt(); // 恢复中断状态 + throw new RuntimeException(e); + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + // 关闭线程池 + executorService.shutdown(); + try { + // 设置合理的超时时间，防止无限期阻塞 + if (!executorService.awaitTermination(10, TimeUnit.SECONDS)) { + executorService.shutdownNow(); // 如果超时，强制关闭 + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); // 恢复中断状态 + } + } + } } diff --git a/src/main/java/io/v/nutz/fitnessWalk/controller/FitnessWalkStepRankingController.java b/src/main/java/io/v/nutz/fitnessWalk/controller/FitnessWalkStepRankingController.java index 8b897cb..782bb3a 100644 --- a/src/main/java/io/v/nutz/fitnessWalk/controller/FitnessWalkStepRankingController.java +++ b/src/main/java/io/v/nutz/fitnessWalk/controller/FitnessWalkStepRankingController.java @@ -203,7 +203,7 @@ public class FitnessWalkStepRankingController { entityList.add(new ExcelExportEntity("工号", "loginname", 20)); entityList.add(new ExcelExportEntity("分工会", "unionname", 20)); entityList.add(new ExcelExportEntity("单位", "unitname", 20)); - entityList.add(new ExcelExportEntity("today".equals(mode) ? "今日步数" :"活动期间总步数", "step", 20)); + entityList.add(new ExcelExportEntity("today".equals(mode) ? "今日步数" :"活动期间总步数", "total_steps", 20)); entityList.add(new ExcelExportEntity("达标天数(天)","standardsDays",20)); try { ViTool.excelResponse(response, ( "today".equals(mode) ? DateUtil.format(new Date(),"yyyy年MM月dd日") : "活动期间总") + "步数榜.xlsx"); diff --git a/src/main/java/io/v/nutz/fitnessWalk/controller/FitnessWalkStepStatisticsController.java b/src/main/java/io/v/nutz/fitnessWalk/controller/FitnessWalkStepStatisticsController.java index c79ebf7..b644fd8 100644 --- a/src/main/java/io/v/nutz/fitnessWalk/controller/FitnessWalkStepStatisticsController.java +++ b/src/main/java/io/v/nutz/fitnessWalk/controller/FitnessWalkStepStatisticsController.java @@ -29,6 +29,7 @@ import org.nutz.lang.util.NutMap; import org.nutz.mvc.annotation.At; import org.nutz.mvc.annotation.Ok; import org.nutz.mvc.annotation.Param; + import javax.servlet.http.HttpServletResponse; import java.util.*; @@ -90,19 +91,21 @@ public class FitnessWalkStepStatisticsController { cnd.and("date_format(al.applyDate,'%Y-%m-%d')", ">=", activityDate[0]); cnd.and("date_format(al.applyDate,'%Y-%m-%d')", "<=", activityDate[1]); } - if(StrUtil.isNotBlank(pageForm.getSearchKeyword())){ + if (StrUtil.isNotBlank(pageForm.getSearchKeyword())) { SqlExpressionGroup seg = new SqlExpressionGroup(); - seg.orLike("u.username",pageForm.getSearchKeyword()); - seg.orLike("u.loginname",pageForm.getSearchKeyword()); + seg.orLike("u.username", pageForm.getSearchKeyword()); + seg.orLike("u.loginname", pageForm.getSearchKeyword()); cnd.and(seg); } - cnd.and("u.id","in",Sqls.create("select userId from activity_user_scope where groupId = @groupId").setParam("groupId",activity.getGroupId())); + cnd.and("u.id", "in", Sqls.create("select userId from activity_user_scope where groupId = @groupId").setParam("groupId", activity.getGroupId())); cnd.andEX("YEAR(al.applyDate)", "=", year); cnd.andEX("al.activityId", "=", activityId); cnd.andEX("u.unionid", "=", unionId); cnd.andEX("u.unitId", "=", unitId); + cnd.desc("al.userId"); cnd.desc("al.applyDate"); cnd.desc("u.unioncode"); + cnd.groupBy("al.applyDate", "al.userId"); sql.setCondition(cnd); return baseService.listPageMap(pageForm.getPageNumber(), pageForm.getPageSize(), sql); @@ -132,9 +135,11 @@ public class FitnessWalkStepStatisticsController { cnd.andEX("al.activityId", "=", activityId); cnd.andEX("u.unionid", "=", unionId); cnd.andEX("u.unitId", "=", unitId); - cnd.and("u.id","in",Sqls.create("select userId from activity_user_scope where groupId = @groupId").setParam("groupId",activity.getGroupId())); + cnd.and("u.id", "in", Sqls.create("select userId from activity_user_scope where groupId = @groupId").setParam("groupId", activity.getGroupId())); + cnd.desc("al.userId"); cnd.desc("al.applyDate"); cnd.desc("u.unioncode"); + cnd.groupBy("al.applyDate", "al.userId"); sql.setCondition(cnd); List exportList = baseService.listMap(sql); diff --git a/src/main/java/io/v/nutz/fitnessWalk/service/impl/FitnessWalkCommonServiceImpl.java b/src/main/java/io/v/nutz/fitnessWalk/service/impl/FitnessWalkCommonServiceImpl.java index 63778ae..6e011e0 100644 --- a/src/main/java/io/v/nutz/fitnessWalk/service/impl/FitnessWalkCommonServiceImpl.java +++ b/src/main/java/io/v/nutz/fitnessWalk/service/impl/FitnessWalkCommonServiceImpl.java @@ -384,8 +384,8 @@ public class FitnessWalkCommonServiceImpl extends ViServiceImpl implements Fitne Sql sql = Sqls.create(""" SELECT - s.userId, - SUM(s.step) AS step, + u.id, + COALESCE(SUM(sub.step), 0) AS total_steps, u.username, u.loginname, u.unionname, @@ -393,7 +393,21 @@ public class FitnessWalkCommonServiceImpl extends ViServiceImpl implements Fitne over3k.standardsDays FROM `user` u - LEFT JOIN `fitness_walk_step` s ON u.id = s.userId + LEFT JOIN ( + SELECT + userId, + applyDate, + step + FROM + `fitness_walk_step` + WHERE + step >= @step + AND activityId=@activityId + AND $dateSql + GROUP BY + userId, + applyDate + ) AS sub ON u.id = sub.userId LEFT JOIN ( SELECT userId, @@ -402,7 +416,8 @@ public class FitnessWalkCommonServiceImpl extends ViServiceImpl implements Fitne `fitness_walk_step` WHERE step >= @step - AND activityId=@activityId and $dateSql + AND activityId=@activityId + AND $dateSql GROUP BY userId ) AS over3k ON u.id = over3k.userId @@ -411,17 +426,16 @@ public class FitnessWalkCommonServiceImpl extends ViServiceImpl implements Fitne if ("today".equals(mode)) { sql.setVar("dateSql",new Static("DATE(applyDate) = '"+DateUtil.today()+"'")); - cnd.and("DATE( applyDate )", "=", DateUtil.today()); +// cnd.and("DATE( s.applyDate )", "=", DateUtil.today()); } else { sql.setVar("dateSql",new Static("DATE(applyDate) >= '" +DateUtil.format(DateUtil.date(activity.getStartTime()), "yyyy-MM-dd")+"' and DATE(applyDate) <= '" +DateUtil.format(DateUtil.date(activity.getEndTime()), "yyyy-MM-dd")+"'")); - cnd.and("DATE( applyDate )", ">=", DateUtil.format(DateUtil.date(activity.getStartTime()), "yyyy-MM-dd")); - cnd.and("DATE( applyDate )", "<=", DateUtil.format(DateUtil.date(activity.getEndTime()), "yyyy-MM-dd")); +// cnd.and("DATE( s.applyDate )", ">=", DateUtil.format(DateUtil.date(activity.getStartTime()), "yyyy-MM-dd")); +// cnd.and("DATE( s.applyDate )", "<=", DateUtil.format(DateUtil.date(activity.getEndTime()), "yyyy-MM-dd")); } - cnd.and("s.activityId", "=", activityId); if (Strings.isNotBlank(unitId)) { cnd.and("u.unitId", "=", unitId); } @@ -433,8 +447,8 @@ public class FitnessWalkCommonServiceImpl extends ViServiceImpl implements Fitne cnd.and(searchName, "LIKE", "%" + searchKeyword + "%"); } cnd.and("u.id","in",Sqls.create("select userId from activity_user_scope where groupId = @groupId").setParam("groupId",activity.getGroupId())); - cnd.groupBy("s.userId"); - cnd.desc("s.step"); + cnd.groupBy("u.id","u.username","u.loginname","u.unionname","u.unitname","over3k.standardsDays"); + cnd.desc("total_steps"); sql.setCondition(cnd); return sql; } diff --git a/src/main/java/io/v/nutz/task/job/UserUpdateJobTask.java b/src/main/java/io/v/nutz/task/job/UserUpdateJobTask.java index 9fc8b03..e92a805 100644 --- a/src/main/java/io/v/nutz/task/job/UserUpdateJobTask.java +++ b/src/main/java/io/v/nutz/task/job/UserUpdateJobTask.java @@ -30,17 +30,6 @@ public class UserUpdateJobTask implements Job { @Override public void execute(JobExecutionContext context) throws JobExecutionException { - try { - sourceUserService.pullSourceData(true); - Date pullTime = (Date) dao.func2(UserSource.class, "max", "pullTime"); - sourceUserService.updateSysUser(DateUtil.formatDateTime(pullTime), "add",null,null,null,false); - - DateTime agoWeekDate = DateUtil.offsetWeek(new Date(), -1); - String formatAgoWeekDate = DateUtil.format(agoWeekDate, "yyyy-MM-dd"); - sourceUserService.clear(Cnd.where("DATE(pullTime)","<",formatAgoWeekDate)); - } catch (Exception e) { - e.printStackTrace(); - } - + sourceUserService.dailyScheduledUpdates(); } } diff --git a/src/main/java/io/v/nutz/task/job/fitnessWalk/FitnessWalkLotteryJob.java b/src/main/java/io/v/nutz/task/job/fitnessWalk/FitnessWalkLotteryJob.java index b1a39d5..3d52807 100644 --- a/src/main/java/io/v/nutz/task/job/fitnessWalk/FitnessWalkLotteryJob.java +++ b/src/main/java/io/v/nutz/task/job/fitnessWalk/FitnessWalkLotteryJob.java @@ -69,34 +69,63 @@ public class FitnessWalkLotteryJob implements Job { //查询出符合条件的用户 Sql sql = Sqls.create(""" SELECT - jws.userId, - u.loginname, - u.username, - u.unionid, - u.unionname, - u.unitid, - u.unitname + u.id, + COALESCE(SUM(sub.step), 0) AS total_steps, + u.loginname, + u.username, + u.unionid, + u.unionname, + u.unitid, + u.unitname FROM - fitness_walk_step jws - LEFT JOIN `user` u ON u.id = jws.userId - WHERE - jws.activityId = @activityId - AND DATE ( jws.applyDate ) >= @startDate - AND DATE ( jws.applyDate ) <= @endDate - $lotteryQualificationDaySql + `user` u + LEFT JOIN ( + SELECT + userId, + applyDate, + step + FROM + `fitness_walk_step` + WHERE + activityId = @activityId + AND DATE(applyDate) >= @startDate + AND DATE(applyDate) <= @endDate + $lotteryQualificationDaySql + GROUP BY + userId, + applyDate + ) AS sub ON u.id = sub.userId + LEFT JOIN ( + SELECT + userId, + COUNT(DISTINCT DATE(applyDate)) AS standardsDays + FROM + `fitness_walk_step` + WHERE + activityId = @activityId + AND DATE(applyDate) >= @startDate + AND DATE(applyDate) <= @endDate + $lotteryQualificationDaySql + GROUP BY + userId + ) AS over3k ON u.id = over3k.userId GROUP BY - jws.userId + u.id, + u.username, + u.loginname, + u.unionname, + u.unitname HAVING $stepHavingSql ORDER BY - rand() - LIMIT @lotteryUserNum + rand() + LIMIT @lotteryUserNum """); if (Strings.isNotBlank(lotteryQualificationDay)) { - sql.setVar("lotteryQualificationDaySql", new Static(" AND jws.step > '%s' ".formatted(lotteryQualificationStep))); - sql.setVar("stepHavingSql", new Static(" COUNT( jws.applyDate ) > '%s' ".formatted(lotteryQualificationDay))); + sql.setVar("lotteryQualificationDaySql", new Static(" AND step > '%s' ".formatted(lotteryQualificationStep))); + sql.setVar("stepHavingSql", new Static(" over3k.standardsDays > '%s' ".formatted(lotteryQualificationDay))); } else { - sql.setVar("stepHavingSql", new Static(" SUM( jws.step ) > '%s' ".formatted(lotteryQualificationStep))); + sql.setVar("stepHavingSql", new Static(" COALESCE(SUM(sub.step), 0) > '%s' ".formatted(lotteryQualificationStep))); } sql.setParam("activityId", activityId); sql.setParam("step", lotteryQualificationStep); diff --git a/src/main/resources/views/platform/fitnessWalk/stepRanking.html b/src/main/resources/views/platform/fitnessWalk/stepRanking.html index 01c1ad0..b0c4bf2 100644 --- a/src/main/resources/views/platform/fitnessWalk/stepRanking.html +++ b/src/main/resources/views/platform/fitnessWalk/stepRanking.html @@ -141,7 +141,7 @@ layout("/layouts/platform.html"){ {label: '工号', prop: 'loginname'}, {label: '分工会', prop: 'unionname', sortable: true}, {label: '单位', prop: 'unitname'}, - {label: '步数', prop: 'step', sortable: true}, + {label: '步数', prop: 'total_steps', sortable: true}, {label: '达标天数', prop: 'standardsDays', sortable: true}, ], }