c-zhouhf1/zhgh_jvic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

commit

Browse Source
This commit is contained in:
Paidax
2024-07-17 16:12:27 +08:00
parent 4e2c0dc558
commit 730b5d38cc
8 changed files with 235 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/io/v/nutz/data/service/SourceUserService.java
+4
View File
@@ -25,4 +25,8 @@ public interface SourceUserService extends ViService<UserSource> {
*/
void updateSysUser(String pullTime, String sourceType,String[] columnNames,String isUpSet,String partGroupId,boolean isInvert);
/**
* 每日定时更新
*/
void dailyScheduledUpdates();
}
src/main/java/io/v/nutz/data/service/impl/SourceUserServiceImpl.java
+145 -88
View File
@@ -1,6 +1,8 @@
package io.v.nutz.data.service.impl;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.StrUtil;
import io.v.nutz.base.utils.ChineseCharacterStrokeUtil;
import io.v.nutz.base.utils.Roles;
@@ -39,10 +41,10 @@ import org.nutz.plugins.wkcache.annotation.CacheDefaults;
import org.springframework.beans.BeanUtils;
import org.springframework.util.StringUtils;
import java.io.UnsupportedEncodingException;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.FutureTask;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
@@ -111,7 +113,7 @@ public class SourceUserServiceImpl extends ViServiceImpl<UserSource> implements
sources = query(Cnd.where("pullTime", "=", pullTime).groupBy("loginname"));
}
Dao filterColumnDao = dao();
AtomicReference<Dao> filterColumnDao = new AtomicReference<>(dao());
//全部更新时，要修改的用户表
List<Sys_user> needDoUpdateList = new ArrayList<>();
@@ -122,58 +124,79 @@ public class SourceUserServiceImpl extends ViServiceImpl<UserSource> implements
//添加角色的用户
List<Sys_user_role> userRoles = new ArrayList<>();
for (UserSource source : sources) {
source.setMember(null);
source.setWelfareMember(null);
source.setRetired(null);
Sys_user user = userMap.get(source.getLoginname().toLowerCase());
// 创建一个固定大小的线程池 可以根据服务器性能调整线程池大小
int numberOfThreads = Runtime.getRuntime().availableProcessors() * 2;
List<CompletableFuture<Void>> futures = new ArrayList<>();
UserChangeType changeType = getChangeType(source, user);
Sys_user u = new Sys_user();
BeanUtils.copyProperties(source, u);
switch (sourceType) {
case "all" -> {
//复制属性到一个新的user对象
u.setId(user == null ? source.getId() : user.getId());
if (user != null) {
needDoUpdateList.add(u);
// 每批处理的数据量，可以根据实际情况调整
int batchSize = 200;
for (int i = 0; i < sources.size(); i += batchSize) {
final int end = Math.min(i + batchSize, sources.size());
List<UserSource> batch = sources.subList(i, end);
Map<String, String> finalUserPartMap = userPartMap;
CompletableFuture<Void> future = CompletableFuture.runAsync(() -> {
for (UserSource source : batch) {
source.setMember(null);
source.setWelfareMember(null);
source.setRetired(null);
Sys_user user = userMap.get(source.getLoginname().toLowerCase());
UserChangeType changeType = getChangeType(source, user);
Sys_user u = new Sys_user();
BeanUtils.copyProperties(source, u);
switch (sourceType) {
case "all" -> {
//复制属性到一个新的user对象
u.setId(user == null ? source.getId() : user.getId());
if (user != null) {
needDoUpdateList.add(u);
}
}
case "add" -> {
u.setId(source.getId());
Integer strokeCount = 0;
try {
strokeCount = ChineseCharacterStrokeUtil.getStrokeCount(u.getUsername());
} catch (UnsupportedEncodingException e) {
strokeCount = null;
}
u.setUserNameStroke(strokeCount);
}
case "part" -> {
u.setId(user == null ? source.getId() : user.getId());
if (columnNames != null && columnNames.length > 0) {
String lockedColumn = String.join("|", columnNames);
FieldFilter fieldFilter = FieldFilter.create(Sys_user.class, null, "^" + lockedColumn + "$", true);
filterColumnDao.set(Daos.ext(dao(), fieldFilter));
}
if (Strings.isNotBlank(finalUserPartMap.get(u.getLoginname())) && user != null) {
needDoUpdateList.add(u);
}
}
}
if (changeType != null) {
if (changeType == UserChangeType.NEW) {
user = UserMode.initUser(u);
needInitUserList.add(user);
Sys_user_role userRole = new Sys_user_role();
userRole.setUserId(user.getId());
userRole.setRoleId(Roles.PUBLIC);
userRoles.add(userRole);
histories.add(createUserHistory(user, changeType));
} else {
histories.add(createUserHistory(user, changeType));
}
}
}
case "add" -> {
u.setId(source.getId());
int strokeCount = ChineseCharacterStrokeUtil.getStrokeCount(u.getUsername());
u.setUserNameStroke(strokeCount);
}
case "part" -> {
u.setId(user == null ? source.getId() : user.getId());
if (columnNames != null && columnNames.length > 0) {
String lockedColumn = String.join("|", columnNames);
FieldFilter fieldFilter = FieldFilter.create(Sys_user.class, null, "^" + lockedColumn + "$", true);
filterColumnDao = Daos.ext(dao(), fieldFilter);
}
if (Strings.isNotBlank(userPartMap.get(u.getLoginname())) && user != null) {
needDoUpdateList.add(u);
}
}
}
if (changeType != null) {
if (changeType == UserChangeType.NEW) {
user = UserMode.initUser(u);
needInitUserList.add(user);
Sys_user_role userRole = new Sys_user_role();
userRole.setUserId(user.getId());
userRole.setRoleId(Roles.PUBLIC);
userRoles.add(userRole);
histories.add(createUserHistory(user, changeType));
} else {
histories.add(createUserHistory(user, changeType));
}
}
}, Executors.newFixedThreadPool(numberOfThreads));
futures.add(future);
}
// if (Lang.isNotEmpty(needDoUpdateList)) filterColumnDao.updateIgnoreNull(needDoUpdateList);
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
//如果有新用户，增加到用户表同时增加角色
if (Lang.isNotEmpty(needInitUserList)) {
sysUserService.fastInsert(needInitUserList);
@@ -181,46 +204,13 @@ public class SourceUserServiceImpl extends ViServiceImpl<UserSource> implements
sysUserRoleService.dao().clear(Sys_user_role.class, Cnd.where("userId", "in", userIdList));
sysUserRoleService.insert(userRoles);
}
System.out.println("开始》》》》》》》》》》》》》》" + System.currentTimeMillis());
//全部更新时，修改用户
List<CompletableFuture<Integer>> updateUserFutures = new ArrayList<>();
for (List<Sys_user> userList : CollectionUtil.split(needDoUpdateList, 50)) {
// CompletableFuture<Integer> integerCompletableFuture = CompletableFuture.supplyAsync(() ->
// dao().updateIgnoreNull(userList));
// updateUserFutures.add(integerCompletableFuture);
Dao finalFilterColumnDao = filterColumnDao;
new Thread(() -> {
log.info(Thread.currentThread().getName() + "在更新数据");
if ("part".equals(sourceType)){
finalFilterColumnDao.updateIgnoreNull(userList);
} else {
dao().updateIgnoreNull(userList);
}
}).start();
if ("part".equals(sourceType)) {
massUpdatesAsync(needDoUpdateList, true, filterColumnDao);
} else {
massUpdatesAsync(needDoUpdateList, false, null);
}
// CompletableFuture.allOf(updateUserFutures.toArray(new CompletableFuture[0])).get();
// int sum = updateUserFutures.stream().mapToInt(v -> {
// try {
// return v.get();
// } catch (InterruptedException | ExecutionException e) {
// log.error(e.getMessage());
// throw new RuntimeException();
// }
// }).sum();
//
// if (sum == allDoUpdateList.size()) {
// log.info("全部更新成功" + allDoUpdateList.size() + "条");
// } else {
// log.error("更新失败");
// throw new RuntimeException("人员更新数据与期望数据不符合");
// }
System.out.println("结束》》》》》》》》》》》》》》" + System.currentTimeMillis());
//历史记录表插入数据
historyUserService.dao().fastInsert(histories);
//人员状态及在职状态更新
@@ -232,6 +222,20 @@ public class SourceUserServiceImpl extends ViServiceImpl<UserSource> implements
}
}
@Override
public void dailyScheduledUpdates() {
pullSourceData(true);
Date pullTime = (Date) dao().func2(UserSource.class, "max", "pullTime");
updateSysUser(DateUtil.formatDateTime(pullTime), "add",null,null,null,false);
DateTime agoWeekDate = DateUtil.offsetWeek(new Date(), -1);
String formatAgoWeekDate = DateUtil.format(agoWeekDate, "yyyy-MM-dd");
dao().clear(UserSource.class, Cnd.where("DATE(pullTime)", "<", formatAgoWeekDate));
}
private UserChangeType getChangeType(Sys_user source, Sys_user user) {
if (user == null) {
return UserChangeType.NEW;
@@ -261,4 +265,57 @@ public class SourceUserServiceImpl extends ViServiceImpl<UserSource> implements
history.setChangeType(changeType.getCode());
return history;
}
private void massUpdatesAsync(List<Sys_user> needDoUpdateList, Boolean isPart, AtomicReference<Dao> filterColumnDao) {
// 根据服务器性能和任务特性动态确定线程池大小;通常，线程池大小为CPU核心数的2倍
int numberOfThreads = Runtime.getRuntime().availableProcessors() * 2;
ExecutorService executorService = Executors.newFixedThreadPool(numberOfThreads);
ExecutorCompletionService<Void> ecs = new ExecutorCompletionService<>(executorService);
// 每批处理的数据量
int batchSize = 200;
try {
// 将sources分成若干批
for (int i = 0; i < needDoUpdateList.size(); i += batchSize) {
final int end = Math.min(i + batchSize, needDoUpdateList.size());
List<Sys_user> batch = needDoUpdateList.subList(i, end);
// 提交任务给线程池执行
ecs.submit(() -> {
Dao daoToUse = isPart ? filterColumnDao.get() : dao();
// 执行数据库更新操作
synchronized (daoToUse) {
daoToUse.updateIgnoreNull(batch);
}
return null; // CompletionService需要一个返回值，这里返回null
});
}
// 等待所有任务完成
for (int i = 0; i < needDoUpdateList.size(); i += batchSize) {
try {
Future<Void> future = ecs.take();
future.get(); // 等待任务完成
} catch (InterruptedException | ExecutionException e) {
Thread.currentThread().interrupt(); // 恢复中断状态
throw new RuntimeException(e);
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
// 关闭线程池
executorService.shutdown();
try {
// 设置合理的超时时间，防止无限期阻塞
if (!executorService.awaitTermination(10, TimeUnit.SECONDS)) {
executorService.shutdownNow(); // 如果超时，强制关闭
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt(); // 恢复中断状态
}
}
}
}
src/main/java/io/v/nutz/fitnessWalk/controller/FitnessWalkStepRankingController.java
+1 -1
View File
@@ -203,7 +203,7 @@ public class FitnessWalkStepRankingController {
entityList.add(new ExcelExportEntity("工号", "loginname", 20));
entityList.add(new ExcelExportEntity("分工会", "unionname", 20));
entityList.add(new ExcelExportEntity("单位", "unitname", 20));
entityList.add(new ExcelExportEntity("today".equals(mode) ? "今日步数" :"活动期间总步数", "step", 20));
entityList.add(new ExcelExportEntity("today".equals(mode) ? "今日步数" :"活动期间总步数", "total_steps", 20));
entityList.add(new ExcelExportEntity("达标天数(天)","standardsDays",20));
try {
ViTool.excelResponse(response, ( "today".equals(mode) ? DateUtil.format(new Date(),"yyyy年MM月dd日") : "活动期间总") + "步数榜.xlsx");
src/main/java/io/v/nutz/fitnessWalk/controller/FitnessWalkStepStatisticsController.java
+10 -5
View File
@@ -29,6 +29,7 @@ import org.nutz.lang.util.NutMap;
import org.nutz.mvc.annotation.At;
import org.nutz.mvc.annotation.Ok;
import org.nutz.mvc.annotation.Param;
import javax.servlet.http.HttpServletResponse;
import java.util.*;
@@ -90,19 +91,21 @@ public class FitnessWalkStepStatisticsController {
cnd.and("date_format(al.applyDate,'%Y-%m-%d')", ">=", activityDate[0]);
cnd.and("date_format(al.applyDate,'%Y-%m-%d')", "<=", activityDate[1]);
}
if(StrUtil.isNotBlank(pageForm.getSearchKeyword())){
if (StrUtil.isNotBlank(pageForm.getSearchKeyword())) {
SqlExpressionGroup seg = new SqlExpressionGroup();
seg.orLike("u.username",pageForm.getSearchKeyword());
seg.orLike("u.loginname",pageForm.getSearchKeyword());
seg.orLike("u.username", pageForm.getSearchKeyword());
seg.orLike("u.loginname", pageForm.getSearchKeyword());
cnd.and(seg);
}
cnd.and("u.id","in",Sqls.create("select userId from activity_user_scope where groupId = @groupId").setParam("groupId",activity.getGroupId()));
cnd.and("u.id", "in", Sqls.create("select userId from activity_user_scope where groupId = @groupId").setParam("groupId", activity.getGroupId()));
cnd.andEX("YEAR(al.applyDate)", "=", year);
cnd.andEX("al.activityId", "=", activityId);
cnd.andEX("u.unionid", "=", unionId);
cnd.andEX("u.unitId", "=", unitId);
cnd.desc("al.userId");
cnd.desc("al.applyDate");
cnd.desc("u.unioncode");
cnd.groupBy("al.applyDate", "al.userId");
sql.setCondition(cnd);
return baseService.listPageMap(pageForm.getPageNumber(), pageForm.getPageSize(), sql);
@@ -132,9 +135,11 @@ public class FitnessWalkStepStatisticsController {
cnd.andEX("al.activityId", "=", activityId);
cnd.andEX("u.unionid", "=", unionId);
cnd.andEX("u.unitId", "=", unitId);
cnd.and("u.id","in",Sqls.create("select userId from activity_user_scope where groupId = @groupId").setParam("groupId",activity.getGroupId()));
cnd.and("u.id", "in", Sqls.create("select userId from activity_user_scope where groupId = @groupId").setParam("groupId", activity.getGroupId()));
cnd.desc("al.userId");
cnd.desc("al.applyDate");
cnd.desc("u.unioncode");
cnd.groupBy("al.applyDate", "al.userId");
sql.setCondition(cnd);
List<NutMap> exportList = baseService.listMap(sql);
src/main/java/io/v/nutz/fitnessWalk/service/impl/FitnessWalkCommonServiceImpl.java
+24 -10
View File
@@ -384,8 +384,8 @@ public class FitnessWalkCommonServiceImpl extends ViServiceImpl implements Fitne
Sql sql = Sqls.create("""
SELECT
s.userId,
SUM(s.step) AS step,
u.id,
COALESCE(SUM(sub.step), 0) AS total_steps,
u.username,
u.loginname,
u.unionname,
@@ -393,7 +393,21 @@ public class FitnessWalkCommonServiceImpl extends ViServiceImpl implements Fitne
over3k.standardsDays
FROM
`user` u
LEFT JOIN `fitness_walk_step` s ON u.id = s.userId
LEFT JOIN (
SELECT
userId,
applyDate,
step
FROM
`fitness_walk_step`
WHERE
step >= @step
AND activityId=@activityId
AND $dateSql
GROUP BY
userId,
applyDate
) AS sub ON u.id = sub.userId
LEFT JOIN (
SELECT
userId,
@@ -402,7 +416,8 @@ public class FitnessWalkCommonServiceImpl extends ViServiceImpl implements Fitne
`fitness_walk_step`
WHERE
step >= @step
AND activityId=@activityId and $dateSql
AND activityId=@activityId
AND $dateSql
GROUP BY
userId
) AS over3k ON u.id = over3k.userId
@@ -411,17 +426,16 @@ public class FitnessWalkCommonServiceImpl extends ViServiceImpl implements Fitne
if ("today".equals(mode)) {
sql.setVar("dateSql",new Static("DATE(applyDate) = '"+DateUtil.today()+"'"));
cnd.and("DATE( applyDate )", "=", DateUtil.today());
// cnd.and("DATE( s.applyDate )", "=", DateUtil.today());
} else {
sql.setVar("dateSql",new Static("DATE(applyDate) >= '"
+DateUtil.format(DateUtil.date(activity.getStartTime()), "yyyy-MM-dd")+"' and DATE(applyDate) <= '"
+DateUtil.format(DateUtil.date(activity.getEndTime()), "yyyy-MM-dd")+"'"));
cnd.and("DATE( applyDate )", ">=", DateUtil.format(DateUtil.date(activity.getStartTime()), "yyyy-MM-dd"));
cnd.and("DATE( applyDate )", "<=", DateUtil.format(DateUtil.date(activity.getEndTime()), "yyyy-MM-dd"));
// cnd.and("DATE( s.applyDate )", ">=", DateUtil.format(DateUtil.date(activity.getStartTime()), "yyyy-MM-dd"));
// cnd.and("DATE( s.applyDate )", "<=", DateUtil.format(DateUtil.date(activity.getEndTime()), "yyyy-MM-dd"));
}
cnd.and("s.activityId", "=", activityId);
if (Strings.isNotBlank(unitId)) {
cnd.and("u.unitId", "=", unitId);
}
@@ -433,8 +447,8 @@ public class FitnessWalkCommonServiceImpl extends ViServiceImpl implements Fitne
cnd.and(searchName, "LIKE", "%" + searchKeyword + "%");
}
cnd.and("u.id","in",Sqls.create("select userId from activity_user_scope where groupId = @groupId").setParam("groupId",activity.getGroupId()));
cnd.groupBy("s.userId");
cnd.desc("s.step");
cnd.groupBy("u.id","u.username","u.loginname","u.unionname","u.unitname","over3k.standardsDays");
cnd.desc("total_steps");
sql.setCondition(cnd);
return sql;
}
src/main/java/io/v/nutz/task/job/UserUpdateJobTask.java
+1 -12
View File
@@ -30,17 +30,6 @@ public class UserUpdateJobTask implements Job {
@Override
public void execute(JobExecutionContext context) throws JobExecutionException {
try {
sourceUserService.pullSourceData(true);
Date pullTime = (Date) dao.func2(UserSource.class, "max", "pullTime");
sourceUserService.updateSysUser(DateUtil.formatDateTime(pullTime), "add",null,null,null,false);
DateTime agoWeekDate = DateUtil.offsetWeek(new Date(), -1);
String formatAgoWeekDate = DateUtil.format(agoWeekDate, "yyyy-MM-dd");
sourceUserService.clear(Cnd.where("DATE(pullTime)","<",formatAgoWeekDate));
} catch (Exception e) {
e.printStackTrace();
}
sourceUserService.dailyScheduledUpdates();
}
}
src/main/java/io/v/nutz/task/job/fitnessWalk/FitnessWalkLotteryJob.java
+49 -20
View File
@@ -69,34 +69,63 @@ public class FitnessWalkLotteryJob implements Job {
//查询出符合条件的用户
Sql sql = Sqls.create("""
SELECT
jws.userId,
u.loginname,
u.username,
u.unionid,
u.unionname,
u.unitid,
u.unitname
u.id,
COALESCE(SUM(sub.step), 0) AS total_steps,
u.loginname,
u.username,
u.unionid,
u.unionname,
u.unitid,
u.unitname
FROM
fitness_walk_step jws
LEFT JOIN `user` u ON u.id = jws.userId
WHERE
jws.activityId = @activityId
AND DATE ( jws.applyDate ) >= @startDate
AND DATE ( jws.applyDate ) <= @endDate
$lotteryQualificationDaySql
`user` u
LEFT JOIN (
SELECT
userId,
applyDate,
step
FROM
`fitness_walk_step`
WHERE
activityId = @activityId
AND DATE(applyDate) >= @startDate
AND DATE(applyDate) <= @endDate
$lotteryQualificationDaySql
GROUP BY
userId,
applyDate
) AS sub ON u.id = sub.userId
LEFT JOIN (
SELECT
userId,
COUNT(DISTINCT DATE(applyDate)) AS standardsDays
FROM
`fitness_walk_step`
WHERE
activityId = @activityId
AND DATE(applyDate) >= @startDate
AND DATE(applyDate) <= @endDate
$lotteryQualificationDaySql
GROUP BY
userId
) AS over3k ON u.id = over3k.userId
GROUP BY
jws.userId
u.id,
u.username,
u.loginname,
u.unionname,
u.unitname
HAVING
$stepHavingSql
ORDER BY
rand()
LIMIT @lotteryUserNum
rand()
LIMIT @lotteryUserNum
""");
if (Strings.isNotBlank(lotteryQualificationDay)) {
sql.setVar("lotteryQualificationDaySql", new Static(" AND jws.step > '%s' ".formatted(lotteryQualificationStep)));
sql.setVar("stepHavingSql", new Static(" COUNT( jws.applyDate ) > '%s' ".formatted(lotteryQualificationDay)));
sql.setVar("lotteryQualificationDaySql", new Static(" AND step > '%s' ".formatted(lotteryQualificationStep)));
sql.setVar("stepHavingSql", new Static(" over3k.standardsDays > '%s' ".formatted(lotteryQualificationDay)));
} else {
sql.setVar("stepHavingSql", new Static(" SUM( jws.step ) > '%s' ".formatted(lotteryQualificationStep)));
sql.setVar("stepHavingSql", new Static(" COALESCE(SUM(sub.step), 0) > '%s' ".formatted(lotteryQualificationStep)));
}
sql.setParam("activityId", activityId);
sql.setParam("step", lotteryQualificationStep);
src/main/resources/views/platform/fitnessWalk/stepRanking.html
+1 -1
View File
@@ -141,7 +141,7 @@ layout("/layouts/platform.html"){
{label: '工号', prop: 'loginname'},
{label: '分工会', prop: 'unionname', sortable: true},
{label: '单位', prop: 'unitname'},
{label: '步数', prop: 'step', sortable: true},
{label: '步数', prop: 'total_steps', sortable: true},
{label: '达标天数', prop: 'standardsDays', sortable: true},
],
}