From 671f586737b1765703e8ecccbdfa60d8c4a75ff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=8C=E5=85=88=E8=88=9C?= <2499071264@qq.com> Date: Thu, 12 Nov 2026 11:34:53 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BA=BA=E5=91=98=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=B2=BB=E7=90=86=E8=BF=81=E7=A7=BB=E5=8A=9F=E8=83=BD=E4=BC=98?= =?UTF-8?q?=E5=8C=962?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/PersonApiConfigServiceImpl.java | 155 +++++++++++------- .../personGovernance/apiConfig/form.html | 89 ++++++---- tools/person-governance-api-mock.js | 6 +- 3 files changed, 158 insertions(+), 92 deletions(-) diff --git a/src/main/java/com/budwk/app/zhgh/personGovernance/apiConfig/service/impl/PersonApiConfigServiceImpl.java b/src/main/java/com/budwk/app/zhgh/personGovernance/apiConfig/service/impl/PersonApiConfigServiceImpl.java index 1f4e3b2..c3e8fdf 100644 --- a/src/main/java/com/budwk/app/zhgh/personGovernance/apiConfig/service/impl/PersonApiConfigServiceImpl.java +++ b/src/main/java/com/budwk/app/zhgh/personGovernance/apiConfig/service/impl/PersonApiConfigServiceImpl.java @@ -57,6 +57,8 @@ public class PersonApiConfigServiceImpl extends BaseServiceImpl private static final Pattern SAFE_IDENTIFIER = Pattern.compile("^[A-Za-z0-9_]+$"); private static final String SYNC_OPTIONS_CHUNKS = "@chunks:"; private static final String USER_FIELD_DICTIONARIES = "USER_TABLE_COLUMN_DICTS"; + private static final int FIELD_QUERY_MAX_PAGE_SIZE = 100; + private static final int FIELD_QUERY_MAX_PAGES = 10000; /** JSON节点删除标记：Token接口未配置时，用于彻底移除数据请求中的Token字段或数组项。 */ private static final Object OMIT_TOKEN_PARAMETER = new Object(); @@ -507,12 +509,11 @@ public class PersonApiConfigServiceImpl extends BaseServiceImpl List params = payload.get("params") == null ? new ArrayList<>() : Json.fromJsonAsList(PersonApiRequestParam.class, Json.toJson(payload.get("params"))); - return queryResponse(config, params, requestScope, false); + return queryResponse(config, params, requestScope); } - private Object queryResponse(PersonApiConfig config, List params, - String requestScope, boolean fieldQuery) { - NutMap response = executeHttp(config, params, requestScope, null, fieldQuery); + private Object queryResponse(PersonApiConfig config, List params, String requestScope) { + NutMap response = executeHttp(config, params, requestScope, null); Assert.isTrue(response.getBoolean("success"), response.getString("error") == null ? "第三方返回HTTP错误：" + response.get("status") : response.getString("error")); String body = response.getString("data"); @@ -541,50 +542,79 @@ public class PersonApiConfigServiceImpl extends BaseServiceImpl List params = payload.get("params") == null ? List.of() : Json.fromJsonAsList(PersonApiRequestParam.class, Json.toJson(payload.get("params"))); - setFieldQueryPage(config, params, 100); - Object firstPage = queryResponse(config, params, "DATA", true); + NutMap paging = pagination(config, params); + String mode = paging.getString("mode"); + boolean legacyPaging = "NONE".equals(mode) + && setFieldQueryPage(config, params, 1, FIELD_QUERY_MAX_PAGE_SIZE); + long cursor = legacyPaging ? 1 : paging.getLong("cursor"); + int pageSize = legacyPaging ? FIELD_QUERY_MAX_PAGE_SIZE : Math.min(paging.getInt("size"), FIELD_QUERY_MAX_PAGE_SIZE); + long step = legacyPaging || "PAGE".equals(mode) ? 1 : pageSize; + boolean singleRequest = pageSize == 0 || "LIMIT".equals(mode); + boolean fromStart = "PAGE".equals(mode) || legacyPaging ? cursor <= 1 : cursor == 0; + boolean checkTotal = !singleRequest && fromStart; String totalPath = Strings.isBlank(config.getResponseTotalPath()) ? "data->total" : config.getResponseTotalPath(); - int total = fieldQueryTotal(firstPage, totalPath); - if (total == 0) { - return NutMap.NEW().addv("fields", names).addv("rows", List.of()).addv("total", 0); - } - - setFieldQueryPage(config, params, total); - Object response = queryResponse(config, params, "DATA", true); - int returnedTotal = fieldQueryTotal(response, totalPath); - String listPath = config.getResponseDataPath(); - if (Strings.isBlank(listPath) || "$.data".equals(listPath.trim()) || "data".equals(listPath.trim())) { - Object configured = readResponsePath(response, listPath); - if (!(configured instanceof List)) listPath = "data->list"; - } - Object dataset = readResponsePath(response, listPath); - Assert.isTrue(dataset instanceof List, "数据列表路径未找到数组：" + listPath); - List records = (List) dataset; - Assert.isTrue(records.size() >= Math.max(total, returnedTotal), - "接口未返回全部数据，预计 " + Math.max(total, returnedTotal) + " 条，实际返回 " + records.size() + " 条"); - LinkedHashSet> groups = new LinkedHashSet<>(); LinkedHashSet found = new LinkedHashSet<>(); - for (Object record : records) { - Assert.isTrue(record instanceof Map, "数据列表中的记录必须为对象"); - Map row = (Map) record; - List values = new ArrayList<>(names.size()); - for (String name : names) { - if (row.containsKey(name)) found.add(name); - values.add(row.get(name)); + LinkedHashSet fingerprints = new LinkedHashSet<>(); + int total = 0; + Integer expectedTotal = null; + for (int count = 0; ; count++) { + Assert.isTrue(count < FIELD_QUERY_MAX_PAGES, "超过最大分页次数，已终止字段值查询"); + Object response = querySyncPage(config, params, cursor, pageSize, true); + Integer returnedTotal = fieldQueryTotal(response, totalPath, checkTotal + && Strings.isNotBlank(config.getResponseTotalPath())); + if (checkTotal && returnedTotal != null) { + expectedTotal = expectedTotal == null ? returnedTotal : Math.max(expectedTotal, returnedTotal); } - groups.add(values); + List records = fieldQueryRecords(response, config.getResponseDataPath(), returnedTotal); + if (!singleRequest && !records.isEmpty()) { + Assert.isTrue(fingerprints.add(DigestUtil.sha256Hex(Json.toJson(records))), "接口重复返回同一页，已终止字段值查询"); + } + for (Object record : records) { + Assert.isTrue(record instanceof Map, "数据列表中的记录必须为对象"); + Map row = (Map) record; + List values = new ArrayList<>(names.size()); + for (String name : names) { + if (row.containsKey(name)) found.add(name); + values.add(row.get(name)); + } + groups.add(values); + } + total += records.size(); + if (singleRequest || (expectedTotal != null && total >= expectedTotal)) break; + if (records.size() < pageSize) { + Assert.isTrue(expectedTotal == null, + "接口未返回全部数据，预计 " + expectedTotal + " 条，实际返回 " + total + " 条"); + break; + } + cursor = Math.addExact(cursor, step); + if (legacyPaging) setFieldQueryPage(config, params, cursor, pageSize); } - List missing = names.stream().filter(name -> !found.contains(name)).toList(); - Assert.isTrue(missing.isEmpty(), "返回数据中找不到字段：" + String.join("、", missing)); - return NutMap.NEW().addv("fields", names).addv("rows", new ArrayList<>(groups)).addv("total", records.size()); + if (total > 0) { + List missing = names.stream().filter(name -> !found.contains(name)).toList(); + Assert.isTrue(missing.isEmpty(), "返回数据中找不到字段：" + String.join("、", missing)); + } + return NutMap.NEW().addv("fields", names).addv("rows", new ArrayList<>(groups)).addv("total", total); } - private int fieldQueryTotal(Object response, String path) { + private List fieldQueryRecords(Object response, String listPath, Integer returnedTotal) { + Object dataset = readResponsePath(response, listPath); + if (!(dataset instanceof List) + && (Strings.isBlank(listPath) || "$.data".equals(listPath.trim()) || "data".equals(listPath.trim()))) { + listPath = "data->list"; + dataset = readResponsePath(response, listPath); + } + if (dataset == null && Integer.valueOf(0).equals(returnedTotal)) return List.of(); + Assert.isTrue(dataset instanceof List, "数据列表路径未找到数组：" + listPath); + return (List) dataset; + } + + private Integer fieldQueryTotal(Object response, String path, boolean required) { if (response instanceof Map map) { Assert.isTrue(!Boolean.FALSE.equals(map.get("status")), "数据接口返回失败：" + Strings.sNull(map.get("message"))); } Object value = readResponsePath(response, path); + if (value == null && !required) return null; try { int total = new java.math.BigDecimal(String.valueOf(value)).intValueExact(); if (total >= 0) return total; @@ -593,33 +623,35 @@ public class PersonApiConfigServiceImpl extends BaseServiceImpl throw new IllegalArgumentException("返回总数必须为非负整数，请检查路径：" + path); } - private void setFieldQueryPage(PersonApiConfig config, List params, int pageSize) { + private boolean setFieldQueryPage(PersonApiConfig config, List params, long page, int pageSize) { boolean sendBody = !List.of("GET", "HEAD").contains(Strings.sNull(config.getRequestMethod()).toUpperCase(Locale.ROOT)); - Map paging = Map.of("pageNumber", 1, "pageSize", pageSize); - LinkedHashSet replaced = new LinkedHashSet<>(); + Map paging = Map.of("pageNumber", page, "pageSize", pageSize); + com.alibaba.fastjson.JSONObject json = null; if (sendBody && "JSON".equals(config.getBodyType()) && Strings.isNotBlank(config.getBodyContent())) { Object body = parseResponseObject(config.getBodyContent()); - if (body instanceof com.alibaba.fastjson.JSONObject json) { - paging.forEach((name, value) -> { - if (json.containsKey(name)) { - json.put(name, value); - replaced.add(name); - } - }); - config.setBodyContent(com.alibaba.fastjson.JSON.toJSONString(json, - com.alibaba.fastjson.serializer.SerializerFeature.WriteMapNullValue)); + if (body instanceof com.alibaba.fastjson.JSONObject object) json = object; + } + List pagingParams = params.stream() + .filter(param -> "DATA".equals(normalizeScope(param.getRequestScope()))) + .filter(param -> "QUERY".equals(param.getParamLocation()) + || (sendBody && "FORM_DATA".equals(config.getBodyType()) && "BODY".equals(param.getParamLocation()))) + .filter(param -> param.getParamName() != null && paging.containsKey(param.getParamName())) + .toList(); + for (String name : paging.keySet()) { + if ((json == null || !json.containsKey(name)) + && pagingParams.stream().noneMatch(param -> name.equals(param.getParamName()))) { + return false; } } - for (PersonApiRequestParam param : params) { - boolean sent = "QUERY".equals(param.getParamLocation()) - || (sendBody && "FORM_DATA".equals(config.getBodyType()) && "BODY".equals(param.getParamLocation())); - if (sent && "DATA".equals(normalizeScope(param.getRequestScope())) - && param.getParamName() != null && paging.containsKey(param.getParamName())) { - param.setParamValue(String.valueOf(paging.get(param.getParamName()))); - replaced.add(param.getParamName()); + pagingParams.forEach(param -> param.setParamValue(String.valueOf(paging.get(param.getParamName())))); + if (json != null) { + for (Map.Entry entry : paging.entrySet()) { + if (json.containsKey(entry.getKey())) json.put(entry.getKey(), entry.getValue()); } + config.setBodyContent(com.alibaba.fastjson.JSON.toJSONString(json, + com.alibaba.fastjson.serializer.SerializerFeature.WriteMapNullValue)); } - Assert.isTrue(replaced.size() == paging.size(), "请在数据请求参数中配置 pageNumber 和 pageSize"); + return true; } /** @@ -630,6 +662,11 @@ public class PersonApiConfigServiceImpl extends BaseServiceImpl @Override @Aop(TransAop.READ_COMMITTED) public Object querySyncPage(PersonApiConfig config, List params, long page, int pageSize) { + return querySyncPage(config, params, page, pageSize, false); + } + + private Object querySyncPage(PersonApiConfig config, List params, + long page, int pageSize, boolean fieldQuery) { boolean sendBody = !List.of("GET", "HEAD").contains(Strings.sNull(config.getRequestMethod()).toUpperCase(Locale.ROOT)); boolean tokenConfigured = Strings.isNotBlank(config.getTokenRequestUrl()); List dataParams = params.stream() @@ -666,7 +703,7 @@ public class PersonApiConfigServiceImpl extends BaseServiceImpl } requestConfig.setBodyContent(com.alibaba.fastjson.JSON.toJSONString(fillJsonParameters(body, token, page, pageSize))); } - NutMap response = executeHttp(requestConfig, requestParams, "DATA", null); + NutMap response = executeHttp(requestConfig, requestParams, "DATA", null, fieldQuery); Assert.isTrue(response.getBoolean("success"), "数据接口请求失败：" + (response.get("error") == null ? response.get("status") : response.get("error"))); return parseResponseObject(response.getString("data")); @@ -895,7 +932,7 @@ public class PersonApiConfigServiceImpl extends BaseServiceImpl String scope, String testCondition, boolean fieldQuery) { Assert.isTrue(List.of("TOKEN", "DATA").contains(scope), "请求配置范围不正确"); long start = System.currentTimeMillis(); - boolean omitToken = !fieldQuery && "DATA".equals(scope) && Strings.isBlank(config.getTokenRequestUrl()); + boolean omitToken = "DATA".equals(scope) && Strings.isBlank(config.getTokenRequestUrl()); params = params.stream() .filter(param -> scope.equals(normalizeScope(param.getRequestScope()))) // 未配置Token接口时，Header、Query及Form Data中的Token参数均不进入实际请求。 diff --git a/src/main/resources/views/platform/personGovernance/apiConfig/form.html b/src/main/resources/views/platform/personGovernance/apiConfig/form.html index 6fab95c..3a259a7 100644 --- a/src/main/resources/views/platform/personGovernance/apiConfig/form.html +++ b/src/main/resources/views/platform/personGovernance/apiConfig/form.html @@ -253,6 +253,14 @@ + + + + + +