fix: 人员数据治理迁移功能优化2

src/main/java/com/budwk/app/zhgh/personGovernance/apiConfig/service/impl/PersonApiConfigServiceImpl.java

@@ -57,6 +57,8 @@ public class PersonApiConfigServiceImpl extends BaseServiceImpl<PersonApiConfig>
     private static final Pattern SAFE_IDENTIFIER = Pattern.compile("^[A-Za-z0-9_]+$");
     private static final String SYNC_OPTIONS_CHUNKS = "@chunks:";
     private static final String USER_FIELD_DICTIONARIES = "USER_TABLE_COLUMN_DICTS";
+    private static final int FIELD_QUERY_MAX_PAGE_SIZE = 100;
+    private static final int FIELD_QUERY_MAX_PAGES = 10000;
     /** JSON节点删除标记：Token接口未配置时，用于彻底移除数据请求中的Token字段或数组项。 */
     private static final Object OMIT_TOKEN_PARAMETER = new Object();
 
@@ -507,12 +509,11 @@ public class PersonApiConfigServiceImpl extends BaseServiceImpl<PersonApiConfig>
         List<PersonApiRequestParam> params = payload.get("params") == null
                 ? new ArrayList<>()
                 : Json.fromJsonAsList(PersonApiRequestParam.class, Json.toJson(payload.get("params")));
-        return queryResponse(config, params, requestScope, false);
+        return queryResponse(config, params, requestScope);
     }
 
-    private Object queryResponse(PersonApiConfig config, List<PersonApiRequestParam> params,
+    private Object queryResponse(PersonApiConfig config, List<PersonApiRequestParam> params, String requestScope) {
-                                 String requestScope, boolean fieldQuery) {
+        NutMap response = executeHttp(config, params, requestScope, null);
-        NutMap response = executeHttp(config, params, requestScope, null, fieldQuery);
         Assert.isTrue(response.getBoolean("success"), response.getString("error") == null
                 ? "第三方返回HTTP错误：" + response.get("status") : response.getString("error"));
         String body = response.getString("data");
@@ -541,50 +542,79 @@ public class PersonApiConfigServiceImpl extends BaseServiceImpl<PersonApiConfig>
         List<PersonApiRequestParam> params = payload.get("params") == null ? List.of()
                 : Json.fromJsonAsList(PersonApiRequestParam.class, Json.toJson(payload.get("params")));
 
-        setFieldQueryPage(config, params, 100);
+        NutMap paging = pagination(config, params);
-        Object firstPage = queryResponse(config, params, "DATA", true);
+        String mode = paging.getString("mode");
+        boolean legacyPaging = "NONE".equals(mode)
+                && setFieldQueryPage(config, params, 1, FIELD_QUERY_MAX_PAGE_SIZE);
+        long cursor = legacyPaging ? 1 : paging.getLong("cursor");
+        int pageSize = legacyPaging ? FIELD_QUERY_MAX_PAGE_SIZE : Math.min(paging.getInt("size"), FIELD_QUERY_MAX_PAGE_SIZE);
+        long step = legacyPaging || "PAGE".equals(mode) ? 1 : pageSize;
+        boolean singleRequest = pageSize == 0 || "LIMIT".equals(mode);
+        boolean fromStart = "PAGE".equals(mode) || legacyPaging ? cursor <= 1 : cursor == 0;
+        boolean checkTotal = !singleRequest && fromStart;
         String totalPath = Strings.isBlank(config.getResponseTotalPath()) ? "data->total" : config.getResponseTotalPath();
-        int total = fieldQueryTotal(firstPage, totalPath);
-        if (total == 0) {
-            return NutMap.NEW().addv("fields", names).addv("rows", List.of()).addv("total", 0);
-        }
-
-        setFieldQueryPage(config, params, total);
-        Object response = queryResponse(config, params, "DATA", true);
-        int returnedTotal = fieldQueryTotal(response, totalPath);
-        String listPath = config.getResponseDataPath();
-        if (Strings.isBlank(listPath) || "$.data".equals(listPath.trim()) || "data".equals(listPath.trim())) {
-            Object configured = readResponsePath(response, listPath);
-            if (!(configured instanceof List<?>)) listPath = "data->list";
-        }
-        Object dataset = readResponsePath(response, listPath);
-        Assert.isTrue(dataset instanceof List<?>, "数据列表路径未找到数组：" + listPath);
-        List<?> records = (List<?>) dataset;
-        Assert.isTrue(records.size() >= Math.max(total, returnedTotal),
-                "接口未返回全部数据，预计 " + Math.max(total, returnedTotal) + " 条，实际返回 " + records.size() + " 条");
-
         LinkedHashSet<List<Object>> groups = new LinkedHashSet<>();
         LinkedHashSet<String> found = new LinkedHashSet<>();
-        for (Object record : records) {
+        LinkedHashSet<String> fingerprints = new LinkedHashSet<>();
-            Assert.isTrue(record instanceof Map<?, ?>, "数据列表中的记录必须为对象");
+        int total = 0;
-            Map<?, ?> row = (Map<?, ?>) record;
+        Integer expectedTotal = null;
-            List<Object> values = new ArrayList<>(names.size());
+        for (int count = 0; ; count++) {
-            for (String name : names) {
+            Assert.isTrue(count < FIELD_QUERY_MAX_PAGES, "超过最大分页次数，已终止字段值查询");
-                if (row.containsKey(name)) found.add(name);
+            Object response = querySyncPage(config, params, cursor, pageSize, true);
-                values.add(row.get(name));
+            Integer returnedTotal = fieldQueryTotal(response, totalPath, checkTotal
+                    && Strings.isNotBlank(config.getResponseTotalPath()));
+            if (checkTotal && returnedTotal != null) {
+                expectedTotal = expectedTotal == null ? returnedTotal : Math.max(expectedTotal, returnedTotal);
             }
-            groups.add(values);
+            List<?> records = fieldQueryRecords(response, config.getResponseDataPath(), returnedTotal);
+            if (!singleRequest && !records.isEmpty()) {
+                Assert.isTrue(fingerprints.add(DigestUtil.sha256Hex(Json.toJson(records))), "接口重复返回同一页，已终止字段值查询");
+            }
+            for (Object record : records) {
+                Assert.isTrue(record instanceof Map<?, ?>, "数据列表中的记录必须为对象");
+                Map<?, ?> row = (Map<?, ?>) record;
+                List<Object> values = new ArrayList<>(names.size());
+                for (String name : names) {
+                    if (row.containsKey(name)) found.add(name);
+                    values.add(row.get(name));
+                }
+                groups.add(values);
+            }
+            total += records.size();
+            if (singleRequest || (expectedTotal != null && total >= expectedTotal)) break;
+            if (records.size() < pageSize) {
+                Assert.isTrue(expectedTotal == null,
+                        "接口未返回全部数据，预计 " + expectedTotal + " 条，实际返回 " + total + " 条");
+                break;
+            }
+            cursor = Math.addExact(cursor, step);
+            if (legacyPaging) setFieldQueryPage(config, params, cursor, pageSize);
         }
-        List<String> missing = names.stream().filter(name -> !found.contains(name)).toList();
+        if (total > 0) {
-        Assert.isTrue(missing.isEmpty(), "返回数据中找不到字段：" + String.join("、", missing));
+            List<String> missing = names.stream().filter(name -> !found.contains(name)).toList();
-        return NutMap.NEW().addv("fields", names).addv("rows", new ArrayList<>(groups)).addv("total", records.size());
+            Assert.isTrue(missing.isEmpty(), "返回数据中找不到字段：" + String.join("、", missing));
+        }
+        return NutMap.NEW().addv("fields", names).addv("rows", new ArrayList<>(groups)).addv("total", total);
     }
 
-    private int fieldQueryTotal(Object response, String path) {
+    private List<?> fieldQueryRecords(Object response, String listPath, Integer returnedTotal) {
+        Object dataset = readResponsePath(response, listPath);
+        if (!(dataset instanceof List<?>)
+                && (Strings.isBlank(listPath) || "$.data".equals(listPath.trim()) || "data".equals(listPath.trim()))) {
+            listPath = "data->list";
+            dataset = readResponsePath(response, listPath);
+        }
+        if (dataset == null && Integer.valueOf(0).equals(returnedTotal)) return List.of();
+        Assert.isTrue(dataset instanceof List<?>, "数据列表路径未找到数组：" + listPath);
+        return (List<?>) dataset;
+    }
+
+    private Integer fieldQueryTotal(Object response, String path, boolean required) {
         if (response instanceof Map<?, ?> map) {
             Assert.isTrue(!Boolean.FALSE.equals(map.get("status")), "数据接口返回失败：" + Strings.sNull(map.get("message")));
         }
         Object value = readResponsePath(response, path);
+        if (value == null && !required) return null;
         try {
             int total = new java.math.BigDecimal(String.valueOf(value)).intValueExact();
             if (total >= 0) return total;
@@ -593,33 +623,35 @@ public class PersonApiConfigServiceImpl extends BaseServiceImpl<PersonApiConfig>
         throw new IllegalArgumentException("返回总数必须为非负整数，请检查路径：" + path);
     }
 
-    private void setFieldQueryPage(PersonApiConfig config, List<PersonApiRequestParam> params, int pageSize) {
+    private boolean setFieldQueryPage(PersonApiConfig config, List<PersonApiRequestParam> params, long page, int pageSize) {
         boolean sendBody = !List.of("GET", "HEAD").contains(Strings.sNull(config.getRequestMethod()).toUpperCase(Locale.ROOT));
-        Map<String, Integer> paging = Map.of("pageNumber", 1, "pageSize", pageSize);
+        Map<String, Number> paging = Map.of("pageNumber", page, "pageSize", pageSize);
-        LinkedHashSet<String> replaced = new LinkedHashSet<>();
+        com.alibaba.fastjson.JSONObject json = null;
         if (sendBody && "JSON".equals(config.getBodyType()) && Strings.isNotBlank(config.getBodyContent())) {
             Object body = parseResponseObject(config.getBodyContent());
-            if (body instanceof com.alibaba.fastjson.JSONObject json) {
+            if (body instanceof com.alibaba.fastjson.JSONObject object) json = object;
-                paging.forEach((name, value) -> {
+        }
-                    if (json.containsKey(name)) {
+        List<PersonApiRequestParam> pagingParams = params.stream()
-                        json.put(name, value);
+                .filter(param -> "DATA".equals(normalizeScope(param.getRequestScope())))
-                        replaced.add(name);
+                .filter(param -> "QUERY".equals(param.getParamLocation())
-                    }
+                        || (sendBody && "FORM_DATA".equals(config.getBodyType()) && "BODY".equals(param.getParamLocation())))
-                });
+                .filter(param -> param.getParamName() != null && paging.containsKey(param.getParamName()))
-                config.setBodyContent(com.alibaba.fastjson.JSON.toJSONString(json,
+                .toList();
-                        com.alibaba.fastjson.serializer.SerializerFeature.WriteMapNullValue));
+        for (String name : paging.keySet()) {
+            if ((json == null || !json.containsKey(name))
+                    && pagingParams.stream().noneMatch(param -> name.equals(param.getParamName()))) {
+                return false;
             }
         }
-        for (PersonApiRequestParam param : params) {
+        pagingParams.forEach(param -> param.setParamValue(String.valueOf(paging.get(param.getParamName()))));
-            boolean sent = "QUERY".equals(param.getParamLocation())
+        if (json != null) {
-                    || (sendBody && "FORM_DATA".equals(config.getBodyType()) && "BODY".equals(param.getParamLocation()));
+            for (Map.Entry<String, Number> entry : paging.entrySet()) {
-            if (sent && "DATA".equals(normalizeScope(param.getRequestScope()))
+                if (json.containsKey(entry.getKey())) json.put(entry.getKey(), entry.getValue());
-                    && param.getParamName() != null && paging.containsKey(param.getParamName())) {
-                param.setParamValue(String.valueOf(paging.get(param.getParamName())));
-                replaced.add(param.getParamName());
             }
+            config.setBodyContent(com.alibaba.fastjson.JSON.toJSONString(json,
+                    com.alibaba.fastjson.serializer.SerializerFeature.WriteMapNullValue));
         }
-        Assert.isTrue(replaced.size() == paging.size(), "请在数据请求参数中配置 pageNumber 和 pageSize");
+        return true;
     }
 
     /**
@@ -630,6 +662,11 @@ public class PersonApiConfigServiceImpl extends BaseServiceImpl<PersonApiConfig>
     @Override
     @Aop(TransAop.READ_COMMITTED)
     public Object querySyncPage(PersonApiConfig config, List<PersonApiRequestParam> params, long page, int pageSize) {
+        return querySyncPage(config, params, page, pageSize, false);
+    }
+
+    private Object querySyncPage(PersonApiConfig config, List<PersonApiRequestParam> params,
+                                 long page, int pageSize, boolean fieldQuery) {
         boolean sendBody = !List.of("GET", "HEAD").contains(Strings.sNull(config.getRequestMethod()).toUpperCase(Locale.ROOT));
         boolean tokenConfigured = Strings.isNotBlank(config.getTokenRequestUrl());
         List<PersonApiRequestParam> dataParams = params.stream()
@@ -666,7 +703,7 @@ public class PersonApiConfigServiceImpl extends BaseServiceImpl<PersonApiConfig>
             }
             requestConfig.setBodyContent(com.alibaba.fastjson.JSON.toJSONString(fillJsonParameters(body, token, page, pageSize)));
         }
-        NutMap response = executeHttp(requestConfig, requestParams, "DATA", null);
+        NutMap response = executeHttp(requestConfig, requestParams, "DATA", null, fieldQuery);
         Assert.isTrue(response.getBoolean("success"), "数据接口请求失败："
                 + (response.get("error") == null ? response.get("status") : response.get("error")));
         return parseResponseObject(response.getString("data"));
@@ -895,7 +932,7 @@ public class PersonApiConfigServiceImpl extends BaseServiceImpl<PersonApiConfig>
                                String scope, String testCondition, boolean fieldQuery) {
         Assert.isTrue(List.of("TOKEN", "DATA").contains(scope), "请求配置范围不正确");
         long start = System.currentTimeMillis();
-        boolean omitToken = !fieldQuery && "DATA".equals(scope) && Strings.isBlank(config.getTokenRequestUrl());
+        boolean omitToken = "DATA".equals(scope) && Strings.isBlank(config.getTokenRequestUrl());
         params = params.stream()
                 .filter(param -> scope.equals(normalizeScope(param.getRequestScope())))
                 // 未配置Token接口时，Header、Query及Form Data中的Token参数均不进入实际请求。

src/main/resources/views/platform/personGovernance/apiConfig/form.html

@@ -253,6 +253,14 @@
                             <el-input v-model="row.sourceComment" disabled></el-input>
                         </template>
                     </el-table-column>
+                    <el-table-column v-if="form.sourceType==='HTTP_API'" label="来源字段">
+                        <template v-slot="{row}">
+                            <el-input v-model="row.sourceField"></el-input>
+                        </template>
+                    </el-table-column>
+                    <el-table-column v-if="form.sourceType==='HTTP_API'" label="来源字段说明">
+                        <template v-slot="{row}"><el-input v-model="row.sourceComment"></el-input></template>
+                    </el-table-column>
                     <el-table-column label="系统字段名">
                         <template v-slot="{row}">
                             <el-select
@@ -273,14 +281,6 @@
                     <el-table-column label="系统字段说明">
                         <template v-slot="{row}"><el-input v-model="row.targetComment" disabled></el-input></template>
                     </el-table-column>
-                    <el-table-column v-if="form.sourceType==='HTTP_API'" label="来源字段">
-                        <template v-slot="{row}">
-                            <el-input v-model="row.sourceField"></el-input>
-                        </template>
-                    </el-table-column>
-                    <el-table-column v-if="form.sourceType==='HTTP_API'" label="来源字段说明">
-                        <template v-slot="{row}"><el-input v-model="row.sourceComment"></el-input></template>
-                    </el-table-column>
                     <el-table-column label="是否必填" width="100">
                         <template v-slot="{row}"><el-switch v-model="row.required"></el-switch></template>
                     </el-table-column>
@@ -484,7 +484,7 @@
     </el-dialog>
     <el-dialog title="快捷映射" :visible.sync="quickMappingVisible" append-to-body
         width="720px" :close-on-click-modal="false" custom-class="person-governance-dialog">
-        <el-alert title="输入 JSON 数组，以系统字段名为键。仅更新填写的字段，确认后回填，保存配置后生效。"
+        <el-alert title="输入 JSON 数组，以目标系统字段名为键，id 填已有来源字段名。确认后回填右侧目标字段。"
             type="info" :closable="false"></el-alert>
         <el-input v-model="quickMappingText" type="textarea" :rows="12" :placeholder="quickMappingExample"></el-input>
         <div slot="footer">
@@ -558,7 +558,7 @@
                 mappingsClearConfirming: false,
                 quickMappingVisible: false,
                 quickMappingText: '',
-                quickMappingExample: '[{"loginname":{"id":"gh","sourceComment":"工号"}},\n {"username":{"id":"xm","sourceComment":"姓名"}}]',
+                quickMappingExample: '[{"loginname":{"id":"gh"}},\n {"username":{"id":"xm"}}]',
                 testResults: { TOKEN: '', DATA: '' },
                 responseWarnings: { TOKEN: '', DATA: '' },
                 queryLoading: { TOKEN: false, DATA: false },
@@ -749,40 +749,69 @@
                                 throw new Error(targetField + ' 的映射必须为对象')
                             }
                             const sourceField = value.sourceField === undefined ? value.id : value.sourceField
-                            const sourceComment = value.sourceComment === undefined ? value['来源字段说明'] : value.sourceComment
                             if (typeof sourceField !== 'string' || !sourceField.trim()) {
                                 throw new Error(targetField + ' 缺少来源字段 id 或 sourceField')
                             }
-                            if (sourceComment !== undefined && typeof sourceComment !== 'string') {
+                            if (this.form.sourceType === 'SYSTEM_TABLE'
-                                throw new Error(targetField + ' 的来源字段说明必须为字符串')
+                                && !this.sourceColumns.some((item) => item.columnName === sourceField.trim())) {
-                            }
-                            const source = this.sourceColumns.find((item) => item.columnName === sourceField.trim())
-                            if (this.form.sourceType === 'SYSTEM_TABLE' && !source) {
                                 throw new Error('来源表不存在字段：' + sourceField)
                             }
-                            updates.push({ column, sourceField: sourceField.trim(), sourceComment,
+                            updates.push({ column, sourceField: sourceField.trim() })
-                                defaultComment: source ? source.columnComment || '' : '' })
                         })
                     })
                 } catch (error) {
                     this.notifyWarning(error instanceof SyntaxError ? 'JSON 格式不正确，请使用双引号并参考示例填写' : error.message)
                     return
                 }
-                updates.forEach((update) => {
+                if (!this.fillQuickMappingTargets(updates)) return
-                    let row = this.mappings.find((item) => item.targetField === update.column.columnName)
-                    if (!row) {
-                        row = { sourceField: '', sourceComment: '', targetField: update.column.columnName,
-                            targetComment: update.column.columnComment || '', required: update.column.isNullable === 'NO', identityFlag: false }
-                        this.mappings.push(row)
-                    }
-                    this.$set(row, 'sourceField', update.sourceField)
-                    this.$set(row, 'sourceComment', update.sourceComment === undefined
-                        ? row.sourceComment || update.defaultComment : update.sourceComment)
-                })
                 this.$set(this, 'quickMappingVisible', false)
                 this.layoutTable('mappingTable')
                 this.notifySuccess('已填充 ' + updates.length + ' 条字段映射')
             },
+            fillQuickMappingTargets(updates) {
+                const targetKey = (field) => (field || '').trim().toLowerCase()
+                const assignments = new Map()
+                const targets = new Map(updates.map((update) => [targetKey(update.column.columnName), update]))
+                this.mappings.forEach((row) => {
+                    const update = targets.get(targetKey(row.targetField))
+                    if (update && update.sourceField === (row.sourceField || '').trim() && !update.row) {
+                        update.row = row
+                        assignments.set(row, update)
+                    }
+                })
+                for (const update of updates) {
+                    if (update.row) continue
+                    const available = this.mappings.filter((row) =>
+                        (row.sourceField || '').trim() === update.sourceField && !assignments.has(row))
+                    const row = available.find((item) => !item.targetField) || available[0]
+                    if (!row) {
+                        this.notifyWarning('没有可用的来源字段行：' + update.sourceField)
+                        return false
+                    }
+                    update.row = row
+                    assignments.set(row, update)
+                }
+                const identityTarget = this.identityRow && this.identityRow.targetField
+                this.mappings.forEach((row) => {
+                    const update = assignments.get(row)
+                    if (!update && !targets.has(targetKey(row.targetField))) return
+                    const column = update && update.column
+                    const targetField = column ? column.columnName : ''
+                    if (row.targetField !== targetField) {
+                        this.$set(row, 'targetField', targetField)
+                        this.$set(row, 'targetComment', column ? column.columnComment || '' : '')
+                        this.$set(row, 'dictCode', '')
+                        this.$set(row, 'defaultValue', '')
+                        this.$set(row, 'identityFlag', false)
+                    }
+                })
+                if (identityTarget) {
+                    const identity = this.mappings.find((row) =>
+                        targetKey(row.targetField) === targetKey(identityTarget) && this.isIdentityMapping(row))
+                    this.$set(this, 'identityRow', identity || null)
+                }
+                return true
+            },
             isIdentityMapping(row) {
                 return Boolean(row && row.sourceField && row.sourceField.trim()
                     && row.targetField && row.targetField.trim()
@@ -1219,7 +1248,7 @@
                     fields,
                     data: JSON.stringify({
                         config: this.form,
-                        params: this.params.filter((param) => param.requestScope === 'DATA'),
+                        params: this.params,
                     }),
                 }).then((res) => {
                     if (res.code === 0) {

tools/person-governance-api-mock.js

@@ -3,7 +3,7 @@
 /*
  * 人员数据治理 API 配置测试服务，仅使用 Node.js 内置模块。
  * 启动此文件后默认监听 http://127.0.0.1:18080，可通过 HOST、PORT 环境变量修改。
- * 所有人员信息均为模拟数据，每次启动生成相同的 100 条记录，不连接数据库。
+ * 所有人员信息均为模拟数据，每次启动生成相同的 240 条记录，不连接数据库。
  *
  * API 配置示例（地址中的主机需保证 Java 服务能够访问）：
  * Token 地址：http://127.0.0.1:18080/api/token，POST，JSON 请求体 {}
@@ -20,7 +20,7 @@
  * 人员配置：bmmz→unitId，值与机构接口的 dwdm 相同。
  * 分页 Query 参数：pageNumber=${页码}、pageSize=${数量}，例如起始页码 1、数量 20。
  * POST JSON 也可使用 {"pageNumber":"${页码}","pageSize":"${数量}"}。
- * 也支持 page 和 pagesize。不配置分页参数时返回全部 100 条人员或 5 条机构。
+ * 也支持 page 和 pagesize。不配置分页参数时默认返回第 1 页，每页 100 条。
  * 默认不要求认证；MOCK_REQUIRE_TOKEN=1 时，数据接口要求以下任意一种凭证：
  * Header Authorization: Bearer ${Access_Token}
  * Query 或 JSON 参数 access_token: ${Access_Token}
@@ -43,7 +43,7 @@ const units = [
     { dwdm: 'MOCK-D5', dwmc: '教务处', bmflmc: '行政单位' }
 ];
 
-const persons = Array.from({ length: 100 }, (_, index) => {
+const persons = Array.from({ length: 240 }, (_, index) => {
     const number = index + 1;
     const suffix = String(number).padStart(3, '0');
     const { dwdm: departmentCode, dwmc: departmentName } = units[index % units.length];