From 83cfd57245007ccf72f155d0471f7a3d5147ce94 Mon Sep 17 00:00:00 2001 From: luch <32321690+luch1994@users.noreply.github.com> Date: Thu, 4 Jan 2024 15:13:37 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E5=9B=9E=E6=94=B6?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=AD=98=E5=82=A8=E5=8A=A0=E5=AF=86=20(#37)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 4 +- server/package.json | 2 + server/src/apps/security/config/index.ts | 3 + .../dataSecurityPlugins/aesPlugin/index.ts | 35 +++++++++++ .../dataSecurityPlugins/aesPlugin/util.ts | 19 ++++++ .../security/dataSecurityPlugins/interface.ts | 6 ++ server/src/apps/security/index.ts | 44 +++++++++++++ server/src/apps/security/utils/index.ts | 4 +- .../surveyManage/service/surveyService.ts | 62 +++++++++---------- server/src/apps/surveyManage/utils/index.ts | 18 ------ server/src/apps/surveyPublish/config/index.ts | 6 +- .../service/surveySubmitService.ts | 35 +++++++++-- .../src/apps/surveyPublish/utils/checkSign.ts | 13 ++-- server/src/apps/user/config/index.ts | 2 +- server/src/apps/user/index.ts | 2 +- server/src/router.ts | 4 +- web/src/management/api/base.js | 2 - web/src/management/pages/analysis/index.vue | 22 ++++--- 18 files changed, 199 insertions(+), 84 deletions(-) create mode 100644 server/src/apps/security/dataSecurityPlugins/aesPlugin/index.ts create mode 100644 server/src/apps/security/dataSecurityPlugins/aesPlugin/util.ts create mode 100644 server/src/apps/security/dataSecurityPlugins/interface.ts diff --git a/Dockerfile b/Dockerfile index 877c5c46..b9d76658 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,9 +23,9 @@ RUN npm config set registry https://registry.npmjs.org/ # 安装项目依赖 RUN cd /xiaoju-survey/web && npm install && npm run build -RUN cd /xiaoju-survey/server && npm install && npm run build +RUN cd /xiaoju-survey && cp -af ./web/dist/* ./server/src/apps/ui/public/ -RUN cd /xiaoju-survey && mkdir -p ./build/apps/ui/public/ && cp -af ./web/dist/* ./server/build/apps/ui/public/ +RUN cd /xiaoju-survey/server && npm install && npm run copy && npm run build # 暴露端口 需要跟server的port一致 EXPOSE 3000 diff --git a/server/package.json b/server/package.json index f8bfe74b..d0b5c031 100644 --- a/server/package.json +++ b/server/package.json @@ -4,6 +4,7 @@ "description": "survey server template", "main": "index.js", "scripts": { + "copy": "mkdir -p ./build/ && cp -rf ./src/* ./build/", "build": "tsc", "start:stable": "SERVER_ENV=stable node ./build/index.js", "start:preonline": "SERVER_ENV=preonline node ./build/index.js", @@ -29,6 +30,7 @@ "typescript": "^4.8.4" }, "dependencies": { + "cheerio": "^1.0.0-rc.12", "crypto-js": "^4.2.0", "glob": "^10.3.10", "joi": "^17.9.2", diff --git a/server/src/apps/security/config/index.ts b/server/src/apps/security/config/index.ts index e13a5b72..7c235f48 100644 --- a/server/src/apps/security/config/index.ts +++ b/server/src/apps/security/config/index.ts @@ -2,6 +2,9 @@ const config = { mongo: { url: process.env.xiaojuSurveyMongoUrl || 'mongodb://localhost:27017', dbName: 'xiaojuSurvey' + }, + aesEncrypt: { + key: process.env.xiaojuSurveyDataAesEncryptSecretKey || 'dataAesEncryptSecretKey' } }; diff --git a/server/src/apps/security/dataSecurityPlugins/aesPlugin/index.ts b/server/src/apps/security/dataSecurityPlugins/aesPlugin/index.ts new file mode 100644 index 00000000..1dcc06ef --- /dev/null +++ b/server/src/apps/security/dataSecurityPlugins/aesPlugin/index.ts @@ -0,0 +1,35 @@ +import { DataSecurityPlugin } from '../interface'; +import * as CryptoJS from 'crypto-js'; +import { isAddress, isEmail, isGender, isIdCard, isPhone } from './util'; + + +export default class AesDataSecurityPlugin implements DataSecurityPlugin { + secretKey: string; + constructor({ secretKey }) { + this.secretKey = secretKey; + } + isDataSensitive(data: string): boolean { + const testArr = [isPhone, isIdCard, isAddress, isEmail, isGender]; + for (const test of testArr) { + if (test(data)) { + return true; + } + } + return false; + } + encryptData(data: string): string { + return CryptoJS.AES.encrypt(data, this.secretKey).toString(); + } + decryptData(data: string): string { + return CryptoJS.AES.decrypt(data, this.secretKey).toString(CryptoJS.enc.Utf8); + } + desensitiveData(data: string): string { + if (data.length === 1) { + return '*'; + } + if (data.length === 2) { + return data[0] + '*'; + } + return data[0] + '***' + data[data.length - 1]; + } +} \ No newline at end of file diff --git a/server/src/apps/security/dataSecurityPlugins/aesPlugin/util.ts b/server/src/apps/security/dataSecurityPlugins/aesPlugin/util.ts new file mode 100644 index 00000000..d2065488 --- /dev/null +++ b/server/src/apps/security/dataSecurityPlugins/aesPlugin/util.ts @@ -0,0 +1,19 @@ +const phoneRegex = /^1[3456789]\d{9}$/; // 手机号码正则表达式 +const idCardRegex = /(^\d{15}$)|(^\d{18}$)|(^\d{17}(\d|X|x)$)/; // 身份证号码正则表达式 +const addressRegex = /.*省|.*自治区|.*市|.*区|.*镇|.*县/; // 地址正则表达式 +const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; // 邮箱正则 +const genderArr = ['男', '女']; // 性别 +const nameRegex = /^([\u4e00-\u9fa5]{1,6}|[a-zA-Z.\s]{1,20})$/; // 只能识别是否包含中文,无法识别是否是姓名,暂时不启用 + + +export const isPhone = data => phoneRegex.test(data); + +export const isIdCard = data => idCardRegex.test(data); + +export const isAddress = data => addressRegex.test(data); + +export const isEmail = data => emailRegex.test(data); + +export const isGender = data => genderArr.includes(data); + +export const isName = data => nameRegex.test(data); \ No newline at end of file diff --git a/server/src/apps/security/dataSecurityPlugins/interface.ts b/server/src/apps/security/dataSecurityPlugins/interface.ts new file mode 100644 index 00000000..cb8a8696 --- /dev/null +++ b/server/src/apps/security/dataSecurityPlugins/interface.ts @@ -0,0 +1,6 @@ +export interface DataSecurityPlugin { + isDataSensitive(data: string): boolean; + encryptData(data: string): string; + decryptData(data: string): string; + desensitiveData(data: string): string; +} \ No newline at end of file diff --git a/server/src/apps/security/index.ts b/server/src/apps/security/index.ts index ad1e695f..51044a3a 100644 --- a/server/src/apps/security/index.ts +++ b/server/src/apps/security/index.ts @@ -1,5 +1,12 @@ import { SurveyApp, SurveyServer } from '../../decorator'; import { securityService } from './service/securityService'; +import { isString } from './utils'; +import AesDataSecurityPlugin from './dataSecurityPlugins/aesPlugin/index'; +import { load } from 'cheerio'; +import { getConfig } from './config/index'; + +const config = getConfig(); +const pluginInstance = new AesDataSecurityPlugin({ secretKey: config.aesEncrypt.key }); @SurveyApp('/api/security') export default class Security { @@ -14,4 +21,41 @@ export default class Security { context, // 上下文主要是传递调用方信息使用,比如traceid }; } + + @SurveyServer({ type: 'rpc' }) + isDataSensitive(data) { + if (!isString(data)) { + return false; + } + const $ = load(data); + const text = $.text(); + return pluginInstance.isDataSensitive(text); + } + + @SurveyServer({ type: 'rpc' }) + encryptData(data) { + if (!isString(data)) { + return data; + } + return pluginInstance.encryptData(data); + } + + @SurveyServer({ type: 'rpc' }) + decryptData(data) { + if (!isString(data)) { + return data; + } + return pluginInstance.decryptData(data); + } + + @SurveyServer({ type: 'rpc' }) + desensitiveData(data) { + // 数据脱敏 + if (!isString(data)) { + return '*'; + } + const $ = load(data); + const text = $.text(); + return pluginInstance.desensitiveData(text); + } } \ No newline at end of file diff --git a/server/src/apps/security/utils/index.ts b/server/src/apps/security/utils/index.ts index 5e73cdf4..cc836eae 100644 --- a/server/src/apps/security/utils/index.ts +++ b/server/src/apps/security/utils/index.ts @@ -13,4 +13,6 @@ export function participle({ content, minLen, maxLen }: { content: string, minLe } } return keys; -} \ No newline at end of file +} + +export const isString = data => typeof data === 'string'; diff --git a/server/src/apps/surveyManage/service/surveyService.ts b/server/src/apps/surveyManage/service/surveyService.ts index e6895f4a..355f7549 100644 --- a/server/src/apps/surveyManage/service/surveyService.ts +++ b/server/src/apps/surveyManage/service/surveyService.ts @@ -1,11 +1,10 @@ import { mongo } from '../db/mongo'; import { rpcInvote } from '../../../rpc'; import { SURVEY_STATUS, QUESTION_TYPE, CommonError, UserType, DICT_TYPE } from '../../../types/index'; -import { getStatusObject, genSurveyPath, hanleSensitiveDate } from '../utils/index'; +import { getStatusObject, genSurveyPath } from '../utils/index'; import * as path from 'path'; -import * as _ from 'lodash'; +import { keyBy, merge, cloneDeep } from 'lodash'; import * as moment from 'moment'; -import { getFile } from '../utils/index'; import { DataItem } from '../../../types/survey'; class SurveyService { @@ -19,7 +18,7 @@ class SurveyService { async getBannerData() { const bannerConfPath = path.resolve(__dirname, '../template/banner/index.json'); - return require(bannerConfPath); + return await import(bannerConfPath); } async getCodeData({ @@ -31,11 +30,9 @@ class SurveyService { `../template/surveyTemplate/survey/${questionType}.json`, ); - const [baseConfStr, templateConfStr] = await Promise.all([getFile(baseConfPath), getFile(templateConfPath)]); - - const baseConf = JSON.parse(baseConfStr); - const templateConf = JSON.parse(templateConfStr); - const codeData = _.merge(baseConf, templateConf); + const baseConf = cloneDeep(await import(baseConfPath)); + const templateConf = cloneDeep(await import(templateConfPath)); + const codeData = merge(baseConf, templateConf); const nowMoment = moment(); codeData.baseConf.begTime = nowMoment.format('YYYY-MM-DD HH:mm:ss'); codeData.baseConf.endTime = nowMoment.add(10, 'years').format('YYYY-MM-DD HH:mm:ss'); @@ -192,49 +189,50 @@ class SurveyService { const publishConf = await surveyPublish.findOne({ pageId: condition.surveyId }); const dataList = publishConf?.code?.dataConf?.dataList || []; const listHead = this.getListHeadByDataList(dataList); - const dataListMap = _.keyBy(dataList, 'field'); + const dataListMap = keyBy(dataList, 'field'); const surveySubmit = await mongo.getCollection({ collectionName: 'surveySubmit' }); - const surveySubmitData = await surveySubmit.find({ pageId: condition.surveyId }) + const surveySubmitDataList = await surveySubmit.find({ pageId: condition.surveyId }) .sort({ createDate: -1 }) .limit(condition.pageSize) .skip((condition.pageNum - 1) * condition.pageSize) .toArray(); - const listBody = surveySubmitData.map((surveySubmitResList) => { - const data = surveySubmitResList.data; + + const listBody = surveySubmitDataList.map(submitedData => { + const data = submitedData.data; + const secretKeys = submitedData.secretKeys; const dataKeys = Object.keys(data); + for (const itemKey of dataKeys) { if (typeof itemKey !== 'string') { continue; } if (itemKey.indexOf('data') !== 0) { continue; } + // 获取题目id const itemConfigKey = itemKey.split('_')[0]; + // 获取题目 const itemConfig: DataItem = dataListMap[itemConfigKey]; // 题目删除会出现,数据列表报错 if (!itemConfig) { continue; } - const doSecretData = (data) => { - if (condition.isShowSecret) { - return hanleSensitiveDate(data); - } else { - return data; - } - }; - data[itemKey] = doSecretData(data[itemKey]); - // 处理选项 + // 处理选项的更多输入框 if (itemConfig.type === 'radio-star' && !data[`${itemConfigKey}_custom`]) { data[`${itemConfigKey}_custom`] = data[`${itemConfigKey}_${data[itemConfigKey]}`]; } - if (!itemConfig?.options?.length) { continue; } - const options = itemConfig.options; - const optionsMap = _.keyBy(options, 'hash'); - const getText = e => doSecretData(optionsMap?.[e]?.text || e); - if (Array.isArray(data[itemKey])) { - data[itemKey] = data[itemKey].map(getText); - } else { - data[itemKey] = getText(data[itemKey]); + // 解密数据 + if (secretKeys.includes(itemKey)) { + data[itemKey] = Array.isArray(data[itemKey]) ? data[itemKey].map(item => rpcInvote('security.decryptData', item)) : rpcInvote('security.decryptData', data[itemKey]); + } + // 将选项id还原成选项文案 + if (Array.isArray(itemConfig.options) && itemConfig.options.length > 0) { + const optionTextMap = keyBy(itemConfig.options, 'hash'); + data[itemKey] = Array.isArray(data[itemKey]) ? data[itemKey].map(item => optionTextMap[item]?.text || item).join(',') : optionTextMap[data[itemKey]]?.text || data[itemKey]; + } + // 数据脱敏 + if (condition.isShowSecret && rpcInvote('security.isDataSensitive', data[itemKey])) { + data[itemKey] = rpcInvote('security.desensitiveData', data[itemKey]); } } return { ...data, - difTime: (surveySubmitResList.difTime / 1000).toFixed(2), - createDate: moment(surveySubmitResList.createDate).format('YYYY-MM-DD HH:mm:ss') + difTime: (submitedData.difTime / 1000).toFixed(2), + createDate: moment(submitedData.createDate).format('YYYY-MM-DD HH:mm:ss') }; }); const total = await surveySubmit.countDocuments({ pageId: condition.surveyId }); diff --git a/server/src/apps/surveyManage/utils/index.ts b/server/src/apps/surveyManage/utils/index.ts index 80d2b5a4..fb4c856a 100644 --- a/server/src/apps/surveyManage/utils/index.ts +++ b/server/src/apps/surveyManage/utils/index.ts @@ -22,24 +22,6 @@ export function genSurveyPath() { return hex2Base58(process.hrtime.bigint().toString(16)); } -export function hanleSensitiveDate(value: string = ''): string { - if (!value) { - return '*'; - } - - let str = '' + value; - if (str.length === 1) { - str = '*'; - } - if (str.length === 2) { - str = str[0] + '*'; - } - if (str.length >= 3) { - str = str[0] + '***' + str.slice(str.length - 1); - } - return str; -} - export const getFile = function(path, { encoding }: { encoding } = { encoding: 'utf-8' }): Promise { return new Promise((resolve, reject) => { diff --git a/server/src/apps/surveyPublish/config/index.ts b/server/src/apps/surveyPublish/config/index.ts index 5ded0551..6353423e 100644 --- a/server/src/apps/surveyPublish/config/index.ts +++ b/server/src/apps/surveyPublish/config/index.ts @@ -1,13 +1,13 @@ const config = { mongo: { - url: process.env.xiaojuSurveyMongoUrl ||'mongodb://localhost:27017', + url: process.env.xiaojuSurveyMongoUrl || 'mongodb://localhost:27017', dbName: 'xiaojuSurvey', }, session: { - expireTime: parseInt(process.env.xiaojuSurveySessionExpireTime) || 8*3600*1000 + expireTime: parseInt(process.env.xiaojuSurveySessionExpireTime) || 8 * 3600 * 1000 }, encrypt: { - type: process.env.xiaojuSurveyEncryptType ||'aes', + type: process.env.xiaojuSurveyEncryptType || 'aes', aesCodelength: parseInt(process.env.xiaojuSurveyAesCodelength) || 10 //aes密钥长度 } }; diff --git a/server/src/apps/surveyPublish/service/surveySubmitService.ts b/server/src/apps/surveyPublish/service/surveySubmitService.ts index 8fd97ba1..d17915d9 100644 --- a/server/src/apps/surveyPublish/service/surveySubmitService.ts +++ b/server/src/apps/surveyPublish/service/surveySubmitService.ts @@ -7,6 +7,7 @@ import * as CryptoJS from 'crypto-js'; import * as aes from 'crypto-js/aes'; import * as moment from 'moment'; import { keyBy } from 'lodash'; +import { rpcInvote } from '../../../rpc'; const config = getConfig(); @@ -98,26 +99,48 @@ class SurveySubmitService { throw new CommonError('超出提交总数限制'); } } - // 投票信息保存 const dataList = publishConf?.code?.dataConf?.dataList || []; const dataListMap = keyBy(dataList, 'field'); + const surveySubmitDataKeys = Object.keys(surveySubmitData.data); + + const secretKeys = []; + for (const field of surveySubmitDataKeys) { const configData = dataListMap[field]; + const value = surveySubmitData.data[field]; + const values = Array.isArray(value) ? value : [value]; + if (configData && /vote/.exec(configData.type)) { + // 投票信息保存 const voteData = (await surveyKeyStoreService.get({ surveyPath: surveySubmitData.surveyPath, key: field, type: 'vote' })) || { total: 0 }; voteData.total++; - const fields = Array.isArray(surveySubmitData.data[field]) ? surveySubmitData.data[field] : [surveySubmitData.data[field]]; - for (const field of fields) { - if (!voteData[field]) { - voteData[field] = 1; + for (const val of values) { + if (!voteData[val]) { + voteData[val] = 1; } else { - voteData[field]++; + voteData[val]++; } } await surveyKeyStoreService.set({ surveyPath: surveySubmitData.surveyPath, key: field, data: voteData, type: 'vote' }); } + // 检查敏感数据,对敏感数据进行加密存储 + let isSecret = false; + for (const val of values) { + if (rpcInvote('security.isDataSensitive', val)) { + isSecret = true; + break; + } + } + if (isSecret) { + secretKeys.push(field); + surveySubmitData.data[field] = Array.isArray(value) ? value.map(item => rpcInvote('security.encryptData', item)) : rpcInvote('security.encryptData', value); + } + } + + surveySubmitData.secretKeys = secretKeys; + // 提交问卷 const surveySubmitRes = await surveySubmit.insertOne({ ...surveySubmitData, diff --git a/server/src/apps/surveyPublish/utils/checkSign.ts b/server/src/apps/surveyPublish/utils/checkSign.ts index 978e02cf..f970eafd 100644 --- a/server/src/apps/surveyPublish/utils/checkSign.ts +++ b/server/src/apps/surveyPublish/utils/checkSign.ts @@ -1,6 +1,4 @@ -import { - createHash -} from 'crypto'; +import { createHash } from 'crypto'; import { CommonError } from '../../../types/index'; const hash256 = (text) => { @@ -17,7 +15,7 @@ const undefinedToString = (data) => { } return res; }; - + const getSignByData = (sourceData, ts) => { const data = undefinedToString(sourceData); const keysArr = Object.keys(data); @@ -33,16 +31,15 @@ const getSignByData = (sourceData, ts) => { }; export const checkSign = (sourceData) => { - const sign = sourceData.sign; - if(!sign) { + const sign = sourceData.sign; + if (!sign) { throw new CommonError('请求签名不存在'); } delete sourceData.sign; const [inSign, ts] = sign.split('.'); const realSign = getSignByData(sourceData, ts); - if(inSign!==realSign) { + if (inSign !== realSign) { throw new CommonError('请求签名异常'); } return true; }; - \ No newline at end of file diff --git a/server/src/apps/user/config/index.ts b/server/src/apps/user/config/index.ts index 265b6c9d..5401164f 100644 --- a/server/src/apps/user/config/index.ts +++ b/server/src/apps/user/config/index.ts @@ -1,6 +1,6 @@ const config = { mongo: { - url: process.env.xiaojuSurveyMongoUrl ||'mongodb://localhost:27017', + url: process.env.xiaojuSurveyMongoUrl || 'mongodb://localhost:27017', dbName: 'xiaojuSurvey', }, jwt: { diff --git a/server/src/apps/user/index.ts b/server/src/apps/user/index.ts index 117bb5c2..0c133b85 100644 --- a/server/src/apps/user/index.ts +++ b/server/src/apps/user/index.ts @@ -49,7 +49,7 @@ export default class User { username: userInfo.username, password: userInfo.password, }); - // 删除验证码 + // 删除验证码 captchaService.deleteCaptcha({ id: userInfo.captchaId }); return { code: 200, diff --git a/server/src/router.ts b/server/src/router.ts index 70b24d0d..eac1cd1f 100644 --- a/server/src/router.ts +++ b/server/src/router.ts @@ -8,7 +8,9 @@ import appRegistry from './registry'; export async function initRouter(app) { const rootRouter = new Router(); - const entries = await glob(path.join(__dirname, './apps/*/index.{ts,js}')); + const jsEntries = await glob(path.join(__dirname, './apps/*/index.js')); + const tsEntries = await glob(path.join(__dirname, './apps/*/index.ts')); + const entries = Array.isArray(jsEntries) && jsEntries.length > 0 ? jsEntries : tsEntries; for (const entry of entries) { const module = await import(entry); diff --git a/web/src/management/api/base.js b/web/src/management/api/base.js index 9e6c50c2..c096d79d 100644 --- a/web/src/management/api/base.js +++ b/web/src/management/api/base.js @@ -2,7 +2,6 @@ import axios from 'axios'; import store from '@/management/store/index'; import router from '@/management/router/index'; import { get as _get } from 'lodash'; -import { Message } from 'element-ui'; const instance = axios.create({ baseURL: '/api', @@ -25,7 +24,6 @@ instance.interceptors.response.use( } }, (err) => { - Message.error(err || 'http请求出错'); throw new Error(err); } ); diff --git a/web/src/management/pages/analysis/index.vue b/web/src/management/pages/analysis/index.vue index 4afd88ef..c8d6b1eb 100644 --- a/web/src/management/pages/analysis/index.vue +++ b/web/src/management/pages/analysis/index.vue @@ -72,16 +72,20 @@ export default { return; } this.mainTableLoading = true; - const res = await getRecycleList({ - page: this.currentPage, - surveyId: this.$route.params.id, - isShowSecret: !this.tmpIsShowOriginData, // 发起请求的时候,isShowOriginData还没改变,暂存了一个字段 - }); + try { + const res = await getRecycleList({ + page: this.currentPage, + surveyId: this.$route.params.id, + isShowSecret: !this.tmpIsShowOriginData, // 发起请求的时候,isShowOriginData还没改变,暂存了一个字段 + }); - if (res.code === 200) { - const listHead = this.formatHead(res.data.listHead); - this.tableData = { ...res.data, listHead }; - this.mainTableLoading = false; + if (res.code === 200) { + const listHead = this.formatHead(res.data.listHead); + this.tableData = { ...res.data, listHead }; + this.mainTableLoading = false; + } + } catch (error) { + this.$message.error('查询回收数据失败,请重试'); } }, handleCurrentChange(current) {