feat: 新增回收数据存储加密 (#37)

2024-01-04 15:13:37 +08:00 · 2024-01-04 15:13:37 +08:00 · 83cfd57245
commit 83cfd57245
parent 08f3bb0578
18 changed files with 199 additions and 84 deletions
--- a/4
+++ b/4
@ -23,9 +23,9 @@ RUN npm config set registry https://registry.npmjs.org/
 # 安装项目依赖
 RUN cd /xiaoju-survey/web && npm install && npm run build

-RUN cd /xiaoju-survey/server && npm install && npm run build
+RUN cd /xiaoju-survey && cp -af ./web/dist/* ./server/src/apps/ui/public/

-RUN cd /xiaoju-survey && mkdir -p ./build/apps/ui/public/ && cp -af ./web/dist/* ./server/build/apps/ui/public/
+RUN cd /xiaoju-survey/server && npm install && npm run copy && npm run build

 # 暴露端口 需要跟server的port一致
 EXPOSE 3000
--- a/server/package.json
+++ b/server/package.json
@ -4,6 +4,7 @@
  "description": "survey server template",
  "main": "index.js",
  "scripts": {
+    "copy": "mkdir -p ./build/ && cp -rf ./src/* ./build/",
    "build": "tsc",
    "start:stable": "SERVER_ENV=stable node ./build/index.js",
    "start:preonline": "SERVER_ENV=preonline node ./build/index.js",
@ -29,6 +30,7 @@
    "typescript": "^4.8.4"
  },
  "dependencies": {
+    "cheerio": "^1.0.0-rc.12",
    "crypto-js": "^4.2.0",
    "glob": "^10.3.10",
    "joi": "^17.9.2",
--- a/server/src/apps/security/config/index.ts
+++ b/server/src/apps/security/config/index.ts
@ -2,6 +2,9 @@ const config = {
  mongo: {
    url: process.env.xiaojuSurveyMongoUrl || 'mongodb://localhost:27017',
    dbName: 'xiaojuSurvey'
+  },
+  aesEncrypt: {
+    key: process.env.xiaojuSurveyDataAesEncryptSecretKey || 'dataAesEncryptSecretKey'
  }
 };

--- a/server/src/apps/security/dataSecurityPlugins/aesPlugin/index.ts
+++ b/server/src/apps/security/dataSecurityPlugins/aesPlugin/index.ts
@ -0,0 +1,35 @@
+import { DataSecurityPlugin } from '../interface';
+import * as CryptoJS from 'crypto-js';
+import { isAddress, isEmail, isGender, isIdCard, isPhone } from './util';
+
+
+export default class AesDataSecurityPlugin implements DataSecurityPlugin {
+  secretKey: string;
+  constructor({ secretKey }) {
+    this.secretKey = secretKey;
+  }
+  isDataSensitive(data: string): boolean {
+    const testArr = [isPhone, isIdCard, isAddress, isEmail, isGender];
+    for (const test of testArr) {
+      if (test(data)) {
+        return true;
+      }
+    }
+    return false;
+  }
+  encryptData(data: string): string {
+    return CryptoJS.AES.encrypt(data, this.secretKey).toString();
+  }
+  decryptData(data: string): string {
+    return CryptoJS.AES.decrypt(data, this.secretKey).toString(CryptoJS.enc.Utf8);
+  }
+  desensitiveData(data: string): string {
+    if (data.length === 1) {
+      return '*';
+    }
+    if (data.length === 2) {
+      return data[0] + '*';
+    }
+    return data[0] + '***' + data[data.length - 1];
+  }
+}
--- a/server/src/apps/security/dataSecurityPlugins/aesPlugin/util.ts
+++ b/server/src/apps/security/dataSecurityPlugins/aesPlugin/util.ts
@ -0,0 +1,19 @@
+const phoneRegex = /^1[3456789]\d{9}$/; // 手机号码正则表达式
+const idCardRegex = /(^\d{15}$)|(^\d{18}$)|(^\d{17}(\d|X|x)$)/; // 身份证号码正则表达式
+const addressRegex = /.*省|.*自治区|.*市|.*区|.*镇|.*县/; // 地址正则表达式
+const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; // 邮箱正则
+const genderArr = ['男', '女']; // 性别
+const nameRegex = /^([\u4e00-\u9fa5]{1,6}|[a-zA-Z.\s]{1,20})$/; // 只能识别是否包含中文，无法识别是否是姓名，暂时不启用
+
+
+export const isPhone = data => phoneRegex.test(data);
+
+export const isIdCard = data => idCardRegex.test(data);
+
+export const isAddress = data => addressRegex.test(data);
+
+export const isEmail = data => emailRegex.test(data);
+
+export const isGender = data => genderArr.includes(data);
+
+export const isName = data => nameRegex.test(data);
--- a/server/src/apps/security/dataSecurityPlugins/interface.ts
+++ b/server/src/apps/security/dataSecurityPlugins/interface.ts
@ -0,0 +1,6 @@
+export interface DataSecurityPlugin {
+  isDataSensitive(data: string): boolean;
+  encryptData(data: string): string;
+  decryptData(data: string): string;
+  desensitiveData(data: string): string;
+}
--- a/server/src/apps/security/index.ts
+++ b/server/src/apps/security/index.ts
@ -1,5 +1,12 @@
 import { SurveyApp, SurveyServer } from '../../decorator';
 import { securityService } from './service/securityService';
+import { isString } from './utils';
+import AesDataSecurityPlugin from './dataSecurityPlugins/aesPlugin/index';
+import { load } from 'cheerio';
+import { getConfig } from './config/index';
+
+const config = getConfig();
+const pluginInstance = new AesDataSecurityPlugin({ secretKey: config.aesEncrypt.key });

@SurveyApp('/api/security')
 export default class Security {
@ -14,4 +21,41 @@ export default class Security {
      context, // 上下文主要是传递调用方信息使用，比如traceid
    };
  }
+
+  @SurveyServer({ type: 'rpc' })
+  isDataSensitive(data) {
+    if (!isString(data)) {
+      return false;
+    }
+    const $ = load(data);
+    const text = $.text();
+    return pluginInstance.isDataSensitive(text);
+  }
+
+  @SurveyServer({ type: 'rpc' })
+  encryptData(data) {
+    if (!isString(data)) {
+      return data;
+    }
+    return pluginInstance.encryptData(data);
+  }
+
+  @SurveyServer({ type: 'rpc' })
+  decryptData(data) {
+    if (!isString(data)) {
+      return data;
+    }
+    return pluginInstance.decryptData(data);
+  }
+
+  @SurveyServer({ type: 'rpc' })
+  desensitiveData(data) {
+    // 数据脱敏
+    if (!isString(data)) {
+      return '*';
+    }
+    const $ = load(data);
+    const text = $.text();
+    return pluginInstance.desensitiveData(text);
+  }
 }
--- a/server/src/apps/security/utils/index.ts
+++ b/server/src/apps/security/utils/index.ts
@ -13,4 +13,6 @@ export function participle({ content, minLen, maxLen }: { content: string, minLe
    }
  }
  return keys;
-}
+}
+
+export const isString = data => typeof data === 'string';
--- a/server/src/apps/surveyManage/service/surveyService.ts
+++ b/server/src/apps/surveyManage/service/surveyService.ts
@ -1,11 +1,10 @@
 import { mongo } from '../db/mongo';
 import { rpcInvote } from '../../../rpc';
 import { SURVEY_STATUS, QUESTION_TYPE, CommonError, UserType, DICT_TYPE } from '../../../types/index';
-import { getStatusObject, genSurveyPath, hanleSensitiveDate } from '../utils/index';
+import { getStatusObject, genSurveyPath } from '../utils/index';
 import * as path from 'path';
-import * as _ from 'lodash';
+import { keyBy, merge, cloneDeep } from 'lodash';
 import * as moment from 'moment';
-import { getFile } from '../utils/index';
 import { DataItem } from '../../../types/survey';

 class SurveyService {
@ -19,7 +18,7 @@ class SurveyService {

  async getBannerData() {
    const bannerConfPath = path.resolve(__dirname, '../template/banner/index.json');
-    return require(bannerConfPath);
+    return await import(bannerConfPath);
  }

  async getCodeData({
@ -31,11 +30,9 @@ class SurveyService {
      `../template/surveyTemplate/survey/${questionType}.json`,
    );

-    const [baseConfStr, templateConfStr] = await Promise.all([getFile(baseConfPath), getFile(templateConfPath)]);
-
-    const baseConf = JSON.parse(baseConfStr);
-    const templateConf = JSON.parse(templateConfStr);
-    const codeData = _.merge(baseConf, templateConf);
+    const baseConf = cloneDeep(await import(baseConfPath));
+    const templateConf = cloneDeep(await import(templateConfPath));
+    const codeData = merge(baseConf, templateConf);
    const nowMoment = moment();
    codeData.baseConf.begTime = nowMoment.format('YYYY-MM-DD HH:mm:ss');
    codeData.baseConf.endTime = nowMoment.add(10, 'years').format('YYYY-MM-DD HH:mm:ss');
@ -192,49 +189,50 @@ class SurveyService {
    const publishConf = await surveyPublish.findOne({ pageId: condition.surveyId });
    const dataList = publishConf?.code?.dataConf?.dataList || [];
    const listHead = this.getListHeadByDataList(dataList);
-    const dataListMap = _.keyBy(dataList, 'field');
+    const dataListMap = keyBy(dataList, 'field');
    const surveySubmit = await mongo.getCollection({ collectionName: 'surveySubmit' });
-    const surveySubmitData = await surveySubmit.find({ pageId: condition.surveyId })
+    const surveySubmitDataList = await surveySubmit.find({ pageId: condition.surveyId })
      .sort({ createDate: -1 })
      .limit(condition.pageSize)
      .skip((condition.pageNum - 1) * condition.pageSize)
      .toArray();
-    const listBody = surveySubmitData.map((surveySubmitResList) => {
-      const data = surveySubmitResList.data;
+
+    const listBody = surveySubmitDataList.map(submitedData => {
+      const data = submitedData.data;
+      const secretKeys = submitedData.secretKeys;
      const dataKeys = Object.keys(data);
+
      for (const itemKey of dataKeys) {
        if (typeof itemKey !== 'string') { continue; }
        if (itemKey.indexOf('data') !== 0) { continue; }
+        // 获取题目id
        const itemConfigKey = itemKey.split('_')[0];
+        // 获取题目
        const itemConfig: DataItem = dataListMap[itemConfigKey];
        // 题目删除会出现，数据列表报错
        if (!itemConfig) { continue; }
-        const doSecretData = (data) => {
-          if (condition.isShowSecret) {
-            return hanleSensitiveDate(data);
-          } else {
-            return data;
-          }
-        };
-        data[itemKey] = doSecretData(data[itemKey]);
-        // 处理选项
+        // 处理选项的更多输入框
        if (itemConfig.type === 'radio-star' && !data[`${itemConfigKey}_custom`]) {
          data[`${itemConfigKey}_custom`] = data[`${itemConfigKey}_${data[itemConfigKey]}`];
        }
-        if (!itemConfig?.options?.length) { continue; }
-        const options = itemConfig.options;
-        const optionsMap = _.keyBy(options, 'hash');
-        const getText = e => doSecretData(optionsMap?.[e]?.text || e);
-        if (Array.isArray(data[itemKey])) {
-          data[itemKey] = data[itemKey].map(getText);
-        } else {
-          data[itemKey] = getText(data[itemKey]);
+        // 解密数据
+        if (secretKeys.includes(itemKey)) {
+          data[itemKey] = Array.isArray(data[itemKey]) ? data[itemKey].map(item => rpcInvote('security.decryptData', item)) : rpcInvote('security.decryptData', data[itemKey]);
+        }
+        // 将选项id还原成选项文案
+        if (Array.isArray(itemConfig.options) && itemConfig.options.length > 0) {
+          const optionTextMap = keyBy(itemConfig.options, 'hash');
+          data[itemKey] = Array.isArray(data[itemKey]) ? data[itemKey].map(item => optionTextMap[item]?.text || item).join(',') : optionTextMap[data[itemKey]]?.text || data[itemKey];
+        }
+        // 数据脱敏
+        if (condition.isShowSecret && rpcInvote('security.isDataSensitive', data[itemKey])) {
+          data[itemKey] = rpcInvote('security.desensitiveData', data[itemKey]);
        }
      }
      return {
        ...data,
-        difTime: (surveySubmitResList.difTime / 1000).toFixed(2),
-        createDate: moment(surveySubmitResList.createDate).format('YYYY-MM-DD HH:mm:ss') 
+        difTime: (submitedData.difTime / 1000).toFixed(2),
+        createDate: moment(submitedData.createDate).format('YYYY-MM-DD HH:mm:ss')
      };
    });
    const total = await surveySubmit.countDocuments({ pageId: condition.surveyId });
--- a/server/src/apps/surveyManage/utils/index.ts
+++ b/server/src/apps/surveyManage/utils/index.ts
@ -22,24 +22,6 @@ export function genSurveyPath() {
  return hex2Base58(process.hrtime.bigint().toString(16));
 }

-export function hanleSensitiveDate(value: string = ''): string {
-  if (!value) {
-    return '*';
-  }
-
-  let str = '' + value;
-  if (str.length === 1) {
-    str = '*';
-  }
-  if (str.length === 2) {
-    str = str[0] + '*';
-  }
-  if (str.length >= 3) {
-    str = str[0] + '***' + str.slice(str.length - 1);
-  }
-  return str;
-}
-
 export const getFile = function(path, { encoding }: { encoding } = { encoding: 'utf-8' }): Promise<string> {
  return new Promise((resolve, reject) => {

--- a/server/src/apps/surveyPublish/config/index.ts
+++ b/server/src/apps/surveyPublish/config/index.ts
@ -1,13 +1,13 @@
 const config = {
  mongo: {
-    url: process.env.xiaojuSurveyMongoUrl ||'mongodb://localhost:27017',
+    url: process.env.xiaojuSurveyMongoUrl || 'mongodb://localhost:27017',
    dbName: 'xiaojuSurvey',
  },
  session: {
-    expireTime: parseInt(process.env.xiaojuSurveySessionExpireTime) || 8*3600*1000
+    expireTime: parseInt(process.env.xiaojuSurveySessionExpireTime) || 8 * 3600 * 1000
  },
  encrypt: {
-    type: process.env.xiaojuSurveyEncryptType ||'aes',
+    type: process.env.xiaojuSurveyEncryptType || 'aes',
    aesCodelength: parseInt(process.env.xiaojuSurveyAesCodelength) || 10 //aes密钥长度
  }
 };
--- a/server/src/apps/surveyPublish/service/surveySubmitService.ts
+++ b/server/src/apps/surveyPublish/service/surveySubmitService.ts
@ -7,6 +7,7 @@ import * as CryptoJS from 'crypto-js';
 import * as aes from 'crypto-js/aes';
 import * as moment from 'moment';
 import { keyBy } from 'lodash';
+import { rpcInvote } from '../../../rpc';

 const config = getConfig();

@ -98,26 +99,48 @@ class SurveySubmitService {
        throw new CommonError('超出提交总数限制');
      }
    }
-    // 投票信息保存
    const dataList = publishConf?.code?.dataConf?.dataList || [];
    const dataListMap = keyBy(dataList, 'field');
+
    const surveySubmitDataKeys = Object.keys(surveySubmitData.data);
+
+    const secretKeys = [];
+
    for (const field of surveySubmitDataKeys) {
      const configData = dataListMap[field];
+      const value = surveySubmitData.data[field];
+      const values = Array.isArray(value) ? value : [value];
+
      if (configData && /vote/.exec(configData.type)) {
+        // 投票信息保存
        const voteData = (await surveyKeyStoreService.get({ surveyPath: surveySubmitData.surveyPath, key: field, type: 'vote' })) || { total: 0 };
        voteData.total++;
-        const fields = Array.isArray(surveySubmitData.data[field]) ? surveySubmitData.data[field] : [surveySubmitData.data[field]];
-        for (const field of fields) {
-          if (!voteData[field]) {
-            voteData[field] = 1;
+        for (const val of values) {
+          if (!voteData[val]) {
+            voteData[val] = 1;
          } else {
-            voteData[field]++;
+            voteData[val]++;
          }
        }
        await surveyKeyStoreService.set({ surveyPath: surveySubmitData.surveyPath, key: field, data: voteData, type: 'vote' });
      }
+      // 检查敏感数据，对敏感数据进行加密存储
+      let isSecret = false;
+      for (const val of values) {
+        if (rpcInvote('security.isDataSensitive', val)) {
+          isSecret = true;
+          break;
+        }
+      }
+      if (isSecret) {
+        secretKeys.push(field);
+        surveySubmitData.data[field] = Array.isArray(value) ? value.map(item => rpcInvote('security.encryptData', item)) : rpcInvote('security.encryptData', value);
+      }
+      
    }
+
+    surveySubmitData.secretKeys = secretKeys;
+
    // 提交问卷
    const surveySubmitRes = await surveySubmit.insertOne({
      ...surveySubmitData,
--- a/server/src/apps/surveyPublish/utils/checkSign.ts
+++ b/server/src/apps/surveyPublish/utils/checkSign.ts
@ -1,6 +1,4 @@
-import { 
-  createHash 
-} from 'crypto';
+import { createHash } from 'crypto';
 import { CommonError } from '../../../types/index';

 const hash256 = (text) => {
@ -17,7 +15,7 @@ const undefinedToString = (data) => {
  }
  return res;
 };
-  
+
 const getSignByData = (sourceData, ts) => {
  const data = undefinedToString(sourceData);
  const keysArr = Object.keys(data);
@ -33,16 +31,15 @@ const getSignByData = (sourceData, ts) => {
 };

 export const checkSign = (sourceData) => {
-  const sign = sourceData.sign; 
-  if(!sign) {
+  const sign = sourceData.sign;
+  if (!sign) {
    throw new CommonError('请求签名不存在');
  }
  delete sourceData.sign;
  const [inSign, ts] = sign.split('.');
  const realSign = getSignByData(sourceData, ts);
-  if(inSign!==realSign) {
+  if (inSign !== realSign) {
    throw new CommonError('请求签名异常');
  }
  return true;
 };
-  
--- a/server/src/apps/user/config/index.ts
+++ b/server/src/apps/user/config/index.ts
@ -1,6 +1,6 @@
 const config = {
  mongo: {
-    url: process.env.xiaojuSurveyMongoUrl ||'mongodb://localhost:27017',
+    url: process.env.xiaojuSurveyMongoUrl || 'mongodb://localhost:27017',
    dbName: 'xiaojuSurvey',
  },
  jwt: {
--- a/server/src/apps/user/index.ts
+++ b/server/src/apps/user/index.ts
@ -49,7 +49,7 @@ export default class User {
      username: userInfo.username,
      password: userInfo.password,
    });
-      // 删除验证码
+    // 删除验证码
    captchaService.deleteCaptcha({ id: userInfo.captchaId });
    return {
      code: 200,
--- a/server/src/router.ts
+++ b/server/src/router.ts
@ -8,7 +8,9 @@ import appRegistry from './registry';

 export async function initRouter(app) {
  const rootRouter = new Router();
-  const entries = await glob(path.join(__dirname, './apps/*/index.{ts,js}'));
+  const jsEntries = await glob(path.join(__dirname, './apps/*/index.js'));
+  const tsEntries = await glob(path.join(__dirname, './apps/*/index.ts'));
+  const entries = Array.isArray(jsEntries) && jsEntries.length > 0 ? jsEntries : tsEntries;

  for (const entry of entries) {
    const module = await import(entry);
--- a/web/src/management/api/base.js
+++ b/web/src/management/api/base.js
@ -2,7 +2,6 @@ import axios from 'axios';
 import store from '@/management/store/index';
 import router from '@/management/router/index';
 import { get as _get } from 'lodash';
-import { Message } from 'element-ui';

 const instance = axios.create({
  baseURL: '/api',
@ -25,7 +24,6 @@ instance.interceptors.response.use(
    }
  },
  (err) => {
-    Message.error(err || 'http请求出错');
    throw new Error(err);
  }
 );
--- a/web/src/management/pages/analysis/index.vue
+++ b/web/src/management/pages/analysis/index.vue
@ -72,16 +72,20 @@ export default {
        return;
      }
      this.mainTableLoading = true;
-      const res = await getRecycleList({
-        page: this.currentPage,
-        surveyId: this.$route.params.id,
-        isShowSecret: !this.tmpIsShowOriginData, // 发起请求的时候，isShowOriginData还没改变，暂存了一个字段
-      });
+      try {
+        const res = await getRecycleList({
+          page: this.currentPage,
+          surveyId: this.$route.params.id,
+          isShowSecret: !this.tmpIsShowOriginData, // 发起请求的时候，isShowOriginData还没改变，暂存了一个字段
+        });

-      if (res.code === 200) {
-        const listHead = this.formatHead(res.data.listHead);
-        this.tableData = { ...res.data, listHead };
-        this.mainTableLoading = false;
+        if (res.code === 200) {
+          const listHead = this.formatHead(res.data.listHead);
+          this.tableData = { ...res.data, listHead };
+          this.mainTableLoading = false;
+        }
+      } catch (error) {
+        this.$message.error('查询回收数据失败，请重试');
      }
    },
    handleCurrentChange(current) {