From 25afb4a7fa2278538ddcff646b6dab0b15ab0952 Mon Sep 17 00:00:00 2001 From: yaoq Date: Wed, 12 Oct 2022 15:43:52 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../system/canal/file/common/AbstractFileWork.java | 8 +- .../system/canal/file/impl/KbSqlFileWork.java | 12 +-- .../src/main/resources/config/application-dev.yaml | 111 ++++++++++++++++++++ .../main/resources/config/application-prod.yaml | 107 +++++++++++++++++++ .../src/main/resources/config/application.yaml | 113 +-------------------- 5 files changed, 233 insertions(+), 118 deletions(-) create mode 100644 smtweb-framework/canal/file/src/main/resources/config/application-dev.yaml create mode 100644 smtweb-framework/canal/file/src/main/resources/config/application-prod.yaml diff --git a/smtweb-framework/canal/file/src/main/java/cc/smtweb/system/canal/file/common/AbstractFileWork.java b/smtweb-framework/canal/file/src/main/java/cc/smtweb/system/canal/file/common/AbstractFileWork.java index 69bb24e..6104eb3 100644 --- a/smtweb-framework/canal/file/src/main/java/cc/smtweb/system/canal/file/common/AbstractFileWork.java +++ b/smtweb-framework/canal/file/src/main/java/cc/smtweb/system/canal/file/common/AbstractFileWork.java @@ -123,7 +123,7 @@ public abstract class AbstractFileWork { deleteSize++; } } - log.debug("[解析canal文件]:::file：" + file.getName() + " insert size：[" + insertSize + "] update size：[" + updateSize + "] delete size：[" + deleteSize + "] create size：[" + createSize + "] alter size：[" + alterSize + "]" + "drop size：[" + dropSize + "]"); + log.debug("[解析canal文件]:::file：" + file.getName() + " insert size：[" + insertSize + "] update size：[" + updateSize + "] delete size：[" + deleteSize + "] create size：[" + createSize + "] alter size：[" + alterSize + "]" + " drop size：[" + dropSize + "]"); } protected abstract String getDbType(); @@ -221,7 +221,11 @@ public abstract class AbstractFileWork { protected Map decodeCreateSql(String sql) { Map map = new HashMap<>(); String data = sql.substring(sql.indexOf("(") + 1, sql.lastIndexOf(")")).trim().replaceAll("`", "").replace("\t", ""); - String[] rows = data.split(",\\n");//data.split(",(?=(?:[^\']*\'[^\']*\')*[^\']*$)", -1); + + String[] rows = data.split(",\n");//data.split(",(?=(?:[^\']*\'[^\']*\')*[^\']*$)", -1); + if (rows.length == 1) { + rows = data.split(",\r\n"); + } for (String row : rows) { //索引不管 if (row.contains("INDEX")) continue; diff --git a/smtweb-framework/canal/file/src/main/java/cc/smtweb/system/canal/file/impl/KbSqlFileWork.java b/smtweb-framework/canal/file/src/main/java/cc/smtweb/system/canal/file/impl/KbSqlFileWork.java index a2ced51..13d2479 100644 --- a/smtweb-framework/canal/file/src/main/java/cc/smtweb/system/canal/file/impl/KbSqlFileWork.java +++ b/smtweb-framework/canal/file/src/main/java/cc/smtweb/system/canal/file/impl/KbSqlFileWork.java @@ -52,25 +52,25 @@ public class KbSqlFileWork extends AbstractFileWork { //添加sql if (sql.contains("ADD")) { sb.append("ALTER TABLE public.").append(canalVO.getTableName()); - sql = sql.substring(sql.indexOf("ADD COLUMN")); + sql = sql.substring(sql.indexOf("ADD")); String[] sqls = sql.split(",(?=(?:[^\']*\'[^\']*\')*[^\']*$)", -1); for (String str : sqls) { String[] ss = str.split(" "); - sb.append(" ADD COLUMN ").append(ss[2]).append(" ").append(convertFieldType(ss[3])).append(" NULL ").append(","); + sb.append(" ADD if not exists ").append(ss[1]).append(" ").append(ss[2]).append(" ").append(convertFieldType(ss[3])).append(" NULL ").append(","); } sb.deleteCharAt(sb.length() - 1); } else if (sql.contains("DROP")) { sb.append("ALTER TABLE public.").append(canalVO.getTableName()); - sql = sql.substring(sql.indexOf("DROP COLUMN")); + sql = sql.substring(sql.indexOf("DROP")); String[] sqls = sql.split(",(?=(?:[^\']*\'[^\']*\')*[^\']*$)", -1); for (String str : sqls) { String[] ss = str.split(" "); - sb.append(" DROP COLUMN ").append(ss[2]).append(","); + sb.append(" DROP COLUMN IF EXISTS ").append(ss[2]).append(","); } sb.deleteCharAt(sb.length() - 1); } else if (sql.contains("CHANGE")) { - sql = sql.substring(sql.indexOf("CHANGE COLUMN")); + sql = sql.substring(sql.indexOf("CHANGE")); String[] sqls = sql.split(",(?=(?:[^\']*\'[^\']*\')*[^\']*$)", -1); for (String str : sqls) { String[] ss = str.split(" "); @@ -79,7 +79,7 @@ public class KbSqlFileWork extends AbstractFileWork { sb.deleteCharAt(sb.length() - 1); } else if (sql.contains("MODIFY")) { sb.append("ALTER TABLE public.").append(canalVO.getTableName()); - sql = sql.substring(sql.indexOf("MODIFY COLUMN")); + sql = sql.substring(sql.indexOf("MODIFY")); String[] sqls = sql.split(",(?=(?:[^\']*\'[^\']*\')*[^\']*$)", -1); for (String str : sqls) { String[] ss = str.split(" "); diff --git a/smtweb-framework/canal/file/src/main/resources/config/application-dev.yaml b/smtweb-framework/canal/file/src/main/resources/config/application-dev.yaml new file mode 100644 index 0000000..1672bfe --- /dev/null +++ b/smtweb-framework/canal/file/src/main/resources/config/application-dev.yaml @@ -0,0 +1,111 @@ +smtweb: + machine-id: 1 + enable-job: true + bpm: + mode: 1 + debug: true + code-java-path: 'd:/work/smtweb2/smtweb-framework/canal/file' + db: + type: kbsql + default: + rule: + prefix: _smt_ + replace: smt_ + file: + local-path: + url: + # 服务模块 +devtools: + restart: + enabled: false # 热部署开关 + additional-paths: src/main/java #重启目录 +server: + port: 8081 + servlet: + context-path: +logging: + level: + root: INFO + cc.smtweb: DEBUG +spring: + redis: + host: 127.0.0.1 + port: 6379 + password: + database: 9 + datasource: + driver-class-name: com.kingbase8.Driver +# url: jdbc:kingbase8://10.176.246.198:54321/pgzx?useUnicode=true&characterEncoding=utf-8 +# username: 'zhengxie' +# password: '%TGB6yhn' + url: jdbc:kingbase8://172.28.123.205:54321/HLJTY?useUnicode=true&characterEncoding=utf-8 + username: system + password: system + servlet: + multipart: + max-file-size: 104857600000 + max-request-size: 10485760000000 + cache: + type: caffeine + cache-names: + - core + - bpm + caffeine: + spec: maximumSize=1024,expireAfterWrite=2h + +# canal配置 +canal: + file: + enable: true + #path: /data/canalFile + path: E:/canalFile + +# http 规则配置 +http-config: + xss: #xss 规则 + checkHeader: false #是否进行header校验 + checkParameter: true #是否进行parameter校验 + logIs: true #是否记录日志 + chain: true #是否中断请求 + replace: true #是否开启特殊字符替换 + checkUrl: true #是否开启特殊url校验 + regex: + # 匹配含有字符： alert( ) + - .*[A|a][L|l][E|e][R|r][T|t](.*).* + # 匹配含有字符： window.location + - .*[W|w][I|i][N|n][D|d][O|o][W|w].[L|l][O|o][C|c][A|a][T|t][I|i][O|o][N|n].* + # 匹配含有字符：style = x:ex pression ( ) + - .*[S|s][T|t][Y|y][L|l][E|e]\\s*=.*[X|x]:[E|e][X|x].*[P|p][R|r][E|e][S|s]{1,2}[I|i][O|o][N|n]\\s*\\(.*\\).* + # 匹配含有字符： document.cookie + - .*[D|d][O|o][C|c][U|u][M|m][E|e][N|n][T|t].[C|c][O|o]{2}[K|k][I|i][E|e].* + # 匹配含有字符： eval( ) + - .*[E|e][V|v][A|a][L|l](.*).* + # 匹配含有字符： unescape() + - .*[U|u][N|n][E|e][S|s][C|c][A|a][P|p][E|e](.*).* + # 匹配含有字符： execscript( ) + - .*[E|e][X|x][E|e][C|c][S|s][C|c][R|r][I|i][P|p][T|t](.*).* + # 匹配含有字符： msgbox( ) + - .*[M|m][S|s][G|g][B|b][O|o][X|x](.*).* + # 匹配含有字符： confirm( ) + - .*[C|c][O|o][N|n][F|f][I|i][R|r][M|m](.*).* + # 匹配含有字符： prompt( ) + - .*[P|p][R|r][O|o][M|m][P|p][T|t](.*).* + # 匹配含有字符： + - .*<[S|s][C|c][R|r][I|i][P|p][T|t]>.*.*.* + # 匹配含有字符： + - .*.* + # 匹配含有字符： + - .*<[S|s][C|c][R|r][I|i][P|p][T|t]>.*.*.* + # 匹配含有字符： + - .*.* + # 匹配含有字符： - - .*<[S|s][C|c][R|r][I|i][P|p][T|t]>.*.*.* - # 匹配含有字符： - - .*.* - # 匹配含有字符：