@@ -9,7 +9,7 @@ canal.server.port=11111 | |||
canal.server.username= | |||
canal.server.password= | |||
# 数据库匹配规则.*\\..*, scmz\\..* | |||
canal.server.filter=pgzx\\..* | |||
canal.server.filter=pgzx.*\\..* | |||
# 文件存放路径 E:/canalFile | |||
canal.file.path=/data/canalFile | |||
# mysql、dmsql、kbsql、oracle | |||
@@ -12,6 +12,7 @@ | |||
<groupId>cc.smtweb</groupId> | |||
<artifactId>canal.file</artifactId> | |||
<version>1.1.5</version> | |||
<packaging>war</packaging> | |||
<properties> | |||
<java.version>1.8</java.version> | |||
@@ -28,13 +29,19 @@ | |||
<scope>test</scope> | |||
</dependency> | |||
<!--快重启--> | |||
<dependency> | |||
<groupId>org.springframework.boot</groupId> | |||
<artifactId>spring-boot-devtools</artifactId> | |||
<optional>true</optional> <!-- 可选 --> | |||
<artifactId>spring-boot-starter-tomcat</artifactId> | |||
<scope>provided</scope> | |||
</dependency> | |||
<!--快重启--> | |||
<!--<dependency> | |||
<groupId>org.springframework.boot</groupId> | |||
<artifactId>spring-boot-devtools</artifactId> | |||
<optional>true</optional> <!– 可选 –> | |||
</dependency>--> | |||
<!--平台--> | |||
<dependency> | |||
@@ -2,12 +2,21 @@ package cc.smtweb.system.canal.file; | |||
import org.springframework.boot.SpringApplication; | |||
import org.springframework.boot.autoconfigure.SpringBootApplication; | |||
import org.springframework.boot.builder.SpringApplicationBuilder; | |||
import org.springframework.boot.web.servlet.ServletComponentScan; | |||
import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; | |||
@SpringBootApplication | |||
public class CanalFileApplication { | |||
@ServletComponentScan | |||
public class CanalFileApplication extends SpringBootServletInitializer { | |||
public static void main(String[] args) { | |||
SpringApplication.run(CanalFileApplication.class, args); | |||
} | |||
@Override | |||
protected SpringApplicationBuilder configure(SpringApplicationBuilder builder) { | |||
return builder.sources(CanalFileApplication.class); | |||
} | |||
public static void main(String[] args) { | |||
SpringApplication.run(CanalFileApplication.class, args); | |||
} | |||
} |
@@ -46,7 +46,7 @@ public abstract class AbstractFileWork { | |||
path = path + "/"; | |||
} | |||
path = path + "increment/"; | |||
log.debug("[解析canal文件] path:" + path); | |||
log.debug("[解析canal文件]:::path:" + path); | |||
File dir = new File(path); | |||
File[] fs = dir.listFiles(new FileFilter() { | |||
@Override | |||
@@ -57,7 +57,7 @@ public abstract class AbstractFileWork { | |||
if (fs == null) return; | |||
List<File> files = Arrays.asList(fs); | |||
Collections.sort(files, (o1, o2) -> StringUtil.chineseCompare(o1.getName(), o2.getName())); | |||
log.debug("[解析canal文件] size:" + files.size()); | |||
log.debug("[解析canal文件]:::size:" + files.size()); | |||
for (File f : files) { | |||
try { | |||
//为保证数据一致性,发现错误,不能继续,卡在这 | |||
@@ -75,7 +75,7 @@ public abstract class AbstractFileWork { | |||
FileUtil.copyFile(f, path + "/bak/" + DateUtil.getNowYm() + "/" + f.getName(), true); | |||
} catch (Exception e) { | |||
//出错了跳出,避免数据遗失 | |||
log.error("[解析canal文件] file:" + f.getName() + " 写入数据失败", e); | |||
log.error("[解析canal文件]:::file:" + f.getName() + " 写入数据失败", e); | |||
break; | |||
} | |||
} | |||
@@ -116,7 +116,7 @@ public abstract class AbstractFileWork { | |||
deleteSize++; | |||
} | |||
} | |||
log.debug("[解析canal文件] file:" + file.getName() + " insert size:[" + insertSize + "] update size:[" + updateSize + "] delete size:[" + deleteSize + "] create size:[" + createSize + "] alter size:[" + alterSize + "]"); | |||
log.debug("[解析canal文件]:::file:" + file.getName() + " insert size:[" + insertSize + "] update size:[" + updateSize + "] delete size:[" + deleteSize + "] create size:[" + createSize + "] alter size:[" + alterSize + "]"); | |||
} | |||
protected abstract String getDbType(); | |||
@@ -125,12 +125,20 @@ public abstract class AbstractFileWork { | |||
protected abstract void doAlterSql(CanalVO canalVO); | |||
protected String getSchemaName(CanalVO canalVO) { | |||
return ""; | |||
} | |||
protected void doInsertSql(CanalVO canalVO) { | |||
Map<String, String> data = canalVO.getData(); | |||
Set<String> fields = data.keySet(); | |||
StringBuilder sql = new StringBuilder(); | |||
List<Object> args = new ArrayList<>(); | |||
sql.append("insert into ").append(canalVO.getTableName()).append(" ( "); | |||
sql.append("insert into "); | |||
if (!StringUtil.isEmpty(getSchemaName(canalVO))) { | |||
sql.append(getSchemaName(canalVO)).append("."); | |||
} | |||
sql.append(canalVO.getTableName()).append(" ( "); | |||
for (String field : fields) { | |||
sql.append(field).append(","); | |||
} | |||
@@ -155,7 +163,11 @@ public abstract class AbstractFileWork { | |||
Set<String> fields = data.keySet(); | |||
StringBuilder sql = new StringBuilder(); | |||
List<Object> args = new ArrayList<>(); | |||
sql.append("update ").append(canalVO.getTableName()).append(" set "); | |||
sql.append("update "); | |||
if (!StringUtil.isEmpty(getSchemaName(canalVO))) { | |||
sql.append(getSchemaName(canalVO)).append("."); | |||
} | |||
sql.append(canalVO.getTableName()).append(" set "); | |||
for (String field : fields) { | |||
if (field.equals(canalVO.getIdField())) continue; | |||
sql.append(field).append("=").append("?").append(","); | |||
@@ -172,7 +184,11 @@ public abstract class AbstractFileWork { | |||
} | |||
protected void doDeleteSql(CanalVO canalVO) { | |||
getDbEngine().update("delete from " + canalVO.getTableName() + " where " + canalVO.getIdField() + " = ? ", canalVO.getId()); | |||
if (!StringUtil.isEmpty(getSchemaName(canalVO))) { | |||
getDbEngine().update("delete from " + getSchemaName(canalVO) + "." + canalVO.getTableName() + " where " + canalVO.getIdField() + " = ? ", canalVO.getId()); | |||
} else { | |||
getDbEngine().update("delete from " + canalVO.getTableName() + " where " + canalVO.getIdField() + " = ? ", canalVO.getId()); | |||
} | |||
} | |||
/** | |||
@@ -184,8 +200,8 @@ public abstract class AbstractFileWork { | |||
*/ | |||
protected Map<String, String> decodeCreateSql(String sql) { | |||
Map<String, String> map = new HashMap<>(); | |||
String data = sql.substring(sql.indexOf("(") + 1, sql.lastIndexOf(")")).trim().replaceAll("`", ""); | |||
String[] rows = data.split(","); | |||
String data = sql.substring(sql.indexOf("(") + 1, sql.lastIndexOf(")")).trim().replaceAll("`", "").replace("\t", ""); | |||
String[] rows = data.split(",(?=(?:[^\']*\'[^\']*\')*[^\']*$)", -1); | |||
for (String row : rows) { | |||
//索引不管 | |||
if (row.contains("INDEX")) continue; | |||
@@ -4,12 +4,15 @@ import cc.smtweb.framework.core.db.DbEngine; | |||
import cc.smtweb.framework.core.systask.BaseSysService; | |||
import lombok.extern.slf4j.Slf4j; | |||
import java.util.regex.Matcher; | |||
import java.util.regex.Pattern; | |||
@Slf4j | |||
public class FileDecodeService extends BaseSysService { | |||
@Override | |||
public String getTitle() { | |||
return "解析canal文件"; | |||
return "[解析canal文件]"; | |||
} | |||
@Override | |||
@@ -26,4 +29,13 @@ public class FileDecodeService extends BaseSysService { | |||
log.error("解析canal文件失败:", e); | |||
} | |||
} | |||
public static void main( String args[] ) { | |||
String sentence = "^pgzx.*\\..*"; | |||
Pattern pattern = Pattern.compile(sentence); | |||
Matcher matcher = pattern.matcher("pgzx.tb_sys_job"); | |||
Matcher matcher1 = pattern.matcher("pgzx_zx.tb_sys_job"); | |||
System.out.println(matcher.find()); | |||
System.out.println(matcher1.find()); | |||
} | |||
} |
@@ -17,7 +17,6 @@ public class DmSqlFileWork extends AbstractFileWork { | |||
return SwEnum.DbType.DMSQL.value; | |||
} | |||
@Override | |||
protected void doCreateSql(CanalVO canalVO) { | |||
throw new SwException("暂不支持新增表"); | |||
@@ -21,6 +21,11 @@ public class KbSqlFileWork extends AbstractFileWork { | |||
} | |||
@Override | |||
protected String getSchemaName(CanalVO canalVO) { | |||
return "public"; | |||
} | |||
@Override | |||
protected void doCreateSql(CanalVO canalVO) { | |||
Map<String, String> sqlMap = decodeCreateSql(canalVO.getSql()); | |||
StringBuilder sql = new StringBuilder(); | |||
@@ -33,6 +33,11 @@ public class MySqlFileWork extends AbstractFileWork { | |||
} | |||
@Override | |||
protected String getSchemaName(CanalVO canalVO) { | |||
return canalVO.getSchemaName(); | |||
} | |||
@Override | |||
protected void doInsertSql(CanalVO canalVO) { | |||
Map<String, String> data = canalVO.getData(); | |||
Set<String> fields = data.keySet(); | |||
@@ -17,10 +17,10 @@ smtweb: | |||
# 服务模块 | |||
devtools: | |||
restart: | |||
enabled: true # 热部署开关 | |||
enabled: false # 热部署开关 | |||
additional-paths: src/main/java #重启目录 | |||
server: | |||
port: 8888 | |||
port: 8088 | |||
servlet: | |||
context-path: | |||
logging: | |||
@@ -32,13 +32,12 @@ spring: | |||
host: 127.0.0.1 | |||
port: 6379 | |||
password: | |||
database: 9 | |||
datasource: | |||
driver-class-name: com.kingbase8.Driver | |||
# url: jdbc:mysql://139.9.38.43:6032/smt?useUnicode=true&characterEncoding=utf-8&allowMultiQueries=true&useSSL=false | |||
# url: jdbc:mysql://127.0.0.1:3306/tzrs?useUnicode=true&characterEncoding=utf-8&allowMultiQueries=true&useSSL=false | |||
url: jdbc:kingbase8://172.28.123.205:54321/HLJTY?useUnicode=true&characterEncoding=utf-8 | |||
username: system | |||
password: system | |||
url: jdbc:kingbase8://10.176.246.198:54321/pgzx?useUnicode=true&characterEncoding=utf-8 | |||
username: 'zhengxie' | |||
password: '%TGB6yhn' | |||
servlet: | |||
multipart: | |||
max-file-size: 104857600000 | |||
@@ -54,7 +53,8 @@ spring: | |||
# canal配置 | |||
canal: | |||
file: | |||
path: E:/canalFile | |||
enable: true | |||
path: /data/canalFile | |||
# http 规则配置 | |||
http-config: | |||
@@ -85,9 +85,9 @@ | |||
</archive> | |||
<excludes> | |||
<exclude>**/logback.xml</exclude> | |||
<exclude>**/canal.properties</exclude> | |||
<!--<exclude>**/canal.properties</exclude> | |||
<exclude>**/spring/**</exclude> | |||
<exclude>**/example/**</exclude> | |||
<exclude>**/example/**</exclude>--> | |||
<exclude>**/mq.yml</exclude> | |||
</excludes> | |||
</configuration> | |||
@@ -37,7 +37,7 @@ canal.instance.enableDruid=false | |||
#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ== | |||
# table regex | |||
canal.instance.filter.regex=pgzx\\..* | |||
canal.instance.filter.regex=pgzx.*\\..* | |||
# table black regex | |||
canal.instance.filter.black.regex=mysql\\.slave_.* | |||
# table field filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2) | |||