diff --git a/smtweb-framework/canal/client/pom.xml b/smtweb-framework/canal/client/pom.xml
new file mode 100644
index 0000000..09c9462
--- /dev/null
+++ b/smtweb-framework/canal/client/pom.xml
@@ -0,0 +1,210 @@
+
+ 4.0.0
+
+ org.springframework.boot
+ spring-boot-starter-parent
+ 2.6.9
+
+
+
+ cc.smtweb
+ canal.client
+ 1.1.6
+
+
+
+ com.alibaba.otter
+ canal.client
+ 1.1.6
+
+
+ com.alibaba.otter
+ canal.protocol
+ 1.1.6
+
+
+ com.alibaba
+ druid
+ 1.2.11
+
+
+
+ cc.smtweb
+ sw-system-bpm
+ 3.1.0-SNAPSHOT
+
+
+
+ mysql
+ mysql-connector-java
+
+
+ org.apache.ddlutils
+ ddlutils
+ 1.0
+
+
+ commons-beanutils
+ commons-beanutils-core
+
+
+ commons-lang
+ commons-lang
+
+
+ commons-dbcp
+ commons-dbcp
+
+
+ commons-pool
+ commons-pool
+
+
+ commons-logging
+ commons-logging-api
+
+
+ dom4j
+ dom4j
+
+
+ stax
+ stax-api
+
+
+ commons-collections
+ commons-collections
+
+
+ commons-digester
+ commons-digester
+
+
+ commons-betwixt
+ commons-betwixt
+
+
+
+
+ org.apache.commons
+ commons-pool2
+ 2.5.0
+
+
+ commons-beanutils
+ commons-beanutils
+ 1.8.2
+
+
+ org.apache.commons
+ commons-lang3
+ 3.7
+
+
+ commons-collections
+ commons-collections
+ 3.2
+
+
+
+
+ junit
+ junit
+ test
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 2.5
+
+
+ true
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-assembly-plugin
+
+ 2.2.1
+
+
+ assemble
+
+ single
+
+ package
+
+
+
+ false
+ false
+
+
+
+
+
+
+
+ dev
+
+ true
+
+ env
+ !release
+
+
+
+
+
+
+ maven-assembly-plugin
+
+
+
+ ${basedir}/src/main/assembly/dev.xml
+
+ canal-example
+ ${project.build.directory}
+
+
+
+
+
+
+
+
+ release
+
+
+ env
+ release
+
+
+
+
+
+
+ maven-assembly-plugin
+
+
+
+ ${basedir}/src/main/assembly/release.xml
+
+
+ ${project.artifactId}-1.1.6
+
+ ${project.parent.build.directory}
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientConsts.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientConsts.java
new file mode 100644
index 0000000..c238ae9
--- /dev/null
+++ b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientConsts.java
@@ -0,0 +1,76 @@
+package cc.smtweb.system.canal.client;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.util.Properties;
+
+/**
+ * 启动常用变量
+ *
+ * @author jianghang 2012-11-8 下午03:15:55
+ * @version 1.0.0
+ */
+public class ClientConsts {
+
+ public static final String ROOT = "canal.client";
+ public static final String SIZE = ROOT + "." + "size";
+ public static final String INSTANCE = ROOT + "." + "instance";
+ public static final String IP = ROOT + "." + "ip";
+ public static final String PORT = ROOT + "." + "port";
+ public static final String USERNAME = ROOT + "." + "username";
+ public static final String PASSWORD = ROOT + "." + "password";
+ public static final String FILTER = ROOT + "." + "filter";
+
+
+ public static String getIp(Properties properties){
+ return getProperty(properties,IP,"127.0.0.1");
+ }
+
+ public static int getPort(Properties properties){
+ return Integer.parseInt(getProperty(properties,PORT,"11111"));
+ }
+
+ public static int getBatchSize(Properties properties){
+ return Integer.parseInt(getProperty(properties,SIZE,"50"));
+ }
+
+ public static String getInstance(Properties properties){
+ return getProperty(properties,INSTANCE,"example");
+ }
+
+ public static String getUsername(Properties properties){
+ return getProperty(properties,USERNAME,"");
+ }
+
+ public static String getPassword(Properties properties){
+ return getProperty(properties,PASSWORD,"");
+ }
+
+ public static String getFilter(Properties properties){
+ return getProperty(properties,FILTER,".*\\..*");
+ }
+
+ public static String getProperty(Properties properties, String key, String defaultValue) {
+ String value = getProperty(properties, key);
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
+ } else {
+ return value;
+ }
+ }
+
+ public static String getProperty(Properties properties, String key) {
+ key = StringUtils.trim(key);
+ String value = System.getProperty(key);
+
+ if (value == null) {
+ value = System.getenv(key);
+ }
+
+ if (value == null) {
+ value = properties.getProperty(key);
+ }
+
+ return StringUtils.trim(value);
+ }
+}
diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientInstance.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientInstance.java
new file mode 100644
index 0000000..a4415d1
--- /dev/null
+++ b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientInstance.java
@@ -0,0 +1,290 @@
+package cc.smtweb.system.canal.client;
+
+import cc.smtweb.framework.core.util.CommUtil;
+import cc.smtweb.framework.core.util.JsonUtil;
+import com.alibaba.otter.canal.client.CanalConnector;
+import com.alibaba.otter.canal.client.CanalConnectors;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.Message;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+
+import java.io.FileInputStream;
+import java.net.InetSocketAddress;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * @Author yaoq
+ * @Date 2022年09月13日 18:02
+ * @Description
+ */
+public class ClientInstance {
+ private static final String CLASSPATH_URL_PREFIX = "classpath:";
+ private static final Logger logger = LoggerFactory.getLogger(ClientInstance.class);
+ protected volatile boolean running = false;
+ protected Thread thread = null;
+ protected CanalConnector connector;
+ protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
+
+ public void uncaughtException(Thread t, Throwable e) {
+ logger.error("解析出错了哦", e);
+
+ }
+ };
+
+ private static ClientInstance instance = null;
+ private static Properties properties = null;
+
+ static {
+ instance = new ClientInstance();
+ }
+
+ public static ClientInstance getInstance() {
+ return instance;
+ }
+
+ private void loadProperties() {
+ try {
+ String conf = System.getProperty("client.conf", "classpath:client.properties");
+ properties = new Properties();
+ if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
+ conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
+ properties.load(ClientInstance.class.getClassLoader().getResourceAsStream(conf));
+ } else {
+ properties.load(new FileInputStream(conf));
+ }
+ } catch (Exception e) {
+ logger.error("加载canal客户端配置参数失败!", e);
+ }
+ }
+
+ private ClientInstance() {
+ loadProperties();
+ connector = CanalConnectors.newSingleConnector(new
+ InetSocketAddress(ClientConsts.getIp(properties),
+ ClientConsts.getPort(properties)), ClientConsts.getInstance(properties), ClientConsts.getUsername(properties), ClientConsts.getPassword(properties));
+ connector.connect();
+ connector.subscribe(ClientConsts.getFilter(properties));
+ connector.rollback();
+
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ try {
+ logger.info("## stop the canal client");
+ ClientInstance.this.stop();
+ } catch (Throwable e) {
+ logger.warn("##something goes wrong when stopping canal:", e);
+ } finally {
+ logger.info("## canal client is down.");
+ }
+ }));
+ }
+
+ //开始
+ public void start() {
+ Assert.notNull(connector, "connector is null");
+ thread = new Thread(() -> process());
+
+ thread.setUncaughtExceptionHandler(handler);
+ running = true;
+ thread.start();
+ }
+
+ //停止
+ public void stop() {
+ if (!running) {
+ return;
+ }
+ running = false;
+ if (thread != null) {
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ // ignore
+ }
+ }
+ }
+
+ protected void process() {
+ while (running) {
+ try {
+ connector.disconnect();
+ connector.connect();
+ connector.unsubscribe();
+ connector.subscribe(ClientConsts.getFilter(properties));
+ while (running) {
+ Message message = connector.getWithoutAck(ClientConsts.getBatchSize(properties) * 1024); // 获取指定数量的数据
+ long batchId = message.getId();
+ int size = message.getEntries().size();
+ if (batchId == -1 || size == 0) {
+ Thread.sleep(3000L);//无数据时,睡眠。
+ continue;
+ } else {
+ dataHandle(message.getEntries());
+ }
+ if (batchId != -1) {
+ connector.ack(batchId); // 提交确认
+ }
+ }
+ } catch (Throwable e) {
+ logger.error("process error!", e);
+ try {
+ Thread.sleep(1000L);
+ } catch (InterruptedException e1) {
+ // ignore
+ }
+ if (!isReConnection()) {
+ connector.rollback(); // 处理失败, 回滚数据
+ }
+ } finally {
+ connector.disconnect();
+ }
+ }
+ }
+
+ private boolean isReConnection() {
+ if (!connector.checkValid()) return false;
+ int count = 1;
+ int rate = 1;
+ while (connector.checkValid()) {
+ rate = (count + 100) / 100;
+ try {
+ logger.debug("canal服务连接中断,第【" + count + "】次重连开始");
+ connector.disconnect();
+ connector.connect();
+ logger.debug("canal服务连接中断,第【" + count + "】次重连成功");
+ break;
+ } catch (Throwable e2) {
+ if (count > 99) {
+ logger.error("canal服务连接异常!", "重新连接canal服务端已失败【" + count + "】次,请检查canal服务是否启动,请及时启动canal服务端,或联系研发人员!");
+ }
+ logger.debug("canal服务连接中断,第【" + count + "】次重连失败:" + CommUtil.getOrigMsg(e2));
+ logger.debug("canal服务连接中断," + (3 * rate) + "分钟后进行第【" + (count + 1) + "】连接测试!");
+ try {
+ Thread.sleep(180 * rate * 1000L);
+ } catch (InterruptedException e1) {
+ // ignore
+ }
+ }
+ count++;
+ }
+ return true;
+ }
+
+ /**
+ * 数据处理
+ *
+ * @param entrys
+ */
+ private void dataHandle(List entrys) throws Exception {
+ for (CanalEntry.Entry entry : entrys) {
+ if (CanalEntry.EntryType.ROWDATA == entry.getEntryType()) {
+ CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
+ CanalEntry.EventType eventType = rowChange.getEventType();
+ if (eventType == CanalEntry.EventType.DELETE) {
+ saveDeleteSql(entry);
+ } else if (eventType == CanalEntry.EventType.UPDATE) {
+ saveUpdateSql(entry);
+ } else if (eventType == CanalEntry.EventType.INSERT) {
+ saveInsertSql(entry);
+ }
+ }
+ }
+ }
+
+ /**
+ * 保存更新语句
+ *
+ * @param entry
+ */
+ private void saveUpdateSql(CanalEntry.Entry entry) {
+ try {
+ CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
+ List rowDatasList = rowChange.getRowDatasList();
+ for (CanalEntry.RowData rowData : rowDatasList) {
+ List newColumnList = rowData.getAfterColumnsList();
+ Map data = new HashMap<>();
+ newColumnList.stream().forEach(item -> {
+ if (StringUtils.isNotEmpty(item.getValue())) {
+ data.put(lineToHump(item.getName()), item.getValue());
+ }
+ });
+ List oldColumnList = rowData.getBeforeColumnsList();
+ ClientVO update = ClientVO.ok(entry.getHeader().getTableName(), "UPDATE", data, oldColumnList.get(0).getValue());
+ System.out.println("更新返回 : " + JsonUtil.encodeString(update));
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * 保存删除语句
+ *
+ * @param entry
+ */
+ private void saveDeleteSql(CanalEntry.Entry entry) {
+ try {
+ CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
+ List rowDatasList = rowChange.getRowDatasList();
+ for (CanalEntry.RowData rowData : rowDatasList) {
+ List oldColumnList = rowData.getBeforeColumnsList();
+ ClientVO delete = ClientVO.ok(entry.getHeader().getTableName(), "DELETE", null, oldColumnList.get(0).getValue());
+ System.out.println("删除返回 : " + JsonUtil.encodeString(delete));
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * 保存插入语句
+ *
+ * @param entry
+ */
+ private void saveInsertSql(CanalEntry.Entry entry) {
+ try {
+ CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
+ List rowDatasList = rowChange.getRowDatasList();
+ for (CanalEntry.RowData rowData : rowDatasList) {
+ List columnList = rowData.getAfterColumnsList();
+ Map data = new HashMap<>();
+ columnList.stream().forEach(item -> {
+ if (StringUtils.isNotEmpty(item.getValue())) {
+ data.put(lineToHump(item.getName()), item.getValue());
+ }
+ });
+ ClientVO insert = ClientVO.ok(entry.getHeader().getTableName(), "INSERT", data, null);
+ System.out.println("插入返回 : " + JsonUtil.encodeString(insert));
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * 下划线转驼峰
+ *
+ * @param str
+ */
+ private static Pattern linePattern = Pattern.compile("_(\\w)");
+
+ public static String lineToHump(String str) {
+ str = str.toLowerCase();
+ Matcher matcher = linePattern.matcher(str);
+ StringBuffer sb = new StringBuffer();
+ while (matcher.find()) {
+ matcher.appendReplacement(sb, matcher.group(1).toUpperCase());
+ }
+ matcher.appendTail(sb);
+ return sb.toString();
+ }
+}
+
+
diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientStartedListener.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientStartedListener.java
new file mode 100644
index 0000000..5dad1df
--- /dev/null
+++ b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientStartedListener.java
@@ -0,0 +1,32 @@
+package cc.smtweb.system.canal.client;
+
+import cc.smtweb.framework.core.annotation.SwStartListener;
+import cc.smtweb.framework.core.common.SwConsts;
+import cc.smtweb.framework.core.mvc.controller.IStartListener;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * @Author yaoq
+ * @Date 2022年09月06日 10:31
+ * @Description
+ */
+@Slf4j
+@SwStartListener
+public class ClientStartedListener implements IStartListener {
+
+ @Override
+ public int order() {
+ return SwConsts.DEFAULT_ORDER + 3;
+ }
+
+
+ @Override
+ public void init() {
+
+ }
+
+ @Override
+ public void run() {
+ ClientInstance.getInstance().start();
+ }
+}
diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientVO.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientVO.java
new file mode 100644
index 0000000..02cbb46
--- /dev/null
+++ b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientVO.java
@@ -0,0 +1,56 @@
+package cc.smtweb.system.canal.client;
+
+import java.util.Map;
+
+/**
+ * @Author yaoq
+ * @Date 2022年09月13日 18:20
+ * @Description
+ */
+public class ClientVO {
+ private String tableName; // 表名
+ private String type; // 类型(更新、删除、插入)
+ private Map data; // 数据JSON 自己转对应表格实体类
+ private String id; // 更新或删除都是根据ID来
+
+ public static ClientVO ok(String tableName, String type, Map data, String id){
+ ClientVO canalVO=new ClientVO();
+ canalVO.setId(id);
+ canalVO.setTableName(tableName);
+ canalVO.setType(type);
+ canalVO.setData(data);
+ return canalVO;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public Map getData() {
+ return data;
+ }
+
+ public void setData(Map data) {
+ this.data = data;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+}
diff --git a/smtweb-framework/canal/client/src/main/resources/client.properties b/smtweb-framework/canal/client/src/main/resources/client.properties
new file mode 100644
index 0000000..6314051
--- /dev/null
+++ b/smtweb-framework/canal/client/src/main/resources/client.properties
@@ -0,0 +1,14 @@
+#canal ???????
+# ??????? ??MB
+canal.server.size=50
+# ??????
+canal.server.instance=example
+canal.server.ip=127.0.0.1
+canal.server.port=11111
+# ????????? ??????
+canal.server.username=
+canal.server.password=
+# ??????????.*\\..*??????scmz\\..*
+canal.server.filter=.*\\..*
+
+
diff --git a/smtweb-framework/canal/client/target/classes/client.properties b/smtweb-framework/canal/client/target/classes/client.properties
new file mode 100644
index 0000000..6314051
--- /dev/null
+++ b/smtweb-framework/canal/client/target/classes/client.properties
@@ -0,0 +1,14 @@
+#canal ???????
+# ??????? ??MB
+canal.server.size=50
+# ??????
+canal.server.instance=example
+canal.server.ip=127.0.0.1
+canal.server.port=11111
+# ????????? ??????
+canal.server.username=
+canal.server.password=
+# ??????????.*\\..*??????scmz\\..*
+canal.server.filter=.*\\..*
+
+
diff --git a/smtweb-framework/canal/server/pom.xml b/smtweb-framework/canal/server/pom.xml
new file mode 100644
index 0000000..89b0415
--- /dev/null
+++ b/smtweb-framework/canal/server/pom.xml
@@ -0,0 +1,227 @@
+
+ 4.0.0
+
+
+ org.springframework.boot
+ spring-boot-starter-parent
+ 2.6.9
+
+
+
+ cc.smtweb
+ canal.server
+ 1.1.6
+
+
+
+
+ com.alibaba.otter
+ canal.protocol
+ 1.1.6
+
+
+ com.alibaba.otter
+ canal.client
+ 1.1.6
+
+
+
+ com.alibaba.otter
+ canal.server
+ 1.1.6
+
+
+
+
+ com.alibaba.otter
+ canal.prometheus
+ 1.1.6
+ runtime
+
+
+
+ com.alibaba.otter
+ connector.kafka
+ 1.1.6
+
+
+ *
+ *
+
+
+ jar-with-dependencies
+ provided
+
+
+ com.alibaba.otter
+ connector.rocketmq
+ 1.1.6
+
+
+ *
+ *
+
+
+ jar-with-dependencies
+ provided
+
+
+ com.alibaba.otter
+ connector.rabbitmq
+ 1.1.6
+
+
+ *
+ *
+
+
+ jar-with-dependencies
+ provided
+
+
+ com.alibaba.otter
+ connector.pulsarmq
+ 1.1.6
+
+
+ *
+ *
+
+
+ jar-with-dependencies
+ provided
+
+
+ org.springframework.boot
+ spring-boot-autoconfigure
+
+
+ cc.smtweb
+ sw-framework-core
+ 3.1.0-SNAPSHOT
+ compile
+
+
+
+ cc.smtweb
+ sw-system-bpm
+ 3.1.0-SNAPSHOT
+ compile
+
+
+
+
+
+
+ maven-jar-plugin
+
+
+ true
+
+
+ **/logback.xml
+ **/canal.properties
+ **/spring/**
+ **/example/**
+ **/mq.yml
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+ 2.10
+
+
+ copy-dependencies-to-canal-deployer
+ package
+
+ copy-dependencies
+
+
+ jar-with-dependencies
+ ${project.basedir}/target/canal/plugin
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-assembly-plugin
+
+ 2.2.1
+
+
+ assemble
+
+ single
+
+ package
+
+
+
+ false
+ false
+
+
+
+
+
+
+
+ dev
+
+ true
+
+ env
+ !release
+
+
+
+
+
+
+ maven-assembly-plugin
+
+
+
+ ${basedir}/src/main/assembly/dev.xml
+
+ canal
+ ${project.build.directory}
+
+
+
+
+
+
+
+
+ release
+
+
+ env
+ release
+
+
+
+
+
+
+ maven-assembly-plugin
+
+
+
+ ${basedir}/src/main/assembly/release.xml
+
+
+ ${project.artifactId}-1.1.6
+
+ ${project.parent.build.directory}
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/server/src/main/assembly/dev.xml b/smtweb-framework/canal/server/src/main/assembly/dev.xml
new file mode 100644
index 0000000..3c66c4c
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/assembly/dev.xml
@@ -0,0 +1,54 @@
+
+ dist
+
+ dir
+
+ false
+
+
+ .
+ /
+
+ README*
+
+
+
+ ./src/main/bin
+ bin
+
+ **/*
+
+ 0755
+
+
+ ./src/main/conf
+ /conf
+
+ **/*
+
+
+
+ ./src/main/resources
+ /conf
+
+ **/*
+
+
+
+ target
+ logs
+
+ **/*
+
+
+
+
+
+ lib
+
+ junit:junit
+
+
+
+
diff --git a/smtweb-framework/canal/server/src/main/assembly/release.xml b/smtweb-framework/canal/server/src/main/assembly/release.xml
new file mode 100644
index 0000000..601328e
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/assembly/release.xml
@@ -0,0 +1,58 @@
+
+ dist
+
+ tar.gz
+
+ false
+
+
+ .
+ /
+
+ README*
+
+
+
+ ./src/main/bin
+ bin
+
+ **/*
+
+ 0755
+
+
+ ./src/main/conf
+ /conf
+
+ **/*
+
+
+
+ ./src/main/resources
+ /conf
+
+ **/*
+
+
+
+ target
+ logs
+
+ **/*
+
+
+
+ ${project.basedir}/target/canal/plugin
+ /plugin/
+
+
+
+
+ lib
+
+ junit:junit
+
+
+
+
diff --git a/smtweb-framework/canal/server/src/main/bin/restart.sh b/smtweb-framework/canal/server/src/main/bin/restart.sh
new file mode 100644
index 0000000..3623edd
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/bin/restart.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+args=$@
+
+case $(uname) in
+Linux)
+ bin_abs_path=$(readlink -f $(dirname $0))
+ ;;
+*)
+ bin_abs_path=$(cd $(dirname $0) ||exit ; pwd)
+ ;;
+esac
+
+sh "$bin_abs_path"/stop.sh $args
+sh "$bin_abs_path"/startup.sh $args
diff --git a/smtweb-framework/canal/server/src/main/bin/startup.bat b/smtweb-framework/canal/server/src/main/bin/startup.bat
new file mode 100644
index 0000000..0cad8cb
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/bin/startup.bat
@@ -0,0 +1,27 @@
+@echo off
+@if not "%ECHO%" == "" echo %ECHO%
+@if "%OS%" == "Windows_NT" setlocal
+
+set ENV_PATH=.\
+if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0%
+
+set conf_dir=%ENV_PATH%\..\conf
+set canal_conf=%conf_dir%\canal.properties
+@rem set canal_conf=%conf_dir%\canal_local.properties
+if "%1" == "local" set canal_conf=%conf_dir%\canal_local.properties
+set logback_configurationFile=%conf_dir%\logback.xml
+
+set CLASSPATH=%conf_dir%
+set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH%
+
+set JAVA_MEM_OPTS= -Xms128m -Xmx512m -XX:PermSize=128m
+set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8
+set JAVA_DEBUG_OPT= -server -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=9099,server=y,suspend=n
+set CANAL_OPTS= -DappName=otter-canal -Dlogback.configurationFile="%logback_configurationFile%" -Dcanal.conf="%canal_conf%"
+
+set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS%
+
+set CMD_STR= java %JAVA_OPTS% -classpath "%CLASSPATH%" java %JAVA_OPTS% -classpath "%CLASSPATH%" CanalLauncher
+echo start cmd : %CMD_STR%
+
+java %JAVA_OPTS% -classpath "%CLASSPATH%" CanalLauncher
diff --git a/smtweb-framework/canal/server/src/main/bin/startup.sh b/smtweb-framework/canal/server/src/main/bin/startup.sh
new file mode 100644
index 0000000..b8c0ed3
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/bin/startup.sh
@@ -0,0 +1,126 @@
+#!/bin/bash
+
+current_path=`pwd`
+case "`uname`" in
+ Linux)
+ bin_abs_path=$(readlink -f $(dirname $0))
+ ;;
+ *)
+ bin_abs_path=`cd $(dirname $0); pwd`
+ ;;
+esac
+base=${bin_abs_path}/..
+canal_conf=$base/conf/canal.properties
+canal_local_conf=$base/conf/canal_local.properties
+logback_configurationFile=$base/conf/logback.xml
+export LANG=en_US.UTF-8
+export BASE=$base
+
+if [ -f $base/bin/canal.pid ] ; then
+ echo "found canal.pid , Please run stop.sh first ,then startup.sh" 2>&2
+ exit 1
+fi
+
+if [ ! -d $base/logs/canal ] ; then
+ mkdir -p $base/logs/canal
+fi
+
+## set java path
+if [ -z "$JAVA" ] ; then
+ JAVA=$(which java)
+fi
+
+ALIBABA_JAVA="/usr/alibaba/java/bin/java"
+TAOBAO_JAVA="/opt/taobao/java/bin/java"
+if [ -z "$JAVA" ]; then
+ if [ -f $ALIBABA_JAVA ] ; then
+ JAVA=$ALIBABA_JAVA
+ elif [ -f $TAOBAO_JAVA ] ; then
+ JAVA=$TAOBAO_JAVA
+ else
+ echo "Cannot find a Java JDK. Please set either set JAVA or put java (>=1.5) in your PATH." 2>&2
+ exit 1
+ fi
+fi
+
+case "$#"
+in
+0 )
+ ;;
+1 )
+ var=$*
+ if [ "$var" = "local" ]; then
+ canal_conf=$canal_local_conf
+ else
+ if [ -f $var ] ; then
+ canal_conf=$var
+ else
+ echo "THE PARAMETER IS NOT CORRECT.PLEASE CHECK AGAIN."
+ exit
+ fi
+ fi;;
+2 )
+ var=$1
+ if [ "$var" = "local" ]; then
+ canal_conf=$canal_local_conf
+ else
+ if [ -f $var ] ; then
+ canal_conf=$var
+ else
+ if [ "$1" = "debug" ]; then
+ DEBUG_PORT=$2
+ DEBUG_SUSPEND="n"
+ JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND"
+ fi
+ fi
+ fi;;
+* )
+ echo "THE PARAMETERS MUST BE TWO OR LESS.PLEASE CHECK AGAIN."
+ exit;;
+esac
+
+JavaVersion=`$JAVA -version 2>&1 |awk 'NR==1{ gsub(/"/,""); print $3 }' | awk -F '.' '{print $1}'`
+str=`file -L $JAVA | grep 64-bit`
+JAVA_OPTS="$JAVA_OPTS -Xss256k -XX:+AggressiveOpts -XX:-UseBiasedLocking -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$base/logs"
+
+if [ $JavaVersion -ge 11 ] ; then
+ #JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$base_log/gc.log:time "
+ JAVA_OPTS="$JAVA_OPTS"
+else
+ #JAVA_OPTS="$JAVA_OPTS -Xloggc:$base/logs/canal/gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime"
+ JAVA_OPTS="$JAVA_OPTS -XX:+UseFastAccessorMethods -XX:+PrintAdaptiveSizePolicy -XX:+PrintTenuringDistribution"
+fi
+
+if [ -n "$str" ]; then
+ # JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
+ # For G1
+ JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
+else
+ JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m $JAVA_OPTS"
+fi
+
+JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8"
+CANAL_OPTS="-DappName=otter-canal -Dlogback.configurationFile=$logback_configurationFile -Dcanal.conf=$canal_conf"
+
+if [ -e $canal_conf -a -e $logback_configurationFile ]
+then
+
+ for i in $base/lib/*;
+ do CLASSPATH=$i:"$CLASSPATH";
+ done
+ CLASSPATH="$base/conf:$CLASSPATH";
+
+ echo "cd to $bin_abs_path for workaround relative path"
+ cd $bin_abs_path
+
+ echo LOG CONFIGURATION : $logback_configurationFile
+ echo canal conf : $canal_conf
+ echo CLASSPATH :$CLASSPATH
+ $JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.deployer.CanalLauncher 1>>$base/logs/canal/canal_stdout.log 2>&1 &
+ echo $! > $base/bin/canal.pid
+
+ echo "cd to $current_path for continue"
+ cd $current_path
+else
+ echo "canal conf("$canal_conf") OR log configration file($logback_configurationFile) is not exist,please create then first!"
+fi
diff --git a/smtweb-framework/canal/server/src/main/bin/stop.sh b/smtweb-framework/canal/server/src/main/bin/stop.sh
new file mode 100644
index 0000000..f398749
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/bin/stop.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+cygwin=false;
+linux=false;
+case "`uname`" in
+ CYGWIN*)
+ cygwin=true
+ ;;
+ Linux*)
+ linux=true
+ ;;
+esac
+
+get_pid() {
+ STR=$1
+ PID=$2
+ if $cygwin; then
+ JAVA_CMD="$JAVA_HOME\bin\java"
+ JAVA_CMD=`cygpath --path --unix $JAVA_CMD`
+ JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'`
+ else
+ if $linux; then
+ if [ ! -z "$PID" ]; then
+ JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
+ else
+ JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'`
+ fi
+ else
+ if [ ! -z "$PID" ]; then
+ JAVA_PID=`ps aux |grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
+ else
+ JAVA_PID=`ps aux |grep "$STR"|grep -v grep|awk '{print $2}'`
+ fi
+ fi
+ fi
+ echo $JAVA_PID;
+}
+
+base=`dirname $0`/..
+pidfile=$base/bin/canal.pid
+if [ ! -f "$pidfile" ];then
+ echo "canal is not running. exists"
+ exit
+fi
+
+pid=`cat $pidfile`
+if [ "$pid" == "" ] ; then
+ pid=`get_pid "appName=otter-canal"`
+fi
+
+echo -e "`hostname`: stopping canal $pid ... "
+kill $pid
+
+LOOPS=0
+while (true);
+do
+ gpid=`get_pid "appName=otter-canal" "$pid"`
+ if [ "$gpid" == "" ] ; then
+ echo "Oook! cost:$LOOPS"
+ `rm $pidfile`
+ break;
+ fi
+ let LOOPS=LOOPS+1
+ sleep 1
+done
\ No newline at end of file
diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalApplication.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalApplication.java
new file mode 100644
index 0000000..7a644ac
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalApplication.java
@@ -0,0 +1,13 @@
+package cc.smtweb.system.canal.server;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class CanalApplication {
+
+ public static void main(String[] args) {
+ SpringApplication.run(CanalApplication.class, args);
+ }
+
+}
diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalConstants.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalConstants.java
new file mode 100644
index 0000000..d9a6a48
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalConstants.java
@@ -0,0 +1,97 @@
+package cc.smtweb.system.canal.server;
+
+import java.text.MessageFormat;
+
+/**
+ * 启动常用变量
+ *
+ * @author jianghang 2012-11-8 下午03:15:55
+ * @version 1.0.0
+ */
+public class CanalConstants {
+
+ public static final String MDC_DESTINATION = "destination";
+ public static final String ROOT = "canal";
+ public static final String CANAL_ID = ROOT + "." + "id";
+ public static final String CANAL_IP = ROOT + "." + "ip";
+ public static final String CANAL_REGISTER_IP = ROOT + "." + "register.ip";
+ public static final String CANAL_PORT = ROOT + "." + "port";
+ public static final String CANAL_USER = ROOT + "." + "user";
+ public static final String CANAL_PASSWD = ROOT + "." + "passwd";
+ public static final String CANAL_METRICS_PULL_PORT = ROOT + "." + "metrics.pull.port";
+ public static final String CANAL_ADMIN_MANAGER = ROOT + "." + "admin.manager";
+ public static final String CANAL_ADMIN_PORT = ROOT + "." + "admin.port";
+ public static final String CANAL_ADMIN_USER = ROOT + "." + "admin.user";
+ public static final String CANAL_ADMIN_PASSWD = ROOT + "." + "admin.passwd";
+ public static final String CANAL_ADMIN_AUTO_REGISTER = ROOT + "." + "admin.register.auto";
+ public static final String CANAL_ADMIN_AUTO_CLUSTER = ROOT + "." + "admin.register.cluster";
+ public static final String CANAL_ADMIN_REGISTER_NAME = ROOT + "." + "admin.register.name";
+ public static final String CANAL_ZKSERVERS = ROOT + "." + "zkServers";
+ public static final String CANAL_WITHOUT_NETTY = ROOT + "." + "withoutNetty";
+
+ public static final String CANAL_DESTINATIONS = ROOT + "." + "destinations";
+ public static final String CANAL_AUTO_SCAN = ROOT + "." + "auto.scan";
+ public static final String CANAL_AUTO_SCAN_INTERVAL = ROOT + "." + "auto.scan.interval";
+ public static final String CANAL_CONF_DIR = ROOT + "." + "conf.dir";
+ public static final String CANAL_SERVER_MODE = ROOT + "." + "serverMode";
+
+ public static final String CANAL_DESTINATION_SPLIT = ",";
+ public static final String GLOBAL_NAME = "global";
+
+ public static final String INSTANCE_MODE_TEMPLATE = ROOT + "." + "instance.{0}.mode";
+ public static final String INSTANCE_LAZY_TEMPLATE = ROOT + "." + "instance.{0}.lazy";
+ public static final String INSTANCE_MANAGER_ADDRESS_TEMPLATE = ROOT + "." + "instance.{0}.manager.address";
+ public static final String INSTANCE_SPRING_XML_TEMPLATE = ROOT + "." + "instance.{0}.spring.xml";
+
+ public static final String CANAL_DESTINATION_PROPERTY = ROOT + ".instance.destination";
+
+ public static final String CANAL_SOCKETCHANNEL = ROOT + "." + "socketChannel";
+
+ public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey";
+ public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey";
+
+// public static final String CANAL_MQ_SERVERS = ROOT + "." + "mq.servers";
+// public static final String CANAL_MQ_RETRIES = ROOT + "." + "mq.retries";
+// public static final String CANAL_MQ_BATCHSIZE = ROOT + "." + "mq.batchSize";
+// public static final String CANAL_MQ_LINGERMS = ROOT + "." + "mq.lingerMs";
+// public static final String CANAL_MQ_MAXREQUESTSIZE = ROOT + "." + "mq.maxRequestSize";
+// public static final String CANAL_MQ_BUFFERMEMORY = ROOT + "." + "mq.bufferMemory";
+// public static final String CANAL_MQ_CANALBATCHSIZE = ROOT + "." + "mq.canalBatchSize";
+// public static final String CANAL_MQ_CANALGETTIMEOUT = ROOT + "." + "mq.canalGetTimeout";
+// public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage";
+// public static final String CANAL_MQ_PARALLELTHREADSIZE = ROOT + "." + "mq.parallelThreadSize";
+// public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType";
+// public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks";
+// public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction";
+// public static final String CANAL_MQ_PRODUCERGROUP = ROOT + "." + "mq.producerGroup";
+// public static final String CANAL_MQ_PROPERTIES = ROOT + "." + "mq.properties";
+// public static final String CANAL_MQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "mq.enableMessageTrace";
+// public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.accessChannel";
+// public static final String CANAL_MQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "mq.customizedTraceTopic";
+// public static final String CANAL_MQ_NAMESPACE = ROOT + "." + "mq.namespace";
+// public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable";
+// public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH = ROOT + "." + "mq.kafka.kerberos.krb5FilePath";
+// public static final String CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH = ROOT + "." + "mq.kafka.kerberos.jaasFilePath";
+// public static final String CANAL_MQ_USERNAME = ROOT + "." + "mq.username";
+// public static final String CANAL_MQ_PASSWORD = ROOT + "." + "mq.password";
+// public static final String CANAL_MQ_VHOST = ROOT + "." + "mq.vhost";
+// public static final String CANAL_MQ_ALIYUN_UID = ROOT + "." + "mq.aliyunuid";
+// public static final String CANAL_MQ_EXCHANGE = ROOT + "." + "mq.exchange";
+// public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash";
+
+ public static String getInstanceModeKey(String destination) {
+ return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination);
+ }
+
+ public static String getInstanceManagerAddressKey(String destination) {
+ return MessageFormat.format(INSTANCE_MANAGER_ADDRESS_TEMPLATE, destination);
+ }
+
+ public static String getInstancSpringXmlKey(String destination) {
+ return MessageFormat.format(INSTANCE_SPRING_XML_TEMPLATE, destination);
+ }
+
+ public static String getInstancLazyKey(String destination) {
+ return MessageFormat.format(INSTANCE_LAZY_TEMPLATE, destination);
+ }
+}
diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalController.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalController.java
new file mode 100644
index 0000000..75f2339
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalController.java
@@ -0,0 +1,606 @@
+package cc.smtweb.system.canal.server;
+
+import java.util.Map;
+import java.util.Properties;
+
+import cc.smtweb.system.canal.server.monitor.InstanceAction;
+import cc.smtweb.system.canal.server.monitor.InstanceConfigMonitor;
+import cc.smtweb.system.canal.server.monitor.ManagerInstanceConfigMonitor;
+import cc.smtweb.system.canal.server.monitor.SpringInstanceConfigMonitor;
+import org.I0Itec.zkclient.IZkStateListener;
+import org.I0Itec.zkclient.exception.ZkNoNodeException;
+import org.I0Itec.zkclient.exception.ZkNodeExistsException;
+import org.apache.commons.lang.BooleanUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zookeeper.Watcher.Event.KeeperState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+
+import com.alibaba.otter.canal.common.utils.AddressUtils;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningData;
+import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningListener;
+import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitor;
+import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitors;
+import cc.smtweb.system.canal.server.InstanceConfig.InstanceMode;
+import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
+import com.alibaba.otter.canal.instance.manager.PlainCanalInstanceGenerator;
+import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient;
+import com.alibaba.otter.canal.instance.spring.SpringCanalInstanceGenerator;
+import com.alibaba.otter.canal.server.CanalMQStarter;
+import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
+import com.alibaba.otter.canal.server.exception.CanalServerException;
+import com.alibaba.otter.canal.server.netty.CanalServerWithNetty;
+import com.google.common.base.Function;
+import com.google.common.collect.MapMaker;
+import com.google.common.collect.MigrateMap;
+
+/**
+ * canal调度控制器
+ *
+ * @author jianghang 2012-11-8 下午12:03:11
+ * @version 1.0.0
+ */
+public class CanalController {
+
+ private static final Logger logger = LoggerFactory.getLogger(CanalController.class);
+ private String ip;
+ private String registerIp;
+ private int port;
+ private int adminPort;
+ // 默认使用spring的方式载入
+ private Map instanceConfigs;
+ private InstanceConfig globalInstanceConfig;
+ private Map managerClients;
+ // 监听instance config的变化
+ private boolean autoScan = true;
+ private InstanceAction defaultAction;
+ private Map instanceConfigMonitors;
+ private CanalServerWithEmbedded embededCanalServer;
+ private CanalServerWithNetty canalServer;
+
+ private CanalInstanceGenerator instanceGenerator;
+ private ZkClientx zkclientx;
+
+ private CanalMQStarter canalMQStarter;
+ private String adminUser;
+ private String adminPasswd;
+
+ public CanalController(){
+ this(System.getProperties());
+ }
+
+ public CanalController(final Properties properties){
+ managerClients = MigrateMap.makeComputingMap(this::getManagerClient);
+
+ // 初始化全局参数设置
+ globalInstanceConfig = initGlobalConfig(properties);
+ instanceConfigs = new MapMaker().makeMap();
+ // 初始化instance config
+ initInstanceConfig(properties);
+
+ // init socketChannel
+ String socketChannel = getProperty(properties, CanalConstants.CANAL_SOCKETCHANNEL);
+ if (StringUtils.isNotEmpty(socketChannel)) {
+ System.setProperty(CanalConstants.CANAL_SOCKETCHANNEL, socketChannel);
+ }
+
+ // 兼容1.1.0版本的ak/sk参数名
+ String accesskey = getProperty(properties, "canal.instance.rds.accesskey");
+ String secretkey = getProperty(properties, "canal.instance.rds.secretkey");
+ if (StringUtils.isNotEmpty(accesskey)) {
+ System.setProperty(CanalConstants.CANAL_ALIYUN_ACCESSKEY, accesskey);
+ }
+ if (StringUtils.isNotEmpty(secretkey)) {
+ System.setProperty(CanalConstants.CANAL_ALIYUN_SECRETKEY, secretkey);
+ }
+
+ // 准备canal server
+ ip = getProperty(properties, CanalConstants.CANAL_IP);
+ registerIp = getProperty(properties, CanalConstants.CANAL_REGISTER_IP);
+ port = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_PORT, "11111"));
+ adminPort = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_ADMIN_PORT, "11110"));
+ embededCanalServer = CanalServerWithEmbedded.instance();
+ embededCanalServer.setCanalInstanceGenerator(instanceGenerator);// 设置自定义的instanceGenerator
+ int metricsPort = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_METRICS_PULL_PORT, "11112"));
+ embededCanalServer.setMetricsPort(metricsPort);
+
+ this.adminUser = getProperty(properties, CanalConstants.CANAL_ADMIN_USER);
+ this.adminPasswd = getProperty(properties, CanalConstants.CANAL_ADMIN_PASSWD);
+ embededCanalServer.setUser(getProperty(properties, CanalConstants.CANAL_USER));
+ embededCanalServer.setPasswd(getProperty(properties, CanalConstants.CANAL_PASSWD));
+
+ String canalWithoutNetty = getProperty(properties, CanalConstants.CANAL_WITHOUT_NETTY);
+ if (canalWithoutNetty == null || "false".equals(canalWithoutNetty)) {
+ canalServer = CanalServerWithNetty.instance();
+ canalServer.setIp(ip);
+ canalServer.setPort(port);
+ }
+
+ // 处理下ip为空,默认使用hostIp暴露到zk中
+ if (StringUtils.isEmpty(ip) && StringUtils.isEmpty(registerIp)) {
+ ip = registerIp = AddressUtils.getHostIp();
+ }
+
+ if (StringUtils.isEmpty(ip)) {
+ ip = AddressUtils.getHostIp();
+ }
+
+ if (StringUtils.isEmpty(registerIp)) {
+ registerIp = ip; // 兼容以前配置
+ }
+ final String zkServers = getProperty(properties, CanalConstants.CANAL_ZKSERVERS);
+ if (StringUtils.isNotEmpty(zkServers)) {
+ zkclientx = ZkClientx.getZkClient(zkServers);
+ // 初始化系统目录
+ zkclientx.createPersistent(ZookeeperPathUtils.DESTINATION_ROOT_NODE, true);
+ zkclientx.createPersistent(ZookeeperPathUtils.CANAL_CLUSTER_ROOT_NODE, true);
+ }
+
+ final ServerRunningData serverData = new ServerRunningData(registerIp + ":" + port);
+ ServerRunningMonitors.setServerData(serverData);
+ ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap((Function) destination -> {
+ ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
+ runningMonitor.setDestination(destination);
+ runningMonitor.setListener(new ServerRunningListener() {
+
+ public void processActiveEnter() {
+ try {
+ MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
+ embededCanalServer.start(destination);
+ if (canalMQStarter != null) {
+ canalMQStarter.startDestination(destination);
+ }
+ } finally {
+ MDC.remove(CanalConstants.MDC_DESTINATION);
+ }
+ }
+
+ public void processActiveExit() {
+ try {
+ MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
+ if (canalMQStarter != null) {
+ canalMQStarter.stopDestination(destination);
+ }
+ embededCanalServer.stop(destination);
+ } finally {
+ MDC.remove(CanalConstants.MDC_DESTINATION);
+ }
+ }
+
+ public void processStart() {
+ try {
+ if (zkclientx != null) {
+ final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
+ registerIp + ":" + port);
+ initCid(path);
+ zkclientx.subscribeStateChanges(new IZkStateListener() {
+
+ public void handleStateChanged(KeeperState state) throws Exception {
+
+ }
+
+ public void handleNewSession() throws Exception {
+ initCid(path);
+ }
+
+ @Override
+ public void handleSessionEstablishmentError(Throwable error) throws Exception {
+ logger.error("failed to connect to zookeeper", error);
+ }
+ });
+ }
+ } finally {
+ MDC.remove(CanalConstants.MDC_DESTINATION);
+ }
+ }
+
+ public void processStop() {
+ try {
+ MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
+ if (zkclientx != null) {
+ final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
+ registerIp + ":" + port);
+ releaseCid(path);
+ }
+ } finally {
+ MDC.remove(CanalConstants.MDC_DESTINATION);
+ }
+ }
+
+ });
+ if (zkclientx != null) {
+ runningMonitor.setZkClient(zkclientx);
+ }
+ // 触发创建一下cid节点
+ runningMonitor.init();
+ return runningMonitor;
+ }));
+
+ // 初始化monitor机制
+ autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN));
+ if (autoScan) {
+ defaultAction = new InstanceAction() {
+
+ public void start(String destination) {
+ InstanceConfig config = instanceConfigs.get(destination);
+ if (config == null) {
+ // 重新读取一下instance config
+ config = parseInstanceConfig(properties, destination);
+ instanceConfigs.put(destination, config);
+ }
+
+ if (!embededCanalServer.isStart(destination)) {
+ // HA机制启动
+ ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
+ if (!config.getLazy() && !runningMonitor.isStart()) {
+ runningMonitor.start();
+ }
+ }
+
+ logger.info("auto notify start {} successful.", destination);
+ }
+
+ public void stop(String destination) {
+ // 此处的stop,代表强制退出,非HA机制,所以需要退出HA的monitor和配置信息
+ InstanceConfig config = instanceConfigs.remove(destination);
+ if (config != null) {
+ embededCanalServer.stop(destination);
+ ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
+ if (runningMonitor.isStart()) {
+ runningMonitor.stop();
+ }
+ }
+
+ logger.info("auto notify stop {} successful.", destination);
+ }
+
+ public void reload(String destination) {
+ // 目前任何配置变化,直接重启,简单处理
+ stop(destination);
+ start(destination);
+
+ logger.info("auto notify reload {} successful.", destination);
+ }
+
+ @Override
+ public void release(String destination) {
+ // 此处的release,代表强制释放,主要针对HA机制释放运行,让给其他机器抢占
+ InstanceConfig config = instanceConfigs.get(destination);
+ if (config != null) {
+ ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
+ if (runningMonitor.isStart()) {
+ boolean release = runningMonitor.release();
+ if (!release) {
+ // 如果是单机模式,则直接清除配置
+ instanceConfigs.remove(destination);
+ // 停掉服务
+ runningMonitor.stop();
+ if (instanceConfigMonitors.containsKey(InstanceConfig.InstanceMode.MANAGER)) {
+ ManagerInstanceConfigMonitor monitor = (ManagerInstanceConfigMonitor) instanceConfigMonitors.get(InstanceConfig.InstanceMode.MANAGER);
+ Map instanceActions = monitor.getActions();
+ if (instanceActions.containsKey(destination)) {
+ // 清除内存中的autoScan cache
+ monitor.release(destination);
+ }
+ }
+ }
+ }
+ }
+
+ logger.info("auto notify release {} successful.", destination);
+ }
+ };
+
+ instanceConfigMonitors = MigrateMap.makeComputingMap(mode -> {
+ int scanInterval = Integer.valueOf(getProperty(properties,
+ CanalConstants.CANAL_AUTO_SCAN_INTERVAL,
+ "5"));
+
+ if (mode.isSpring()) {
+ SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor();
+ monitor.setScanIntervalInSecond(scanInterval);
+ monitor.setDefaultAction(defaultAction);
+ // 设置conf目录,默认是user.dir + conf目录组成
+ String rootDir = getProperty(properties, CanalConstants.CANAL_CONF_DIR);
+ if (StringUtils.isEmpty(rootDir)) {
+ rootDir = "../conf";
+ }
+
+ if (StringUtils.equals("otter-canal", System.getProperty("appName"))) {
+ monitor.setRootConf(rootDir);
+ } else {
+ // eclipse debug模式
+ monitor.setRootConf("src/main/resources/");
+ }
+ return monitor;
+ } else if (mode.isManager()) {
+ ManagerInstanceConfigMonitor monitor = new ManagerInstanceConfigMonitor();
+ monitor.setScanIntervalInSecond(scanInterval);
+ monitor.setDefaultAction(defaultAction);
+ String managerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER);
+ monitor.setConfigClient(getManagerClient(managerAddress));
+ return monitor;
+ } else {
+ throw new UnsupportedOperationException("unknow mode :" + mode + " for monitor");
+ }
+ });
+ }
+ }
+
+ private InstanceConfig initGlobalConfig(Properties properties) {
+ String adminManagerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER);
+ InstanceConfig globalConfig = new InstanceConfig();
+ String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(CanalConstants.GLOBAL_NAME));
+ if (StringUtils.isNotEmpty(adminManagerAddress)) {
+ // 如果指定了manager地址,则强制适用manager
+ globalConfig.setMode(InstanceMode.MANAGER);
+ } else if (StringUtils.isNotEmpty(modeStr)) {
+ globalConfig.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr)));
+ }
+
+ String lazyStr = getProperty(properties, CanalConstants.getInstancLazyKey(CanalConstants.GLOBAL_NAME));
+ if (StringUtils.isNotEmpty(lazyStr)) {
+ globalConfig.setLazy(Boolean.valueOf(lazyStr));
+ }
+
+ String managerAddress = getProperty(properties,
+ CanalConstants.getInstanceManagerAddressKey(CanalConstants.GLOBAL_NAME));
+ if (StringUtils.isNotEmpty(managerAddress)) {
+ if (StringUtils.equals(managerAddress, "${canal.admin.manager}")) {
+ managerAddress = adminManagerAddress;
+ }
+
+ globalConfig.setManagerAddress(managerAddress);
+ }
+
+ String springXml = getProperty(properties, CanalConstants.getInstancSpringXmlKey(CanalConstants.GLOBAL_NAME));
+ if (StringUtils.isNotEmpty(springXml)) {
+ globalConfig.setSpringXml(springXml);
+ }
+
+ instanceGenerator = destination -> {
+ InstanceConfig config = instanceConfigs.get(destination);
+ if (config == null) {
+ throw new CanalServerException("can't find destination:" + destination);
+ }
+
+ if (config.getMode().isManager()) {
+ PlainCanalInstanceGenerator instanceGenerator = new PlainCanalInstanceGenerator(properties);
+ instanceGenerator.setCanalConfigClient(managerClients.get(config.getManagerAddress()));
+ instanceGenerator.setSpringXml(config.getSpringXml());
+ return instanceGenerator.generate(destination);
+ } else if (config.getMode().isSpring()) {
+ SpringCanalInstanceGenerator instanceGenerator = new SpringCanalInstanceGenerator();
+ instanceGenerator.setSpringXml(config.getSpringXml());
+ return instanceGenerator.generate(destination);
+ } else {
+ throw new UnsupportedOperationException("unknow mode :" + config.getMode());
+ }
+
+ };
+
+ return globalConfig;
+ }
+
+ private PlainCanalConfigClient getManagerClient(String managerAddress) {
+ return new PlainCanalConfigClient(managerAddress, this.adminUser, this.adminPasswd, this.registerIp, adminPort);
+ }
+
+ private void initInstanceConfig(Properties properties) {
+ String destinationStr = getProperty(properties, CanalConstants.CANAL_DESTINATIONS);
+ String[] destinations = StringUtils.split(destinationStr, CanalConstants.CANAL_DESTINATION_SPLIT);
+
+ for (String destination : destinations) {
+ InstanceConfig config = parseInstanceConfig(properties, destination);
+ InstanceConfig oldConfig = instanceConfigs.put(destination, config);
+
+ if (oldConfig != null) {
+ logger.warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config);
+ }
+ }
+ }
+
+ private InstanceConfig parseInstanceConfig(Properties properties, String destination) {
+ String adminManagerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER);
+ InstanceConfig config = new InstanceConfig(globalInstanceConfig);
+ String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(destination));
+ if (StringUtils.isNotEmpty(adminManagerAddress)) {
+ // 如果指定了manager地址,则强制适用manager
+ config.setMode(InstanceMode.MANAGER);
+ } else if (StringUtils.isNotEmpty(modeStr)) {
+ config.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr)));
+ }
+
+ String lazyStr = getProperty(properties, CanalConstants.getInstancLazyKey(destination));
+ if (!StringUtils.isEmpty(lazyStr)) {
+ config.setLazy(Boolean.valueOf(lazyStr));
+ }
+
+ if (config.getMode().isManager()) {
+ String managerAddress = getProperty(properties, CanalConstants.getInstanceManagerAddressKey(destination));
+ if (StringUtils.isNotEmpty(managerAddress)) {
+ if (StringUtils.equals(managerAddress, "${canal.admin.manager}")) {
+ managerAddress = adminManagerAddress;
+ }
+ config.setManagerAddress(managerAddress);
+ }
+ }
+
+ String springXml = getProperty(properties, CanalConstants.getInstancSpringXmlKey(destination));
+ if (StringUtils.isNotEmpty(springXml)) {
+ config.setSpringXml(springXml);
+ }
+
+ return config;
+ }
+
+ public static String getProperty(Properties properties, String key, String defaultValue) {
+ String value = getProperty(properties, key);
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
+ } else {
+ return value;
+ }
+ }
+
+ public static String getProperty(Properties properties, String key) {
+ key = StringUtils.trim(key);
+ String value = System.getProperty(key);
+
+ if (value == null) {
+ value = System.getenv(key);
+ }
+
+ if (value == null) {
+ value = properties.getProperty(key);
+ }
+
+ return StringUtils.trim(value);
+ }
+
+ public void start() throws Throwable {
+ logger.info("## start the canal server[{}({}):{}]", ip, registerIp, port);
+ // 创建整个canal的工作节点
+ final String path = ZookeeperPathUtils.getCanalClusterNode(registerIp + ":" + port);
+ initCid(path);
+ if (zkclientx != null) {
+ this.zkclientx.subscribeStateChanges(new IZkStateListener() {
+
+ public void handleStateChanged(KeeperState state) throws Exception {
+
+ }
+
+ public void handleNewSession() throws Exception {
+ initCid(path);
+ }
+
+ @Override
+ public void handleSessionEstablishmentError(Throwable error) throws Exception {
+ logger.error("failed to connect to zookeeper", error);
+ }
+ });
+ }
+ // 优先启动embeded服务
+ embededCanalServer.start();
+ // 尝试启动一下非lazy状态的通道
+ for (Map.Entry entry : instanceConfigs.entrySet()) {
+ final String destination = entry.getKey();
+ InstanceConfig config = entry.getValue();
+ // 创建destination的工作节点
+ if (!embededCanalServer.isStart(destination)) {
+ // HA机制启动
+ ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
+ if (!config.getLazy() && !runningMonitor.isStart()) {
+ runningMonitor.start();
+ }
+ }
+
+ if (autoScan) {
+ instanceConfigMonitors.get(config.getMode()).register(destination, defaultAction);
+ }
+ }
+
+ if (autoScan) {
+ instanceConfigMonitors.get(globalInstanceConfig.getMode()).start();
+ for (InstanceConfigMonitor monitor : instanceConfigMonitors.values()) {
+ if (!monitor.isStart()) {
+ monitor.start();
+ }
+ }
+ }
+
+ // 启动网络接口
+ if (canalServer != null) {
+ canalServer.start();
+ }
+ }
+
+ public void stop() throws Throwable {
+
+ if (canalServer != null) {
+ canalServer.stop();
+ }
+
+ if (autoScan) {
+ for (InstanceConfigMonitor monitor : instanceConfigMonitors.values()) {
+ if (monitor.isStart()) {
+ monitor.stop();
+ }
+ }
+ }
+
+ for (ServerRunningMonitor runningMonitor : ServerRunningMonitors.getRunningMonitors().values()) {
+ if (runningMonitor.isStart()) {
+ runningMonitor.stop();
+ }
+ }
+
+ // 释放canal的工作节点
+ releaseCid(ZookeeperPathUtils.getCanalClusterNode(registerIp + ":" + port));
+ logger.info("## stop the canal server[{}({}):{}]", ip, registerIp, port);
+
+ if (zkclientx != null) {
+ zkclientx.close();
+ }
+
+ // 关闭时清理缓存
+ if (instanceConfigs != null) {
+ instanceConfigs.clear();
+ }
+ if (managerClients != null) {
+ managerClients.clear();
+ }
+ if (instanceConfigMonitors != null) {
+ instanceConfigMonitors.clear();
+ }
+
+ ZkClientx.clearClients();
+ }
+
+ private void initCid(String path) {
+ // logger.info("## init the canalId = {}", cid);
+ // 初始化系统目录
+ if (zkclientx != null) {
+ try {
+ zkclientx.createEphemeral(path);
+ } catch (ZkNoNodeException e) {
+ // 如果父目录不存在,则创建
+ String parentDir = path.substring(0, path.lastIndexOf('/'));
+ zkclientx.createPersistent(parentDir, true);
+ zkclientx.createEphemeral(path);
+ } catch (ZkNodeExistsException e) {
+ // ignore
+ // 因为第一次启动时创建了cid,但在stop/start的时可能会关闭和新建,允许出现NodeExists问题s
+ }
+
+ }
+ }
+
+ private void releaseCid(String path) {
+ // logger.info("## release the canalId = {}", cid);
+ // 初始化系统目录
+ if (zkclientx != null) {
+ zkclientx.delete(path);
+ }
+ }
+
+ public CanalMQStarter getCanalMQStarter() {
+ return canalMQStarter;
+ }
+
+ public void setCanalMQStarter(CanalMQStarter canalMQStarter) {
+ this.canalMQStarter = canalMQStarter;
+ }
+
+ public Map getInstanceConfigMonitors() {
+ return instanceConfigMonitors;
+ }
+
+ public Map getInstanceConfigs() {
+ return instanceConfigs;
+ }
+
+}
diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalLauncher.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalLauncher.java
new file mode 100644
index 0000000..e84b6f0
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalLauncher.java
@@ -0,0 +1,141 @@
+package cc.smtweb.system.canal.server;
+
+import com.alibaba.otter.canal.common.utils.AddressUtils;
+import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
+import com.alibaba.otter.canal.instance.manager.plain.PlainCanal;
+import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient;
+import org.apache.commons.lang.BooleanUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileInputStream;
+import java.util.Properties;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * canal独立版本启动的入口类
+ *
+ * @author jianghang 2012-11-6 下午05:20:49
+ * @version 1.0.0
+ */
+public class CanalLauncher {
+
+ private static final String CLASSPATH_URL_PREFIX = "classpath:";
+ private static final Logger logger = LoggerFactory.getLogger(CanalLauncher.class);
+ public static final CountDownLatch runningLatch = new CountDownLatch(1);
+ private static ScheduledExecutorService executor = Executors.newScheduledThreadPool(1,
+ new NamedThreadFactory("canal-server-scan"));
+
+ public static void main(String[] args) {
+ startServer();
+ }
+
+ public static void startServer() {
+ try {
+ logger.info("## canal sever start begin");
+ logger.info("## set default uncaught exception handler");
+ setGlobalUncaughtExceptionHandler();
+
+ // 支持rocketmq client 配置日志路径
+ System.setProperty("rocketmq.client.logUseSlf4j", "true");
+
+ logger.info("## load canal configurations");
+ String conf = System.getProperty("canal.conf", "classpath:canal.properties");
+ Properties properties = new Properties();
+ if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
+ conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
+ properties.load(CanalLauncher.class.getClassLoader().getResourceAsStream(conf));
+ } else {
+ properties.load(new FileInputStream(conf));
+ }
+
+ final CanalStarter canalStater = new CanalStarter(properties);
+ String managerAddress = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER);
+ if (StringUtils.isNotEmpty(managerAddress)) {
+ String user = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_USER);
+ String passwd = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PASSWD);
+ String adminPort = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PORT, "11110");
+ boolean autoRegister = BooleanUtils.toBoolean(CanalController.getProperty(properties,
+ CanalConstants.CANAL_ADMIN_AUTO_REGISTER));
+ String autoCluster = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_AUTO_CLUSTER);
+ String name = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_REGISTER_NAME);
+ if (StringUtils.isEmpty(name)) {
+ name = AddressUtils.getHostName();
+ }
+
+ String registerIp = CanalController.getProperty(properties, CanalConstants.CANAL_REGISTER_IP);
+ if (StringUtils.isEmpty(registerIp)) {
+ registerIp = AddressUtils.getHostIp();
+ }
+ final PlainCanalConfigClient configClient = new PlainCanalConfigClient(managerAddress,
+ user,
+ passwd,
+ registerIp,
+ Integer.parseInt(adminPort),
+ autoRegister,
+ autoCluster,
+ name);
+ PlainCanal canalConfig = configClient.findServer(null);
+ if (canalConfig == null) {
+ throw new IllegalArgumentException("managerAddress:" + managerAddress
+ + " can't not found config for [" + registerIp + ":" + adminPort
+ + "]");
+ }
+ Properties managerProperties = canalConfig.getProperties();
+ // merge local
+ managerProperties.putAll(properties);
+ int scanIntervalInSecond = Integer.valueOf(CanalController.getProperty(managerProperties,
+ CanalConstants.CANAL_AUTO_SCAN_INTERVAL,
+ "5"));
+ executor.scheduleWithFixedDelay(new Runnable() {
+
+ private PlainCanal lastCanalConfig;
+
+ public void run() {
+ try {
+ if (lastCanalConfig == null) {
+ lastCanalConfig = configClient.findServer(null);
+ } else {
+ PlainCanal newCanalConfig = configClient.findServer(lastCanalConfig.getMd5());
+ if (newCanalConfig != null) {
+ // 远程配置canal.properties修改重新加载整个应用
+ canalStater.stop();
+ Properties managerProperties = newCanalConfig.getProperties();
+ // merge local
+ managerProperties.putAll(properties);
+ canalStater.setProperties(managerProperties);
+ canalStater.start();
+
+ lastCanalConfig = newCanalConfig;
+ }
+ }
+
+ } catch (Throwable e) {
+ logger.error("scan failed", e);
+ }
+ }
+
+ }, 0, scanIntervalInSecond, TimeUnit.SECONDS);
+ canalStater.setProperties(managerProperties);
+ } else {
+ canalStater.setProperties(properties);
+ }
+
+ canalStater.start();
+ logger.info("## canal server start success");
+ runningLatch.await();
+ executor.shutdownNow();
+ } catch (Throwable e) {
+ logger.error("## Something goes wrong when starting up the canal Server:", e);
+ }
+ }
+
+ private static void setGlobalUncaughtExceptionHandler() {
+ Thread.setDefaultUncaughtExceptionHandler((t, e) -> logger.error("UnCaughtException", e));
+ }
+
+}
diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStartedListener.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStartedListener.java
new file mode 100644
index 0000000..4530945
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStartedListener.java
@@ -0,0 +1,39 @@
+package cc.smtweb.system.canal.server;
+
+import cc.smtweb.framework.core.annotation.SwStartListener;
+import cc.smtweb.framework.core.common.SwConsts;
+import cc.smtweb.framework.core.mvc.controller.IStartListener;
+import cc.smtweb.framework.core.systask.SysThreadPool;
+import cc.smtweb.framework.core.systask.SysThreadWorker;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * @Author yaoq
+ * @Date 2022年09月06日 10:31
+ * @Description
+ */
+@Slf4j
+@SwStartListener
+public class CanalStartedListener implements IStartListener {
+
+ @Override
+ public int order() {
+ return SwConsts.DEFAULT_ORDER + 2;
+ }
+
+
+ @Override
+ public void init() {
+
+ }
+
+ @Override
+ public void run() {
+ SysThreadPool.getInstance().addTask(new SysThreadWorker("canal server") {
+ @Override
+ public void localWork() throws Exception {
+ CanalLauncher.startServer();
+ }
+ });
+ }
+}
diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStarter.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStarter.java
new file mode 100644
index 0000000..58b6621
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStarter.java
@@ -0,0 +1,169 @@
+package cc.smtweb.system.canal.server;
+
+import java.util.Properties;
+
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.admin.netty.CanalAdminWithNetty;
+import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
+import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
+import cc.smtweb.system.canal.server.admin.CanalAdminController;
+import com.alibaba.otter.canal.server.CanalMQStarter;
+
+/**
+ * Canal server 启动类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.2
+ */
+public class CanalStarter {
+
+ private static final Logger logger = LoggerFactory.getLogger(CanalStarter.class);
+
+ private static final String CONNECTOR_SPI_DIR = "/plugin";
+ private static final String CONNECTOR_STANDBY_SPI_DIR = "/canal/plugin";
+
+ private CanalController controller = null;
+ private CanalMQProducer canalMQProducer = null;
+ private Thread shutdownThread = null;
+ private CanalMQStarter canalMQStarter = null;
+ private volatile Properties properties;
+ private volatile boolean running = false;
+
+ private CanalAdminWithNetty canalAdmin;
+
+ public CanalStarter(Properties properties){
+ this.properties = properties;
+ }
+
+ public boolean isRunning() {
+ return running;
+ }
+
+ public Properties getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Properties properties) {
+ this.properties = properties;
+ }
+
+ public CanalController getController() {
+ return controller;
+ }
+
+ /**
+ * 启动方法
+ *
+ * @throws Throwable
+ */
+ public synchronized void start() throws Throwable {
+ String serverMode = CanalController.getProperty(properties, CanalConstants.CANAL_SERVER_MODE);
+ if (!"tcp".equalsIgnoreCase(serverMode)) {
+ ExtensionLoader loader = ExtensionLoader.getExtensionLoader(CanalMQProducer.class);
+ canalMQProducer = loader
+ .getExtension(serverMode.toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR);
+ if (canalMQProducer != null) {
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(canalMQProducer.getClass().getClassLoader());
+ canalMQProducer.init(properties);
+ Thread.currentThread().setContextClassLoader(cl);
+ }
+ }
+
+ if (canalMQProducer != null) {
+ MQProperties mqProperties = canalMQProducer.getMqProperties();
+ // disable netty
+ System.setProperty(CanalConstants.CANAL_WITHOUT_NETTY, "true");
+ if (mqProperties.isFlatMessage()) {
+ // 设置为raw避免ByteString->Entry的二次解析
+ System.setProperty("canal.instance.memory.rawEntry", "false");
+ }
+ }
+
+ logger.info("## start the canal server.");
+ controller = new CanalController(properties);
+ controller.start();
+ logger.info("## the canal server is running now ......");
+ shutdownThread = new Thread(() -> {
+ try {
+ logger.info("## stop the canal server");
+ controller.stop();
+ CanalLauncher.runningLatch.countDown();
+ } catch (Throwable e) {
+ logger.warn("##something goes wrong when stopping canal Server:", e);
+ } finally {
+ logger.info("## canal server is down.");
+ }
+ });
+ Runtime.getRuntime().addShutdownHook(shutdownThread);
+
+ if (canalMQProducer != null) {
+ canalMQStarter = new CanalMQStarter(canalMQProducer);
+ String destinations = CanalController.getProperty(properties, CanalConstants.CANAL_DESTINATIONS);
+ canalMQStarter.start(destinations);
+ controller.setCanalMQStarter(canalMQStarter);
+ }
+
+ // start canalAdmin
+ String port = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PORT);
+ if (canalAdmin == null && StringUtils.isNotEmpty(port)) {
+ String user = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_USER);
+ String passwd = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PASSWD);
+ CanalAdminController canalAdmin = new CanalAdminController(this);
+ canalAdmin.setUser(user);
+ canalAdmin.setPasswd(passwd);
+
+ String ip = CanalController.getProperty(properties, CanalConstants.CANAL_IP);
+
+ logger.debug("canal admin port:{}, canal admin user:{}, canal admin password: {}, canal ip:{}",
+ port,
+ user,
+ passwd,
+ ip);
+
+ CanalAdminWithNetty canalAdminWithNetty = CanalAdminWithNetty.instance();
+ canalAdminWithNetty.setCanalAdmin(canalAdmin);
+ canalAdminWithNetty.setPort(Integer.parseInt(port));
+ canalAdminWithNetty.setIp(ip);
+ canalAdminWithNetty.start();
+ this.canalAdmin = canalAdminWithNetty;
+ }
+
+ running = true;
+ }
+
+ public synchronized void stop() throws Throwable {
+ stop(false);
+ }
+
+ /**
+ * 销毁方法,远程配置变更时调用
+ *
+ * @throws Throwable
+ */
+ public synchronized void stop(boolean stopByAdmin) throws Throwable {
+ if (!stopByAdmin && canalAdmin != null) {
+ canalAdmin.stop();
+ canalAdmin = null;
+ }
+
+ if (controller != null) {
+ controller.stop();
+ controller = null;
+ }
+ if (shutdownThread != null) {
+ Runtime.getRuntime().removeShutdownHook(shutdownThread);
+ shutdownThread = null;
+ }
+ if (canalMQProducer != null && canalMQStarter != null) {
+ canalMQStarter.destroy();
+ canalMQStarter = null;
+ canalMQProducer = null;
+ }
+ running = false;
+ }
+}
diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/InstanceConfig.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/InstanceConfig.java
new file mode 100644
index 0000000..6775ca9
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/InstanceConfig.java
@@ -0,0 +1,93 @@
+package cc.smtweb.system.canal.server;
+
+import org.apache.commons.lang.builder.ToStringBuilder;
+
+import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
+
+/**
+ * 启动的相关配置
+ *
+ * @author jianghang 2012-11-8 下午02:50:54
+ * @version 1.0.0
+ */
+public class InstanceConfig {
+
+ private InstanceConfig globalConfig;
+ private InstanceMode mode;
+ private Boolean lazy;
+ private String managerAddress;
+ private String springXml;
+
+ public InstanceConfig(){
+
+ }
+
+ public InstanceConfig(InstanceConfig globalConfig){
+ this.globalConfig = globalConfig;
+ }
+
+ public static enum InstanceMode {
+ SPRING, MANAGER;
+
+ public boolean isSpring() {
+ return this == InstanceMode.SPRING;
+ }
+
+ public boolean isManager() {
+ return this == InstanceMode.MANAGER;
+ }
+ }
+
+ public Boolean getLazy() {
+ if (lazy == null && globalConfig != null) {
+ return globalConfig.getLazy();
+ } else {
+ return lazy;
+ }
+ }
+
+ public void setLazy(Boolean lazy) {
+ this.lazy = lazy;
+ }
+
+ public InstanceMode getMode() {
+ if (mode == null && globalConfig != null) {
+ return globalConfig.getMode();
+ } else {
+ return mode;
+ }
+ }
+
+ public void setMode(InstanceMode mode) {
+ this.mode = mode;
+ }
+
+ public String getManagerAddress() {
+ if (managerAddress == null && globalConfig != null) {
+ return globalConfig.getManagerAddress();
+ } else {
+ return managerAddress;
+ }
+ }
+
+ public void setManagerAddress(String managerAddress) {
+ this.managerAddress = managerAddress;
+ }
+
+ public String getSpringXml() {
+ if (springXml == null && globalConfig != null) {
+ return globalConfig.getSpringXml();
+ } else {
+ return springXml;
+ }
+ }
+
+ public void setSpringXml(String springXml) {
+ this.springXml = springXml;
+ }
+
+ public String toString() {
+ return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
+ }
+
+}
diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/admin/CanalAdminController.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/admin/CanalAdminController.java
new file mode 100644
index 0000000..d504621
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/admin/CanalAdminController.java
@@ -0,0 +1,256 @@
+package cc.smtweb.system.canal.server.admin;
+
+import java.io.File;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import cc.smtweb.system.canal.server.CanalStarter;
+import cc.smtweb.system.canal.server.monitor.InstanceConfigMonitor;
+import cc.smtweb.system.canal.server.monitor.ManagerInstanceConfigMonitor;
+import cc.smtweb.system.canal.server.monitor.SpringInstanceConfigMonitor;
+import cc.smtweb.system.canal.server.monitor.InstanceAction;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.admin.CanalAdmin;
+import com.alibaba.otter.canal.common.utils.FileUtils;
+import cc.smtweb.system.canal.server.InstanceConfig;
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.protocol.SecurityUtil;
+import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
+import com.google.common.base.Joiner;
+
+/**
+ * 提供canal admin的管理操作
+ *
+ * @author agapple 2019年8月24日 下午11:39:01
+ * @since 1.1.4
+ */
+public class CanalAdminController implements CanalAdmin {
+
+ private static final Logger logger = LoggerFactory.getLogger(CanalAdminController.class);
+ private String user;
+ private String passwd;
+ private CanalStarter canalStater;
+
+ public CanalAdminController(CanalStarter canalStater){
+ this.canalStater = canalStater;
+ }
+
+ @Override
+ public boolean check() {
+ return canalStater.isRunning();
+ }
+
+ @Override
+ public synchronized boolean start() {
+ try {
+ if (!canalStater.isRunning()) {
+ canalStater.start();
+ return true;
+ }
+ } catch (Throwable e) {
+ logger.error(e.getMessage(), e);
+ }
+ return false;
+ }
+
+ @Override
+ public synchronized boolean stop() {
+ try {
+ if (canalStater.isRunning()) {
+ canalStater.stop(true);
+ return true;
+ }
+ } catch (Throwable e) {
+ logger.error(e.getMessage(), e);
+ }
+ return false;
+ }
+
+ @Override
+ public synchronized boolean restart() {
+ stop();
+ return start();
+ }
+
+ @Override
+ public boolean auth(String user, String passwd, byte[] seed) {
+ // 如果user/passwd密码为空,则任何用户账户都能登录
+ if ((StringUtils.isEmpty(this.user) || StringUtils.equals(this.user, user))) {
+ if (StringUtils.isEmpty(this.passwd)) {
+ return true;
+ } else if (StringUtils.isEmpty(passwd)) {
+ // 如果server密码有配置,客户端密码为空,则拒绝
+ return false;
+ }
+
+ try {
+ byte[] passForClient = SecurityUtil.hexStr2Bytes(passwd);
+ return SecurityUtil.scrambleServerAuth(passForClient, SecurityUtil.hexStr2Bytes(this.passwd), seed);
+ } catch (NoSuchAlgorithmException e) {
+ return false;
+ }
+ }
+
+ return false;
+ }
+
+ @Override
+ public String getRunningInstances() {
+ try {
+ Map instances = CanalServerWithEmbedded.instance().getCanalInstances();
+ List runningInstances = new ArrayList<>();
+ instances.forEach((destination, instance) -> {
+ if (instance.isStart()) {
+ runningInstances.add(destination);
+ }
+ });
+
+ return Joiner.on(",").join(runningInstances);
+ } catch (Throwable e) {
+ logger.error(e.getMessage(), e);
+ }
+ return "";
+ }
+
+ @Override
+ public boolean checkInstance(String destination) {
+ Map instances = CanalServerWithEmbedded.instance().getCanalInstances();
+ if (instances == null || !instances.containsKey(destination)) {
+ return false;
+ } else {
+ CanalInstance instance = instances.get(destination);
+ return instance.isStart();
+ }
+ }
+
+ @Override
+ public boolean startInstance(String destination) {
+ try {
+ InstanceAction instanceAction = getInstanceAction(destination);
+ if (instanceAction != null) {
+ instanceAction.start(destination);
+ return true;
+ }
+ } catch (Throwable e) {
+ logger.error(e.getMessage(), e);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean stopInstance(String destination) {
+ try {
+ InstanceAction instanceAction = getInstanceAction(destination);
+ if (instanceAction != null) {
+ instanceAction.stop(destination);
+ return true;
+ }
+ } catch (Throwable e) {
+ logger.error(e.getMessage(), e);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean releaseInstance(String destination) {
+ try {
+ InstanceAction instanceAction = getInstanceAction(destination);
+ if (instanceAction != null) {
+ instanceAction.release(destination);
+ return true;
+ }
+ } catch (Throwable e) {
+ logger.error(e.getMessage(), e);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean restartInstance(String destination) {
+ try {
+ InstanceAction instanceAction = getInstanceAction(destination);
+ if (instanceAction != null) {
+ instanceAction.reload(destination);
+ return true;
+ }
+ } catch (Throwable e) {
+ logger.error(e.getMessage(), e);
+ }
+ return false;
+ }
+
+ @Override
+ public String listCanalLog() {
+ Collection files = org.apache.commons.io.FileUtils.listFiles(new File("../logs/canal/"),
+ TrueFileFilter.TRUE,
+ TrueFileFilter.TRUE);
+ List names = files.stream().map(File::getName).collect(Collectors.toList());
+ return Joiner.on(",").join(names);
+ }
+
+ @Override
+ public String canalLog(int lines) {
+ return FileUtils.readFileFromOffset("../logs/canal/canal.log", lines, "UTF-8");
+ }
+
+ @Override
+ public String listInstanceLog(String destination) {
+ Collection files = org.apache.commons.io.FileUtils.listFiles(new File("../logs/" + destination + "/"),
+ TrueFileFilter.TRUE,
+ TrueFileFilter.TRUE);
+ List names = files.stream().map(File::getName).collect(Collectors.toList());
+ return Joiner.on(",").join(names);
+ }
+
+ @Override
+ public String instanceLog(String destination, String fileName, int lines) {
+ if (StringUtils.isEmpty(fileName)) {
+ fileName = destination + ".log";
+ }
+ return FileUtils.readFileFromOffset("../logs/" + destination + "/" + fileName, lines, "UTF-8");
+ }
+
+ private InstanceAction getInstanceAction(String destination) {
+ Map monitors = canalStater.getController()
+ .getInstanceConfigMonitors();
+
+ InstanceAction instanceAction = null;
+ if (monitors.containsKey(InstanceConfig.InstanceMode.SPRING)) {
+ SpringInstanceConfigMonitor monitor = (SpringInstanceConfigMonitor) monitors.get(InstanceConfig.InstanceMode.SPRING);
+ Map instanceActions = monitor.getActions();
+ instanceAction = instanceActions.get(destination);
+ }
+
+ if (instanceAction != null) {
+ return instanceAction;
+ }
+
+ if (monitors.containsKey(InstanceConfig.InstanceMode.MANAGER)) {
+ ManagerInstanceConfigMonitor monitor = (ManagerInstanceConfigMonitor) monitors.get(InstanceConfig.InstanceMode.MANAGER);
+ Map instanceActions = monitor.getActions();
+ instanceAction = instanceActions.get(destination);
+ }
+ return instanceAction;
+ }
+
+ public void setUser(String user) {
+ this.user = user;
+ }
+
+ public void setPasswd(String passwd) {
+ this.passwd = passwd;
+ }
+
+ public void setCanalStater(CanalStarter canalStater) {
+ this.canalStater = canalStater;
+ }
+
+}
diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceAction.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceAction.java
new file mode 100644
index 0000000..f1ed700
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceAction.java
@@ -0,0 +1,30 @@
+package cc.smtweb.system.canal.server.monitor;
+
+/**
+ * config配置变化
+ *
+ * @author jianghang 2013-2-18 下午01:19:29
+ * @version 1.0.1
+ */
+public interface InstanceAction {
+
+ /**
+ * 启动destination
+ */
+ void start(String destination);
+
+ /**
+ * 主动释放destination运行
+ */
+ void release(String destination);
+
+ /**
+ * 停止destination
+ */
+ void stop(String destination);
+
+ /**
+ * 重载destination,可能需要stop,start操作,或者只是更新下内存配置
+ */
+ void reload(String destination);
+}
diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceConfigMonitor.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceConfigMonitor.java
new file mode 100644
index 0000000..702a2b9
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceConfigMonitor.java
@@ -0,0 +1,16 @@
+package cc.smtweb.system.canal.server.monitor;
+
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+
+/**
+ * 监听instance file的文件变化,触发instance start/stop等操作
+ *
+ * @author jianghang 2013-2-6 下午06:19:56
+ * @version 1.0.1
+ */
+public interface InstanceConfigMonitor extends CanalLifeCycle {
+
+ void register(String destination, InstanceAction action);
+
+ void unregister(String destination);
+}
diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/ManagerInstanceConfigMonitor.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/ManagerInstanceConfigMonitor.java
new file mode 100644
index 0000000..d0aa043
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/ManagerInstanceConfigMonitor.java
@@ -0,0 +1,184 @@
+package cc.smtweb.system.canal.server.monitor;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
+import com.alibaba.otter.canal.instance.manager.plain.PlainCanal;
+import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient;
+import com.google.common.collect.Lists;
+import com.google.common.collect.MapMaker;
+import com.google.common.collect.MigrateMap;
+
+/**
+ * 基于manager配置的实现
+ *
+ * @author agapple 2019年8月26日 下午10:00:20
+ * @since 1.1.4
+ */
+public class ManagerInstanceConfigMonitor extends AbstractCanalLifeCycle implements InstanceConfigMonitor, CanalLifeCycle {
+
+ private static final Logger logger = LoggerFactory.getLogger(ManagerInstanceConfigMonitor.class);
+ private long scanIntervalInSecond = 5;
+ private InstanceAction defaultAction = null;
+ private Map actions = new MapMaker().makeMap();
+ private Map configs = MigrateMap.makeComputingMap(destination -> new PlainCanal());
+ private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1,
+ new NamedThreadFactory("canal-instance-scan"));
+
+ private volatile boolean isFirst = true;
+ private PlainCanalConfigClient configClient;
+
+ public void start() {
+ super.start();
+ executor.scheduleWithFixedDelay(() -> {
+ try {
+ scan();
+ if (isFirst) {
+ isFirst = false;
+ }
+ } catch (Throwable e) {
+ logger.error("scan failed", e);
+ }
+ }, 0, scanIntervalInSecond, TimeUnit.SECONDS);
+ }
+
+ public void stop() {
+ super.stop();
+ executor.shutdownNow();
+ actions.clear();
+ }
+
+ public void register(String destination, InstanceAction action) {
+ if (action != null) {
+ actions.put(destination, action);
+ } else {
+ actions.put(destination, defaultAction);
+ }
+ }
+
+ public void unregister(String destination) {
+ actions.remove(destination);
+ }
+
+ private void scan() {
+ String instances = configClient.findInstances(null);
+ if (instances == null) {
+ return;
+ }
+
+ final List is = Lists.newArrayList(StringUtils.split(instances, ','));
+ List start = new ArrayList<>();
+ List stop = new ArrayList<>();
+ List restart = new ArrayList<>();
+ for (String instance : is) {
+ if (!configs.containsKey(instance)) {
+ PlainCanal newPlainCanal = configClient.findInstance(instance, null);
+ if (newPlainCanal != null) {
+ configs.put(instance, newPlainCanal);
+ start.add(instance);
+ }
+ } else {
+ PlainCanal plainCanal = configs.get(instance);
+ PlainCanal newPlainCanal = configClient.findInstance(instance, plainCanal.getMd5());
+ if (newPlainCanal != null) {
+ // 配置有变化
+ restart.add(instance);
+ configs.put(instance, newPlainCanal);
+ }
+ }
+ }
+
+ configs.forEach((instance, plainCanal) -> {
+ if (!is.contains(instance)) {
+ stop.add(instance);
+ }
+ });
+
+ stop.forEach(instance -> {
+ notifyStop(instance);
+ });
+
+ restart.forEach(instance -> {
+ notifyReload(instance);
+ });
+
+ start.forEach(instance -> {
+ notifyStart(instance);
+ });
+
+ }
+
+ private void notifyStart(String destination) {
+ try {
+ defaultAction.start(destination);
+ actions.put(destination, defaultAction);
+ // 启动成功后记录配置文件信息
+ } catch (Throwable e) {
+ logger.error(String.format("scan add found[%s] but start failed", destination), e);
+ }
+ }
+
+ private void notifyStop(String destination) {
+ InstanceAction action = actions.remove(destination);
+ if (action != null) {
+ try {
+ action.stop(destination);
+ configs.remove(destination);
+ } catch (Throwable e) {
+ logger.error(String.format("scan delete found[%s] but stop failed", destination), e);
+ actions.put(destination, action);// 再重新加回去,下一次scan时再执行删除
+ }
+ }
+ }
+
+ private void notifyReload(String destination) {
+ InstanceAction action = actions.get(destination);
+ if (action != null) {
+ try {
+ action.reload(destination);
+ } catch (Throwable e) {
+ logger.error(String.format("scan reload found[%s] but reload failed", destination), e);
+ }
+ }
+ }
+
+ public void release(String destination) {
+ InstanceAction action = actions.remove(destination);
+ if (action != null) {
+ try {
+ configs.remove(destination);
+ } catch (Throwable e) {
+ logger.error(String.format("scan delete found[%s] but stop failed", destination), e);
+ actions.put(destination, action);// 再重新加回去,下一次scan时再执行删除
+ }
+ }
+ }
+
+ public void setDefaultAction(InstanceAction defaultAction) {
+ this.defaultAction = defaultAction;
+ }
+
+ public void setScanIntervalInSecond(long scanIntervalInSecond) {
+ this.scanIntervalInSecond = scanIntervalInSecond;
+ }
+
+ public void setConfigClient(PlainCanalConfigClient configClient) {
+ this.configClient = configClient;
+ }
+
+ public Map getActions() {
+ return actions;
+ }
+
+}
diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/SpringInstanceConfigMonitor.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/SpringInstanceConfigMonitor.java
new file mode 100644
index 0000000..83a2b13
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/SpringInstanceConfigMonitor.java
@@ -0,0 +1,297 @@
+package cc.smtweb.system.canal.server.monitor;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
+import com.google.common.collect.MapMaker;
+import com.google.common.collect.MigrateMap;
+
+/**
+ * 监听基于spring配置的instance变化
+ *
+ * @author jianghang 2013-2-6 下午06:23:55
+ * @version 1.0.1
+ */
+public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implements InstanceConfigMonitor, CanalLifeCycle {
+
+ private static final Logger logger = LoggerFactory.getLogger(SpringInstanceConfigMonitor.class);
+ private String rootConf;
+ // 扫描周期,单位秒
+ private long scanIntervalInSecond = 5;
+ private InstanceAction defaultAction = null;
+ private Map actions = new MapMaker().makeMap();
+ private Map lastFiles = MigrateMap.makeComputingMap(InstanceConfigFiles::new);
+ private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1,
+ new NamedThreadFactory("canal-instance-scan"));
+
+ private volatile boolean isFirst = true;
+
+ public Map getActions() {
+ return actions;
+ }
+
+ public void start() {
+ super.start();
+ Assert.notNull(rootConf, "root conf dir is null!");
+
+ executor.scheduleWithFixedDelay(() -> {
+ try {
+ scan();
+ if (isFirst) {
+ isFirst = false;
+ }
+ } catch (Throwable e) {
+ logger.error("scan failed", e);
+ }
+ }, 0, scanIntervalInSecond, TimeUnit.SECONDS);
+ }
+
+ public void stop() {
+ super.stop();
+ executor.shutdownNow();
+ actions.clear();
+ lastFiles.clear();
+ }
+
+ public void register(String destination, InstanceAction action) {
+ if (action != null) {
+ actions.put(destination, action);
+ } else {
+ actions.put(destination, defaultAction);
+ }
+ }
+
+ public void unregister(String destination) {
+ actions.remove(destination);
+ }
+
+ public void setRootConf(String rootConf) {
+ this.rootConf = rootConf;
+ }
+
+ private void scan() {
+ File rootdir = new File(rootConf);
+ if (!rootdir.exists()) {
+ return;
+ }
+
+ File[] instanceDirs = rootdir.listFiles(pathname -> {
+ String filename = pathname.getName();
+ return pathname.isDirectory() && !"spring".equalsIgnoreCase(filename);
+ });
+
+ // 扫描目录的新增
+ Set currentInstanceNames = new HashSet<>();
+
+ // 判断目录内文件的变化
+ for (File instanceDir : instanceDirs) {
+ String destination = instanceDir.getName();
+ currentInstanceNames.add(destination);
+ File[] instanceConfigs = instanceDir.listFiles((dir, name) -> {
+ // return !StringUtils.endsWithIgnoreCase(name, ".dat");
+ // 限制一下,只针对instance.properties文件,避免因为.svn或者其他生成的临时文件导致出现reload
+ return StringUtils.equalsIgnoreCase(name, "instance.properties");
+ });
+
+ if (!actions.containsKey(destination) && instanceConfigs.length > 0) {
+ // 存在合法的instance.properties,并且第一次添加时,进行启动操作
+ notifyStart(instanceDir, destination, instanceConfigs);
+ } else if (actions.containsKey(destination)) {
+ // 历史已经启动过
+ if (instanceConfigs.length == 0) { // 如果不存在合法的instance.properties
+ notifyStop(destination);
+ } else {
+ InstanceConfigFiles lastFile = lastFiles.get(destination);
+ // 历史启动过 所以配置文件信息必然存在
+ if (!isFirst && CollectionUtils.isEmpty(lastFile.getInstanceFiles())) {
+ logger.error("[{}] is started, but not found instance file info.", destination);
+ }
+
+ boolean hasChanged = judgeFileChanged(instanceConfigs, lastFile.getInstanceFiles());
+ // 通知变化
+ if (hasChanged) {
+ notifyReload(destination);
+ }
+
+ if (hasChanged || CollectionUtils.isEmpty(lastFile.getInstanceFiles())) {
+ // 更新内容
+ List newFileInfo = new ArrayList<>();
+ for (File instanceConfig : instanceConfigs) {
+ newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified()));
+ }
+
+ lastFile.setInstanceFiles(newFileInfo);
+ }
+ }
+ }
+
+ }
+
+ // 判断目录是否删除
+ Set deleteInstanceNames = new HashSet<>();
+ for (String destination : actions.keySet()) {
+ if (!currentInstanceNames.contains(destination)) {
+ deleteInstanceNames.add(destination);
+ }
+ }
+ for (String deleteInstanceName : deleteInstanceNames) {
+ notifyStop(deleteInstanceName);
+ }
+ }
+
+ private void notifyStart(File instanceDir, String destination, File[] instanceConfigs) {
+ try {
+ defaultAction.start(destination);
+ actions.put(destination, defaultAction);
+
+ // 启动成功后记录配置文件信息
+ InstanceConfigFiles lastFile = lastFiles.get(destination);
+ List newFileInfo = new ArrayList<>();
+ for (File instanceConfig : instanceConfigs) {
+ newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified()));
+ }
+ lastFile.setInstanceFiles(newFileInfo);
+ } catch (Throwable e) {
+ logger.error(String.format("scan add found[%s] but start failed", destination), e);
+ }
+ }
+
+ private void notifyStop(String destination) {
+ InstanceAction action = actions.remove(destination);
+ if (action != null) {
+ try {
+ action.stop(destination);
+ lastFiles.remove(destination);
+ } catch (Throwable e) {
+ logger.error(String.format("scan delete found[%s] but stop failed", destination), e);
+ actions.put(destination, action);// 再重新加回去,下一次scan时再执行删除
+ }
+ }
+ }
+
+ private void notifyReload(String destination) {
+ InstanceAction action = actions.get(destination);
+ if (action != null) {
+ try {
+ action.reload(destination);
+ } catch (Throwable e) {
+ logger.error(String.format("scan reload found[%s] but reload failed", destination), e);
+ }
+ }
+ }
+
+ private boolean judgeFileChanged(File[] instanceConfigs, List fileInfos) {
+ boolean hasChanged = false;
+ for (File instanceConfig : instanceConfigs) {
+ for (FileInfo fileInfo : fileInfos) {
+ if (instanceConfig.getName().equals(fileInfo.getName())) {
+ hasChanged |= (instanceConfig.lastModified() != fileInfo.getLastModified());
+ if (hasChanged) {
+ return hasChanged;
+ }
+ }
+ }
+ }
+
+ return hasChanged;
+ }
+
+ public void setDefaultAction(InstanceAction defaultAction) {
+ this.defaultAction = defaultAction;
+ }
+
+ public void setScanIntervalInSecond(long scanIntervalInSecond) {
+ this.scanIntervalInSecond = scanIntervalInSecond;
+ }
+
+ public static class InstanceConfigFiles {
+
+ private String destination; // instance
+ // name
+ private List springFile = new ArrayList<>(); // spring的instance
+ // xml
+ private FileInfo rootFile; // canal.properties
+ private List instanceFiles = new ArrayList<>(); // instance对应的配置
+
+ public InstanceConfigFiles(String destination){
+ this.destination = destination;
+ }
+
+ public String getDestination() {
+ return destination;
+ }
+
+ public void setDestination(String destination) {
+ this.destination = destination;
+ }
+
+ public List getSpringFile() {
+ return springFile;
+ }
+
+ public void setSpringFile(List springFile) {
+ this.springFile = springFile;
+ }
+
+ public FileInfo getRootFile() {
+ return rootFile;
+ }
+
+ public void setRootFile(FileInfo rootFile) {
+ this.rootFile = rootFile;
+ }
+
+ public List getInstanceFiles() {
+ return instanceFiles;
+ }
+
+ public void setInstanceFiles(List instanceFiles) {
+ this.instanceFiles = instanceFiles;
+ }
+
+ }
+
+ public static class FileInfo {
+
+ private String name;
+ private long lastModified = 0;
+
+ public FileInfo(String name, long lastModified){
+ this.name = name;
+ this.lastModified = lastModified;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public long getLastModified() {
+ return lastModified;
+ }
+
+ public void setLastModified(long lastModified) {
+ this.lastModified = lastModified;
+ }
+
+ }
+}
diff --git a/smtweb-framework/canal/server/src/main/resources/canal.properties b/smtweb-framework/canal/server/src/main/resources/canal.properties
new file mode 100644
index 0000000..82992bc
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/resources/canal.properties
@@ -0,0 +1,182 @@
+#################################################
+######### common argument #############
+#################################################
+# tcp bind ip
+canal.ip =
+# register ip to zookeeper
+canal.register.ip =
+canal.port = 11111
+canal.metrics.pull.port = 11112
+# canal instance user/passwd
+# canal.user = canal
+# canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458
+
+# canal admin config
+#canal.admin.manager = 127.0.0.1:8089
+canal.admin.port = 11110
+canal.admin.user = admin
+canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
+# admin auto register
+#canal.admin.register.auto = true
+#canal.admin.register.cluster =
+#canal.admin.register.name =
+
+canal.zkServers =
+# flush data to zk
+canal.zookeeper.flush.period = 1000
+canal.withoutNetty = false
+# tcp, kafka, rocketMQ, rabbitMQ, pulsarMQ
+canal.serverMode = tcp
+# flush meta cursor/parse position to file
+canal.file.data.dir = ${canal.conf.dir}
+canal.file.flush.period = 1000
+## memory store RingBuffer size, should be Math.pow(2,n)
+canal.instance.memory.buffer.size = 16384
+## memory store RingBuffer used memory unit size , default 1kb
+canal.instance.memory.buffer.memunit = 1024
+## meory store gets mode used MEMSIZE or ITEMSIZE
+canal.instance.memory.batch.mode = MEMSIZE
+canal.instance.memory.rawEntry = true
+
+## detecing config
+canal.instance.detecting.enable = false
+#canal.instance.detecting.sql = insert into retl.xdual values(1,now()) on duplicate key update x=now()
+canal.instance.detecting.sql = select 1
+canal.instance.detecting.interval.time = 3
+canal.instance.detecting.retry.threshold = 3
+canal.instance.detecting.heartbeatHaEnable = false
+
+# support maximum transaction size, more than the size of the transaction will be cut into multiple transactions delivery
+canal.instance.transaction.size = 1024
+# mysql fallback connected to new master should fallback times
+canal.instance.fallbackIntervalInSeconds = 60
+
+# network config
+canal.instance.network.receiveBufferSize = 16384
+canal.instance.network.sendBufferSize = 16384
+canal.instance.network.soTimeout = 30
+
+# binlog filter config
+canal.instance.filter.druid.ddl = true
+canal.instance.filter.query.dcl = false
+canal.instance.filter.query.dml = false
+canal.instance.filter.query.ddl = false
+canal.instance.filter.table.error = false
+canal.instance.filter.rows = false
+canal.instance.filter.transaction.entry = false
+canal.instance.filter.dml.insert = false
+canal.instance.filter.dml.update = false
+canal.instance.filter.dml.delete = false
+
+# binlog format/image check
+canal.instance.binlog.format = ROW,STATEMENT,MIXED
+canal.instance.binlog.image = FULL,MINIMAL,NOBLOB
+
+# binlog ddl isolation
+canal.instance.get.ddl.isolation = false
+
+# parallel parser config
+canal.instance.parser.parallel = true
+## concurrent thread number, default 60% available processors, suggest not to exceed Runtime.getRuntime().availableProcessors()
+#canal.instance.parser.parallelThreadSize = 16
+## disruptor ringbuffer size, must be power of 2
+canal.instance.parser.parallelBufferSize = 256
+
+# table meta tsdb info
+canal.instance.tsdb.enable = true
+canal.instance.tsdb.dir = ${canal.file.data.dir:../conf}/${canal.instance.destination:}
+canal.instance.tsdb.url = jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
+canal.instance.tsdb.dbUsername = canal
+canal.instance.tsdb.dbPassword = canal
+# dump snapshot interval, default 24 hour
+canal.instance.tsdb.snapshot.interval = 24
+# purge snapshot expire , default 360 hour(15 days)
+canal.instance.tsdb.snapshot.expire = 360
+
+#################################################
+######### destinations #############
+#################################################
+canal.destinations = example
+# conf root dir
+canal.conf.dir = ../conf
+# auto scan instance dir add/remove and start/stop instance
+canal.auto.scan = true
+canal.auto.scan.interval = 5
+# set this value to 'true' means that when binlog pos not found, skip to latest.
+# WARN: pls keep 'false' in production env, or if you know what you want.
+canal.auto.reset.latest.pos.mode = false
+
+canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml
+#canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml
+
+canal.instance.global.mode = spring
+canal.instance.global.lazy = false
+canal.instance.global.manager.address = ${canal.admin.manager}
+#canal.instance.global.spring.xml = classpath:spring/memory-instance.xml
+canal.instance.global.spring.xml = classpath:spring/file-instance.xml
+#canal.instance.global.spring.xml = classpath:spring/default-instance.xml
+
+##################################################
+######### MQ Properties #############
+##################################################
+# aliyun ak/sk , support rds/mq
+canal.aliyun.accessKey =
+canal.aliyun.secretKey =
+canal.aliyun.uid=
+
+canal.mq.flatMessage = true
+canal.mq.canalBatchSize = 50
+canal.mq.canalGetTimeout = 100
+# Set this value to "cloud", if you want open message trace feature in aliyun.
+canal.mq.accessChannel = local
+
+canal.mq.database.hash = true
+canal.mq.send.thread.size = 30
+canal.mq.build.thread.size = 8
+
+##################################################
+######### Kafka #############
+##################################################
+kafka.bootstrap.servers = 127.0.0.1:9092
+kafka.acks = all
+kafka.compression.type = none
+kafka.batch.size = 16384
+kafka.linger.ms = 1
+kafka.max.request.size = 1048576
+kafka.buffer.memory = 33554432
+kafka.max.in.flight.requests.per.connection = 1
+kafka.retries = 0
+
+kafka.kerberos.enable = false
+kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
+kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"
+
+##################################################
+######### RocketMQ #############
+##################################################
+rocketmq.producer.group = test
+rocketmq.enable.message.trace = false
+rocketmq.customized.trace.topic =
+rocketmq.namespace =
+rocketmq.namesrv.addr = 127.0.0.1:9876
+rocketmq.retry.times.when.send.failed = 0
+rocketmq.vip.channel.enabled = false
+rocketmq.tag =
+
+##################################################
+######### RabbitMQ #############
+##################################################
+rabbitmq.host =
+rabbitmq.virtual.host =
+rabbitmq.exchange =
+rabbitmq.username =
+rabbitmq.password =
+rabbitmq.deliveryMode =
+
+
+##################################################
+######### Pulsar #############
+##################################################
+pulsarmq.serverUrl =
+pulsarmq.roleToken =
+pulsarmq.topicTenantPrefix =
\ No newline at end of file
diff --git a/smtweb-framework/canal/server/src/main/resources/canal_local.properties b/smtweb-framework/canal/server/src/main/resources/canal_local.properties
new file mode 100644
index 0000000..f92b8f3
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/resources/canal_local.properties
@@ -0,0 +1,12 @@
+# register ip
+canal.register.ip =
+
+# canal admin config
+canal.admin.manager = 127.0.0.1:8089
+canal.admin.port = 11110
+canal.admin.user = admin
+canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
+# admin auto register
+canal.admin.register.auto = true
+canal.admin.register.cluster =
+canal.admin.register.name =
\ No newline at end of file
diff --git a/smtweb-framework/canal/server/src/main/resources/example/instance.properties b/smtweb-framework/canal/server/src/main/resources/example/instance.properties
new file mode 100644
index 0000000..7a1f752
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/resources/example/instance.properties
@@ -0,0 +1,59 @@
+#################################################
+## mysql serverId , v1.0.26+ will autoGen
+canal.instance.mysql.slaveId=2
+
+# enable gtid use true/false
+canal.instance.gtidon=false
+
+# position info
+canal.instance.master.address=172.28.123.25:3306
+#canal.instance.master.journal.name=
+#canal.instance.master.position=
+#canal.instance.master.timestamp=
+#canal.instance.master.gtid=
+
+# rds oss binlog
+canal.instance.rds.accesskey=
+canal.instance.rds.secretkey=
+canal.instance.rds.instanceId=
+
+# table meta tsdb info
+canal.instance.tsdb.enable=true
+#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
+#canal.instance.tsdb.dbUsername=canal
+#canal.instance.tsdb.dbPassword=canal
+
+#canal.instance.standby.address =
+#canal.instance.standby.journal.name =
+#canal.instance.standby.position =
+#canal.instance.standby.timestamp =
+#canal.instance.standby.gtid=
+
+# username/password
+canal.instance.dbUsername=root
+canal.instance.dbPassword=root
+canal.instance.connectionCharset = UTF-8
+# enable druid Decrypt database password
+canal.instance.enableDruid=false
+#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==
+
+# table regex
+canal.instance.filter.regex=scmz\\..*
+# table black regex
+canal.instance.filter.black.regex=mysql\\.slave_.*
+# table field filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)
+#canal.instance.filter.field=test1.t_product:id/subject/keywords,test2.t_company:id/name/contact/ch
+# table field black filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)
+#canal.instance.filter.black.field=test1.t_product:subject/product_image,test2.t_company:id/name/contact/ch
+
+# mq config
+canal.mq.topic=example
+# dynamic topic route by schema or table regex
+#canal.mq.dynamicTopic=mytest1.user,topic2:mytest2\\..*,.*\\..*
+canal.mq.partition=0
+# hash partition config
+#canal.mq.enableDynamicQueuePartition=false
+#canal.mq.partitionsNum=3
+#canal.mq.dynamicTopicPartitionNum=test.*:4,mycanal:6
+#canal.mq.partitionHash=test.table:id^name,.*\\..*
+#################################################
diff --git a/smtweb-framework/canal/server/src/main/resources/logback.xml.bak b/smtweb-framework/canal/server/src/main/resources/logback.xml.bak
new file mode 100644
index 0000000..682e80f
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/resources/logback.xml.bak
@@ -0,0 +1,117 @@
+
+
+
+
+ %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
+
+
+
+
+
+
+ destination
+ canal
+
+
+
+ ../logs/${destination}/${destination}.log
+
+
+ ../logs/${destination}/%d{yyyy-MM-dd}/${destination}-%d{yyyy-MM-dd}-%i.log.gz
+
+
+ 512MB
+
+ 60
+
+
+
+ %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
+
+
+
+
+
+
+
+
+ destination
+ canal
+
+
+
+ ../logs/${destination}/meta.log
+
+
+ ../logs/${destination}/%d{yyyy-MM-dd}/meta-%d{yyyy-MM-dd}-%i.log.gz
+
+
+ 32MB
+
+ 60
+
+
+
+ %d{yyyy-MM-dd HH:mm:ss.SSS} - %msg%n
+
+
+
+
+
+
+
+ ../logs/canal/rocketmq_client.log
+
+
+ ../logs/canal/%d{yyyy-MM-dd}/rocketmq_client-%d{yyyy-MM-dd}-%i.log.gz
+
+
+ 512MB
+
+ 60
+
+
+ %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/server/src/main/resources/metrics/Canal_instances_tmpl.json b/smtweb-framework/canal/server/src/main/resources/metrics/Canal_instances_tmpl.json
new file mode 100644
index 0000000..89fd332
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/resources/metrics/Canal_instances_tmpl.json
@@ -0,0 +1,1375 @@
+{
+ "__inputs": [
+ {
+ "name": "DS_PROMETHEUS",
+ "label": "prometheus",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "prometheus",
+ "pluginName": "Prometheus"
+ }
+ ],
+ "__requires": [
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "5.2.2"
+ },
+ {
+ "type": "panel",
+ "id": "graph",
+ "name": "Graph",
+ "version": "5.0.0"
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "5.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "id": null,
+ "iteration": 1536989235272,
+ "links": [],
+ "panels": [
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 30,
+ "panels": [],
+ "title": "Instance status",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "Canal instance 基本信息。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 0,
+ "y": 1
+ },
+ "id": 24,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sideWidth": 500,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "repeat": null,
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "canal_instance{destination=~\"$destination\"}",
+ "format": "time_series",
+ "instant": true,
+ "intervalFactor": 1,
+ "legendFormat": "Destination: {{destination}}",
+ "refId": "A"
+ },
+ {
+ "expr": "canal_instance_parser_mode{destination=~\"$destination\"}",
+ "format": "time_series",
+ "instant": true,
+ "intervalFactor": 1,
+ "legendFormat": "Parallel parser: {{parallel}}",
+ "refId": "B"
+ },
+ {
+ "expr": "canal_instance_store{destination=~\"$destination\"}",
+ "format": "time_series",
+ "instant": true,
+ "intervalFactor": 1,
+ "legendFormat": "Batch mode: {{batchMode}}",
+ "refId": "C"
+ },
+ {
+ "expr": "canal_instance_store{destination=~\"$destination\"}",
+ "format": "time_series",
+ "instant": true,
+ "intervalFactor": 1,
+ "legendFormat": "Buffer size: {{size}}",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Basic",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": false,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "inbound": "#bf1b00"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "Canal instance 网络带宽占用。\ninbound: 读取MySQL binlog.\noutbound: 对Client端传输格式化binlog.",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 6,
+ "y": 1
+ },
+ "id": 6,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"0\"}[2m]) / 1024",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "inbound",
+ "refId": "A"
+ },
+ {
+ "expr": "rate(canal_instance_client_bytes{destination=~\"$destination\"}[2m]) / 1024",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "outbound",
+ "refId": "B"
+ },
+ {
+ "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"1\"}[2m]) / 1024",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "inbound-1",
+ "refId": "C"
+ },
+ {
+ "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"2\"}[2m]) / 1024",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "inbound-2",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Network bandwith",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "KBs",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "ack": "#f29191",
+ "get": "#cca300",
+ "put": "#1f78c1"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "master: Canal server相对于MySQL master的延时。通过master heartbeat机制能刷新idle状态下的延时。\nput: store put操作的时间点为基准。\nget: client get操作的时间点为基准。\nack: client ack操作的时间点为基准。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 12,
+ "y": 1
+ },
+ "id": 4,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "canal_instance_traffic_delay{destination=~\"$destination\"} / 1000",
+ "format": "time_series",
+ "hide": false,
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "master",
+ "refId": "D"
+ },
+ {
+ "expr": "canal_instance_put_delay{destination=~\"$destination\"} / 1000",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "put",
+ "refId": "A"
+ },
+ {
+ "expr": "canal_instance_get_delay{destination=~\"$destination\"} / 1000",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "get",
+ "refId": "B"
+ },
+ {
+ "expr": "canal_instance_ack_delay{destination=~\"$destination\"} / 1000",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "ack",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Delay",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "sink线程blocking占比;dump线程blocking占比(仅parallel mode)。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 18,
+ "y": 1
+ },
+ "hideTimeOverride": false,
+ "id": 2,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"0\"}[2m]), 1000) / 10",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "dump",
+ "refId": "B"
+ },
+ {
+ "expr": "clamp_max(rate(canal_instance_sink_blocking_time{destination=~\"$destination\"}[2m]), 1000) / 10",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "sink",
+ "refId": "A"
+ },
+ {
+ "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"1\"}[2m]), 1000) / 10",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "dump-1",
+ "refId": "C"
+ },
+ {
+ "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"2\"}[2m]), 1000) / 10",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "dump-2",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Blocking",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percent",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 6
+ },
+ "id": 32,
+ "panels": [],
+ "title": "Throughput",
+ "type": "row"
+ },
+ {
+ "aliasColors": {
+ "rowDatas": "#7eb26d",
+ "tableRows": "#c15c17"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "Instance处理binlog的TPS(以master变更行数table rows为基准计算)。\nput: put操作TPS。\nget: get操作TPS。\nack: ack操作TPS。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 0,
+ "y": 7
+ },
+ "id": 14,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(canal_instance_put_rows{destination=~\"$destination\"}[2m])",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "put",
+ "refId": "A"
+ },
+ {
+ "expr": "rate(canal_instance_get_rows{destination=~\"$destination\"}[2m])",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "get",
+ "refId": "B"
+ },
+ {
+ "expr": "rate(canal_instance_ack_rows{destination=~\"$destination\"}[2m])",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "ack",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "TPS(table rows)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "iops",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "transactions": "#f9ba8f"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "Canal instance 处理binlog的TPS,以MySQL transaction为单位计算。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 6,
+ "y": 7
+ },
+ "id": 12,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(canal_instance_transactions{destination=~\"$destination\"}[2m])",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "transactions",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "TPS(MySQL transaction)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "iops",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 12
+ },
+ "id": 34,
+ "panels": [],
+ "title": "Client",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "Canal instance接收到的请求统计,结果按packet type分类。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 0,
+ "y": 13
+ },
+ "id": 16,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "canal_instance_client_packets{destination=~\"$destination\"}",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "{{packetType}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Client requests",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "client 请求的GET与ACK包的QPS。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 6,
+ "y": 13
+ },
+ "id": 38,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(canal_instance_client_packets{destination=~\"$destination\",packetType=\"GET\"}[2m])",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "GET",
+ "refId": "A"
+ },
+ {
+ "expr": "rate(canal_instance_client_packets{destination=~\"$destination\",packetType=\"CLIENTACK\"}[2m])",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "ACK",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Client QPS",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "server响应GET请求,但返回空包的占比。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 12,
+ "y": 13
+ },
+ "id": 26,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(canal_instance_client_empty_batches{destination=~\"$destination\"}[2m])",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "empty",
+ "refId": "A"
+ },
+ {
+ "expr": "rate(canal_instance_client_packets{destination=~\"$destination\", packetType=\"GET\"}[2m])",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "nonempty",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Empty packets",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "wps",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "Canal client 请求响应时间的概况。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 18,
+ "y": 13
+ },
+ "id": 18,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "25.0",
+ "yaxis": 1
+ },
+ {
+ "alias": "100.0",
+ "yaxis": 1
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(canal_instance_client_request_latency_bucket{destination=~\"$destination\"}[2m])",
+ "format": "time_series",
+ "hide": false,
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "{{le}}ms",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Response time",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transparent": false,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 18
+ },
+ "id": 36,
+ "panels": [],
+ "title": "Store",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "Canal instance ringbuffer内未释放的events数量。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 0,
+ "y": 19
+ },
+ "id": 20,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "canal_instance_store_produce_seq{destination=~\"$destination\"} - canal_instance_store_consume_seq{destination=~\"$destination\"}",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "events",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Store remain events",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": null,
+ "format": "none",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "decimals": null,
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "Canal instance ringbuffer 内未释放events占用内存。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 6,
+ "y": 19
+ },
+ "id": 22,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "(canal_instance_store_produce_mem{destination=~\"$destination\"} - canal_instance_store_consume_mem{destination=~\"$destination\"}) / 1024",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "memsize",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Store remain mem",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "deckbytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "refresh": false,
+ "schemaVersion": 16,
+ "style": "dark",
+ "tags": [
+ "canal"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "prometheus",
+ "value": "prometheus"
+ },
+ "hide": 0,
+ "label": "datasource",
+ "name": "datasource",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": null,
+ "current": {},
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": false,
+ "label": "destination",
+ "multi": false,
+ "name": "destination",
+ "options": [],
+ "query": "label_values(canal_instance, destination)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "",
+ "title": "Canal instances",
+ "uid": "8vh8NGpiz",
+ "version": 103
+}
\ No newline at end of file
diff --git a/smtweb-framework/canal/server/src/main/resources/spring/base-instance.xml b/smtweb-framework/canal/server/src/main/resources/spring/base-instance.xml
new file mode 100644
index 0000000..a89b740
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/resources/spring/base-instance.xml
@@ -0,0 +1,39 @@
+
+
+
+
+
+
+
+
+
+ classpath:canal.properties
+ classpath:${canal.instance.destination:}/instance.properties
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/smtweb-framework/canal/server/src/main/resources/spring/default-instance.xml b/smtweb-framework/canal/server/src/main/resources/spring/default-instance.xml
new file mode 100644
index 0000000..9907ae8
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/resources/spring/default-instance.xml
@@ -0,0 +1,211 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ${canal.zkServers:127.0.0.1:2181}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/server/src/main/resources/spring/file-instance.xml b/smtweb-framework/canal/server/src/main/resources/spring/file-instance.xml
new file mode 100644
index 0000000..800f981
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/resources/spring/file-instance.xml
@@ -0,0 +1,197 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/server/src/main/resources/spring/group-instance.xml b/smtweb-framework/canal/server/src/main/resources/spring/group-instance.xml
new file mode 100644
index 0000000..b0b887f
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/resources/spring/group-instance.xml
@@ -0,0 +1,292 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/server/src/main/resources/spring/memory-instance.xml b/smtweb-framework/canal/server/src/main/resources/spring/memory-instance.xml
new file mode 100644
index 0000000..a7dc634
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/resources/spring/memory-instance.xml
@@ -0,0 +1,185 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/server/src/main/resources/spring/tsdb/h2-tsdb.xml b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/h2-tsdb.xml
new file mode 100644
index 0000000..29c7a24
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/h2-tsdb.xml
@@ -0,0 +1,61 @@
+
+
+
+
+
+
+
+
+
+ classpath:canal.properties
+ classpath:${canal.instance.destination:}/instance.properties
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/server/src/main/resources/spring/tsdb/mysql-tsdb.xml b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/mysql-tsdb.xml
new file mode 100644
index 0000000..72d82bf
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/mysql-tsdb.xml
@@ -0,0 +1,63 @@
+
+
+
+
+
+
+
+
+
+ classpath:canal.properties
+ classpath:${canal.instance.destination:}/instance.properties
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap-config.xml b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap-config.xml
new file mode 100644
index 0000000..c27d463
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap-config.xml
@@ -0,0 +1,14 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml
new file mode 100644
index 0000000..206c712
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml
@@ -0,0 +1,44 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ insert into meta_history ()
+ values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#{destination},#{binlogFile},#{binlogOffest},#{binlogMasterId},#{binlogTimestamp},#{useSchema},#{sqlSchema},#{sqlTable},#{sqlText},#{sqlType},#{extra})
+
+
+
+ delete from meta_history
+ where destination=#{destination}
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml
new file mode 100644
index 0000000..a4d1466
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml
@@ -0,0 +1,48 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ insert into meta_snapshot ()
+ values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#{destination},#{binlogFile},#{binlogOffest},#{binlogMasterId},#{binlogTimestamp},#{data},#{extra})
+
+
+
+ update meta_snapshot set gmt_modified=now(),
+ binlog_file=#{binlogFile},binlog_offest=#{binlogOffest},binlog_master_id=#{binlogMasterId},binlog_timestamp=#{binlogTimestamp},data=#{data},extra=#{extra}
+ where destination=#{destination} and binlog_timestamp=0
+
+
+
+ delete from meta_snapshot
+ where destination=#{destination}
+
+
+
+ 0
+ ]]>
+
+
\ No newline at end of file
diff --git a/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql/create_table.sql b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql/create_table.sql
new file mode 100644
index 0000000..8cb82c8
--- /dev/null
+++ b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql/create_table.sql
@@ -0,0 +1,39 @@
+CREATE TABLE IF NOT EXISTS `meta_snapshot` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',
+ `gmt_create` datetime NOT NULL COMMENT '创建时间',
+ `gmt_modified` datetime NOT NULL COMMENT '修改时间',
+ `destination` varchar(128) DEFAULT NULL COMMENT '通道名称',
+ `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名',
+ `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量',
+ `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id',
+ `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳',
+ `data` longtext DEFAULT NULL COMMENT '表结构数据',
+ `extra` text DEFAULT NULL COMMENT '额外的扩展信息',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`),
+ KEY `destination` (`destination`),
+ KEY `destination_timestamp` (`destination`,`binlog_timestamp`),
+ KEY `gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='表结构记录表快照表';
+
+CREATE TABLE IF NOT EXISTS `meta_history` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',
+ `gmt_create` datetime NOT NULL COMMENT '创建时间',
+ `gmt_modified` datetime NOT NULL COMMENT '修改时间',
+ `destination` varchar(128) DEFAULT NULL COMMENT '通道名称',
+ `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名',
+ `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量',
+ `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id',
+ `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳',
+ `use_schema` varchar(1024) DEFAULT NULL COMMENT '执行sql时对应的schema',
+ `sql_schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema',
+ `sql_table` varchar(1024) DEFAULT NULL COMMENT '对应的table',
+ `sql_text` longtext DEFAULT NULL COMMENT '执行的sql',
+ `sql_type` varchar(256) DEFAULT NULL COMMENT 'sql类型',
+ `extra` text DEFAULT NULL COMMENT '额外的扩展信息',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`),
+ KEY `destination` (`destination`),
+ KEY `destination_timestamp` (`destination`,`binlog_timestamp`),
+ KEY `gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='表结构变化明细表';
\ No newline at end of file
diff --git a/smtweb-framework/canal/server/target/classes/canal.properties b/smtweb-framework/canal/server/target/classes/canal.properties
new file mode 100644
index 0000000..82992bc
--- /dev/null
+++ b/smtweb-framework/canal/server/target/classes/canal.properties
@@ -0,0 +1,182 @@
+#################################################
+######### common argument #############
+#################################################
+# tcp bind ip
+canal.ip =
+# register ip to zookeeper
+canal.register.ip =
+canal.port = 11111
+canal.metrics.pull.port = 11112
+# canal instance user/passwd
+# canal.user = canal
+# canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458
+
+# canal admin config
+#canal.admin.manager = 127.0.0.1:8089
+canal.admin.port = 11110
+canal.admin.user = admin
+canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
+# admin auto register
+#canal.admin.register.auto = true
+#canal.admin.register.cluster =
+#canal.admin.register.name =
+
+canal.zkServers =
+# flush data to zk
+canal.zookeeper.flush.period = 1000
+canal.withoutNetty = false
+# tcp, kafka, rocketMQ, rabbitMQ, pulsarMQ
+canal.serverMode = tcp
+# flush meta cursor/parse position to file
+canal.file.data.dir = ${canal.conf.dir}
+canal.file.flush.period = 1000
+## memory store RingBuffer size, should be Math.pow(2,n)
+canal.instance.memory.buffer.size = 16384
+## memory store RingBuffer used memory unit size , default 1kb
+canal.instance.memory.buffer.memunit = 1024
+## meory store gets mode used MEMSIZE or ITEMSIZE
+canal.instance.memory.batch.mode = MEMSIZE
+canal.instance.memory.rawEntry = true
+
+## detecing config
+canal.instance.detecting.enable = false
+#canal.instance.detecting.sql = insert into retl.xdual values(1,now()) on duplicate key update x=now()
+canal.instance.detecting.sql = select 1
+canal.instance.detecting.interval.time = 3
+canal.instance.detecting.retry.threshold = 3
+canal.instance.detecting.heartbeatHaEnable = false
+
+# support maximum transaction size, more than the size of the transaction will be cut into multiple transactions delivery
+canal.instance.transaction.size = 1024
+# mysql fallback connected to new master should fallback times
+canal.instance.fallbackIntervalInSeconds = 60
+
+# network config
+canal.instance.network.receiveBufferSize = 16384
+canal.instance.network.sendBufferSize = 16384
+canal.instance.network.soTimeout = 30
+
+# binlog filter config
+canal.instance.filter.druid.ddl = true
+canal.instance.filter.query.dcl = false
+canal.instance.filter.query.dml = false
+canal.instance.filter.query.ddl = false
+canal.instance.filter.table.error = false
+canal.instance.filter.rows = false
+canal.instance.filter.transaction.entry = false
+canal.instance.filter.dml.insert = false
+canal.instance.filter.dml.update = false
+canal.instance.filter.dml.delete = false
+
+# binlog format/image check
+canal.instance.binlog.format = ROW,STATEMENT,MIXED
+canal.instance.binlog.image = FULL,MINIMAL,NOBLOB
+
+# binlog ddl isolation
+canal.instance.get.ddl.isolation = false
+
+# parallel parser config
+canal.instance.parser.parallel = true
+## concurrent thread number, default 60% available processors, suggest not to exceed Runtime.getRuntime().availableProcessors()
+#canal.instance.parser.parallelThreadSize = 16
+## disruptor ringbuffer size, must be power of 2
+canal.instance.parser.parallelBufferSize = 256
+
+# table meta tsdb info
+canal.instance.tsdb.enable = true
+canal.instance.tsdb.dir = ${canal.file.data.dir:../conf}/${canal.instance.destination:}
+canal.instance.tsdb.url = jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
+canal.instance.tsdb.dbUsername = canal
+canal.instance.tsdb.dbPassword = canal
+# dump snapshot interval, default 24 hour
+canal.instance.tsdb.snapshot.interval = 24
+# purge snapshot expire , default 360 hour(15 days)
+canal.instance.tsdb.snapshot.expire = 360
+
+#################################################
+######### destinations #############
+#################################################
+canal.destinations = example
+# conf root dir
+canal.conf.dir = ../conf
+# auto scan instance dir add/remove and start/stop instance
+canal.auto.scan = true
+canal.auto.scan.interval = 5
+# set this value to 'true' means that when binlog pos not found, skip to latest.
+# WARN: pls keep 'false' in production env, or if you know what you want.
+canal.auto.reset.latest.pos.mode = false
+
+canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml
+#canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml
+
+canal.instance.global.mode = spring
+canal.instance.global.lazy = false
+canal.instance.global.manager.address = ${canal.admin.manager}
+#canal.instance.global.spring.xml = classpath:spring/memory-instance.xml
+canal.instance.global.spring.xml = classpath:spring/file-instance.xml
+#canal.instance.global.spring.xml = classpath:spring/default-instance.xml
+
+##################################################
+######### MQ Properties #############
+##################################################
+# aliyun ak/sk , support rds/mq
+canal.aliyun.accessKey =
+canal.aliyun.secretKey =
+canal.aliyun.uid=
+
+canal.mq.flatMessage = true
+canal.mq.canalBatchSize = 50
+canal.mq.canalGetTimeout = 100
+# Set this value to "cloud", if you want open message trace feature in aliyun.
+canal.mq.accessChannel = local
+
+canal.mq.database.hash = true
+canal.mq.send.thread.size = 30
+canal.mq.build.thread.size = 8
+
+##################################################
+######### Kafka #############
+##################################################
+kafka.bootstrap.servers = 127.0.0.1:9092
+kafka.acks = all
+kafka.compression.type = none
+kafka.batch.size = 16384
+kafka.linger.ms = 1
+kafka.max.request.size = 1048576
+kafka.buffer.memory = 33554432
+kafka.max.in.flight.requests.per.connection = 1
+kafka.retries = 0
+
+kafka.kerberos.enable = false
+kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
+kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"
+
+##################################################
+######### RocketMQ #############
+##################################################
+rocketmq.producer.group = test
+rocketmq.enable.message.trace = false
+rocketmq.customized.trace.topic =
+rocketmq.namespace =
+rocketmq.namesrv.addr = 127.0.0.1:9876
+rocketmq.retry.times.when.send.failed = 0
+rocketmq.vip.channel.enabled = false
+rocketmq.tag =
+
+##################################################
+######### RabbitMQ #############
+##################################################
+rabbitmq.host =
+rabbitmq.virtual.host =
+rabbitmq.exchange =
+rabbitmq.username =
+rabbitmq.password =
+rabbitmq.deliveryMode =
+
+
+##################################################
+######### Pulsar #############
+##################################################
+pulsarmq.serverUrl =
+pulsarmq.roleToken =
+pulsarmq.topicTenantPrefix =
\ No newline at end of file
diff --git a/smtweb-framework/canal/server/target/classes/canal_local.properties b/smtweb-framework/canal/server/target/classes/canal_local.properties
new file mode 100644
index 0000000..f92b8f3
--- /dev/null
+++ b/smtweb-framework/canal/server/target/classes/canal_local.properties
@@ -0,0 +1,12 @@
+# register ip
+canal.register.ip =
+
+# canal admin config
+canal.admin.manager = 127.0.0.1:8089
+canal.admin.port = 11110
+canal.admin.user = admin
+canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
+# admin auto register
+canal.admin.register.auto = true
+canal.admin.register.cluster =
+canal.admin.register.name =
\ No newline at end of file
diff --git a/smtweb-framework/canal/server/target/classes/example/instance.properties b/smtweb-framework/canal/server/target/classes/example/instance.properties
new file mode 100644
index 0000000..7a1f752
--- /dev/null
+++ b/smtweb-framework/canal/server/target/classes/example/instance.properties
@@ -0,0 +1,59 @@
+#################################################
+## mysql serverId , v1.0.26+ will autoGen
+canal.instance.mysql.slaveId=2
+
+# enable gtid use true/false
+canal.instance.gtidon=false
+
+# position info
+canal.instance.master.address=172.28.123.25:3306
+#canal.instance.master.journal.name=
+#canal.instance.master.position=
+#canal.instance.master.timestamp=
+#canal.instance.master.gtid=
+
+# rds oss binlog
+canal.instance.rds.accesskey=
+canal.instance.rds.secretkey=
+canal.instance.rds.instanceId=
+
+# table meta tsdb info
+canal.instance.tsdb.enable=true
+#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
+#canal.instance.tsdb.dbUsername=canal
+#canal.instance.tsdb.dbPassword=canal
+
+#canal.instance.standby.address =
+#canal.instance.standby.journal.name =
+#canal.instance.standby.position =
+#canal.instance.standby.timestamp =
+#canal.instance.standby.gtid=
+
+# username/password
+canal.instance.dbUsername=root
+canal.instance.dbPassword=root
+canal.instance.connectionCharset = UTF-8
+# enable druid Decrypt database password
+canal.instance.enableDruid=false
+#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==
+
+# table regex
+canal.instance.filter.regex=scmz\\..*
+# table black regex
+canal.instance.filter.black.regex=mysql\\.slave_.*
+# table field filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)
+#canal.instance.filter.field=test1.t_product:id/subject/keywords,test2.t_company:id/name/contact/ch
+# table field black filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)
+#canal.instance.filter.black.field=test1.t_product:subject/product_image,test2.t_company:id/name/contact/ch
+
+# mq config
+canal.mq.topic=example
+# dynamic topic route by schema or table regex
+#canal.mq.dynamicTopic=mytest1.user,topic2:mytest2\\..*,.*\\..*
+canal.mq.partition=0
+# hash partition config
+#canal.mq.enableDynamicQueuePartition=false
+#canal.mq.partitionsNum=3
+#canal.mq.dynamicTopicPartitionNum=test.*:4,mycanal:6
+#canal.mq.partitionHash=test.table:id^name,.*\\..*
+#################################################
diff --git a/smtweb-framework/canal/server/target/classes/logback.xml.bak b/smtweb-framework/canal/server/target/classes/logback.xml.bak
new file mode 100644
index 0000000..682e80f
--- /dev/null
+++ b/smtweb-framework/canal/server/target/classes/logback.xml.bak
@@ -0,0 +1,117 @@
+
+
+
+
+ %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
+
+
+
+
+
+
+ destination
+ canal
+
+
+
+ ../logs/${destination}/${destination}.log
+
+
+ ../logs/${destination}/%d{yyyy-MM-dd}/${destination}-%d{yyyy-MM-dd}-%i.log.gz
+
+
+ 512MB
+
+ 60
+
+
+
+ %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
+
+
+
+
+
+
+
+
+ destination
+ canal
+
+
+
+ ../logs/${destination}/meta.log
+
+
+ ../logs/${destination}/%d{yyyy-MM-dd}/meta-%d{yyyy-MM-dd}-%i.log.gz
+
+
+ 32MB
+
+ 60
+
+
+
+ %d{yyyy-MM-dd HH:mm:ss.SSS} - %msg%n
+
+
+
+
+
+
+
+ ../logs/canal/rocketmq_client.log
+
+
+ ../logs/canal/%d{yyyy-MM-dd}/rocketmq_client-%d{yyyy-MM-dd}-%i.log.gz
+
+
+ 512MB
+
+ 60
+
+
+ %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/server/target/classes/metrics/Canal_instances_tmpl.json b/smtweb-framework/canal/server/target/classes/metrics/Canal_instances_tmpl.json
new file mode 100644
index 0000000..89fd332
--- /dev/null
+++ b/smtweb-framework/canal/server/target/classes/metrics/Canal_instances_tmpl.json
@@ -0,0 +1,1375 @@
+{
+ "__inputs": [
+ {
+ "name": "DS_PROMETHEUS",
+ "label": "prometheus",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "prometheus",
+ "pluginName": "Prometheus"
+ }
+ ],
+ "__requires": [
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "5.2.2"
+ },
+ {
+ "type": "panel",
+ "id": "graph",
+ "name": "Graph",
+ "version": "5.0.0"
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "5.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "id": null,
+ "iteration": 1536989235272,
+ "links": [],
+ "panels": [
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 30,
+ "panels": [],
+ "title": "Instance status",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "Canal instance 基本信息。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 0,
+ "y": 1
+ },
+ "id": 24,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sideWidth": 500,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "repeat": null,
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "canal_instance{destination=~\"$destination\"}",
+ "format": "time_series",
+ "instant": true,
+ "intervalFactor": 1,
+ "legendFormat": "Destination: {{destination}}",
+ "refId": "A"
+ },
+ {
+ "expr": "canal_instance_parser_mode{destination=~\"$destination\"}",
+ "format": "time_series",
+ "instant": true,
+ "intervalFactor": 1,
+ "legendFormat": "Parallel parser: {{parallel}}",
+ "refId": "B"
+ },
+ {
+ "expr": "canal_instance_store{destination=~\"$destination\"}",
+ "format": "time_series",
+ "instant": true,
+ "intervalFactor": 1,
+ "legendFormat": "Batch mode: {{batchMode}}",
+ "refId": "C"
+ },
+ {
+ "expr": "canal_instance_store{destination=~\"$destination\"}",
+ "format": "time_series",
+ "instant": true,
+ "intervalFactor": 1,
+ "legendFormat": "Buffer size: {{size}}",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Basic",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": false,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "inbound": "#bf1b00"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "Canal instance 网络带宽占用。\ninbound: 读取MySQL binlog.\noutbound: 对Client端传输格式化binlog.",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 6,
+ "y": 1
+ },
+ "id": 6,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"0\"}[2m]) / 1024",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "inbound",
+ "refId": "A"
+ },
+ {
+ "expr": "rate(canal_instance_client_bytes{destination=~\"$destination\"}[2m]) / 1024",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "outbound",
+ "refId": "B"
+ },
+ {
+ "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"1\"}[2m]) / 1024",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "inbound-1",
+ "refId": "C"
+ },
+ {
+ "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"2\"}[2m]) / 1024",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "inbound-2",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Network bandwith",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "KBs",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "ack": "#f29191",
+ "get": "#cca300",
+ "put": "#1f78c1"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "master: Canal server相对于MySQL master的延时。通过master heartbeat机制能刷新idle状态下的延时。\nput: store put操作的时间点为基准。\nget: client get操作的时间点为基准。\nack: client ack操作的时间点为基准。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 12,
+ "y": 1
+ },
+ "id": 4,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "canal_instance_traffic_delay{destination=~\"$destination\"} / 1000",
+ "format": "time_series",
+ "hide": false,
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "master",
+ "refId": "D"
+ },
+ {
+ "expr": "canal_instance_put_delay{destination=~\"$destination\"} / 1000",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "put",
+ "refId": "A"
+ },
+ {
+ "expr": "canal_instance_get_delay{destination=~\"$destination\"} / 1000",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "get",
+ "refId": "B"
+ },
+ {
+ "expr": "canal_instance_ack_delay{destination=~\"$destination\"} / 1000",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "ack",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Delay",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "sink线程blocking占比;dump线程blocking占比(仅parallel mode)。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 18,
+ "y": 1
+ },
+ "hideTimeOverride": false,
+ "id": 2,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"0\"}[2m]), 1000) / 10",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "dump",
+ "refId": "B"
+ },
+ {
+ "expr": "clamp_max(rate(canal_instance_sink_blocking_time{destination=~\"$destination\"}[2m]), 1000) / 10",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "sink",
+ "refId": "A"
+ },
+ {
+ "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"1\"}[2m]), 1000) / 10",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "dump-1",
+ "refId": "C"
+ },
+ {
+ "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"2\"}[2m]), 1000) / 10",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "dump-2",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Blocking",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percent",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 6
+ },
+ "id": 32,
+ "panels": [],
+ "title": "Throughput",
+ "type": "row"
+ },
+ {
+ "aliasColors": {
+ "rowDatas": "#7eb26d",
+ "tableRows": "#c15c17"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "Instance处理binlog的TPS(以master变更行数table rows为基准计算)。\nput: put操作TPS。\nget: get操作TPS。\nack: ack操作TPS。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 0,
+ "y": 7
+ },
+ "id": 14,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(canal_instance_put_rows{destination=~\"$destination\"}[2m])",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "put",
+ "refId": "A"
+ },
+ {
+ "expr": "rate(canal_instance_get_rows{destination=~\"$destination\"}[2m])",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "get",
+ "refId": "B"
+ },
+ {
+ "expr": "rate(canal_instance_ack_rows{destination=~\"$destination\"}[2m])",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "ack",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "TPS(table rows)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "iops",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "transactions": "#f9ba8f"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "Canal instance 处理binlog的TPS,以MySQL transaction为单位计算。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 6,
+ "y": 7
+ },
+ "id": 12,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(canal_instance_transactions{destination=~\"$destination\"}[2m])",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "transactions",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "TPS(MySQL transaction)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "iops",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 12
+ },
+ "id": 34,
+ "panels": [],
+ "title": "Client",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "Canal instance接收到的请求统计,结果按packet type分类。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 0,
+ "y": 13
+ },
+ "id": 16,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "canal_instance_client_packets{destination=~\"$destination\"}",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "{{packetType}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Client requests",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "client 请求的GET与ACK包的QPS。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 6,
+ "y": 13
+ },
+ "id": 38,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(canal_instance_client_packets{destination=~\"$destination\",packetType=\"GET\"}[2m])",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "GET",
+ "refId": "A"
+ },
+ {
+ "expr": "rate(canal_instance_client_packets{destination=~\"$destination\",packetType=\"CLIENTACK\"}[2m])",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "ACK",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Client QPS",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "server响应GET请求,但返回空包的占比。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 12,
+ "y": 13
+ },
+ "id": 26,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(canal_instance_client_empty_batches{destination=~\"$destination\"}[2m])",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "empty",
+ "refId": "A"
+ },
+ {
+ "expr": "rate(canal_instance_client_packets{destination=~\"$destination\", packetType=\"GET\"}[2m])",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "nonempty",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Empty packets",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "wps",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "Canal client 请求响应时间的概况。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 18,
+ "y": 13
+ },
+ "id": 18,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "25.0",
+ "yaxis": 1
+ },
+ {
+ "alias": "100.0",
+ "yaxis": 1
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(canal_instance_client_request_latency_bucket{destination=~\"$destination\"}[2m])",
+ "format": "time_series",
+ "hide": false,
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "{{le}}ms",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Response time",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transparent": false,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 18
+ },
+ "id": 36,
+ "panels": [],
+ "title": "Store",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "Canal instance ringbuffer内未释放的events数量。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 0,
+ "y": 19
+ },
+ "id": 20,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "canal_instance_store_produce_seq{destination=~\"$destination\"} - canal_instance_store_consume_seq{destination=~\"$destination\"}",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "events",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Store remain events",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": null,
+ "format": "none",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "decimals": null,
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "Canal instance ringbuffer 内未释放events占用内存。",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 6,
+ "y": 19
+ },
+ "id": 22,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "(canal_instance_store_produce_mem{destination=~\"$destination\"} - canal_instance_store_consume_mem{destination=~\"$destination\"}) / 1024",
+ "format": "time_series",
+ "interval": "15s",
+ "intervalFactor": 2,
+ "legendFormat": "memsize",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Store remain mem",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "deckbytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "refresh": false,
+ "schemaVersion": 16,
+ "style": "dark",
+ "tags": [
+ "canal"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "prometheus",
+ "value": "prometheus"
+ },
+ "hide": 0,
+ "label": "datasource",
+ "name": "datasource",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": null,
+ "current": {},
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": false,
+ "label": "destination",
+ "multi": false,
+ "name": "destination",
+ "options": [],
+ "query": "label_values(canal_instance, destination)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "",
+ "title": "Canal instances",
+ "uid": "8vh8NGpiz",
+ "version": 103
+}
\ No newline at end of file
diff --git a/smtweb-framework/canal/server/target/classes/spring/base-instance.xml b/smtweb-framework/canal/server/target/classes/spring/base-instance.xml
new file mode 100644
index 0000000..a89b740
--- /dev/null
+++ b/smtweb-framework/canal/server/target/classes/spring/base-instance.xml
@@ -0,0 +1,39 @@
+
+
+
+
+
+
+
+
+
+ classpath:canal.properties
+ classpath:${canal.instance.destination:}/instance.properties
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/smtweb-framework/canal/server/target/classes/spring/default-instance.xml b/smtweb-framework/canal/server/target/classes/spring/default-instance.xml
new file mode 100644
index 0000000..9907ae8
--- /dev/null
+++ b/smtweb-framework/canal/server/target/classes/spring/default-instance.xml
@@ -0,0 +1,211 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ${canal.zkServers:127.0.0.1:2181}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/server/target/classes/spring/file-instance.xml b/smtweb-framework/canal/server/target/classes/spring/file-instance.xml
new file mode 100644
index 0000000..800f981
--- /dev/null
+++ b/smtweb-framework/canal/server/target/classes/spring/file-instance.xml
@@ -0,0 +1,197 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/server/target/classes/spring/group-instance.xml b/smtweb-framework/canal/server/target/classes/spring/group-instance.xml
new file mode 100644
index 0000000..b0b887f
--- /dev/null
+++ b/smtweb-framework/canal/server/target/classes/spring/group-instance.xml
@@ -0,0 +1,292 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/server/target/classes/spring/memory-instance.xml b/smtweb-framework/canal/server/target/classes/spring/memory-instance.xml
new file mode 100644
index 0000000..a7dc634
--- /dev/null
+++ b/smtweb-framework/canal/server/target/classes/spring/memory-instance.xml
@@ -0,0 +1,185 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/server/target/classes/spring/tsdb/h2-tsdb.xml b/smtweb-framework/canal/server/target/classes/spring/tsdb/h2-tsdb.xml
new file mode 100644
index 0000000..29c7a24
--- /dev/null
+++ b/smtweb-framework/canal/server/target/classes/spring/tsdb/h2-tsdb.xml
@@ -0,0 +1,61 @@
+
+
+
+
+
+
+
+
+
+ classpath:canal.properties
+ classpath:${canal.instance.destination:}/instance.properties
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/server/target/classes/spring/tsdb/mysql-tsdb.xml b/smtweb-framework/canal/server/target/classes/spring/tsdb/mysql-tsdb.xml
new file mode 100644
index 0000000..72d82bf
--- /dev/null
+++ b/smtweb-framework/canal/server/target/classes/spring/tsdb/mysql-tsdb.xml
@@ -0,0 +1,63 @@
+
+
+
+
+
+
+
+
+
+ classpath:canal.properties
+ classpath:${canal.instance.destination:}/instance.properties
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap-config.xml b/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap-config.xml
new file mode 100644
index 0000000..c27d463
--- /dev/null
+++ b/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap-config.xml
@@ -0,0 +1,14 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_history.xml b/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_history.xml
new file mode 100644
index 0000000..206c712
--- /dev/null
+++ b/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_history.xml
@@ -0,0 +1,44 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ insert into meta_history ()
+ values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#{destination},#{binlogFile},#{binlogOffest},#{binlogMasterId},#{binlogTimestamp},#{useSchema},#{sqlSchema},#{sqlTable},#{sqlText},#{sqlType},#{extra})
+
+
+
+ delete from meta_history
+ where destination=#{destination}
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_snapshot.xml b/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_snapshot.xml
new file mode 100644
index 0000000..a4d1466
--- /dev/null
+++ b/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_snapshot.xml
@@ -0,0 +1,48 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ insert into meta_snapshot ()
+ values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#{destination},#{binlogFile},#{binlogOffest},#{binlogMasterId},#{binlogTimestamp},#{data},#{extra})
+
+
+
+ update meta_snapshot set gmt_modified=now(),
+ binlog_file=#{binlogFile},binlog_offest=#{binlogOffest},binlog_master_id=#{binlogMasterId},binlog_timestamp=#{binlogTimestamp},data=#{data},extra=#{extra}
+ where destination=#{destination} and binlog_timestamp=0
+
+
+
+ delete from meta_snapshot
+ where destination=#{destination}
+
+
+
+ 0
+ ]]>
+
+
\ No newline at end of file
diff --git a/smtweb-framework/canal/server/target/classes/spring/tsdb/sql/create_table.sql b/smtweb-framework/canal/server/target/classes/spring/tsdb/sql/create_table.sql
new file mode 100644
index 0000000..8cb82c8
--- /dev/null
+++ b/smtweb-framework/canal/server/target/classes/spring/tsdb/sql/create_table.sql
@@ -0,0 +1,39 @@
+CREATE TABLE IF NOT EXISTS `meta_snapshot` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',
+ `gmt_create` datetime NOT NULL COMMENT '创建时间',
+ `gmt_modified` datetime NOT NULL COMMENT '修改时间',
+ `destination` varchar(128) DEFAULT NULL COMMENT '通道名称',
+ `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名',
+ `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量',
+ `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id',
+ `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳',
+ `data` longtext DEFAULT NULL COMMENT '表结构数据',
+ `extra` text DEFAULT NULL COMMENT '额外的扩展信息',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`),
+ KEY `destination` (`destination`),
+ KEY `destination_timestamp` (`destination`,`binlog_timestamp`),
+ KEY `gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='表结构记录表快照表';
+
+CREATE TABLE IF NOT EXISTS `meta_history` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',
+ `gmt_create` datetime NOT NULL COMMENT '创建时间',
+ `gmt_modified` datetime NOT NULL COMMENT '修改时间',
+ `destination` varchar(128) DEFAULT NULL COMMENT '通道名称',
+ `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名',
+ `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量',
+ `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id',
+ `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳',
+ `use_schema` varchar(1024) DEFAULT NULL COMMENT '执行sql时对应的schema',
+ `sql_schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema',
+ `sql_table` varchar(1024) DEFAULT NULL COMMENT '对应的table',
+ `sql_text` longtext DEFAULT NULL COMMENT '执行的sql',
+ `sql_type` varchar(256) DEFAULT NULL COMMENT 'sql类型',
+ `extra` text DEFAULT NULL COMMENT '额外的扩展信息',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`),
+ KEY `destination` (`destination`),
+ KEY `destination_timestamp` (`destination`,`binlog_timestamp`),
+ KEY `gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='表结构变化明细表';
\ No newline at end of file
diff --git a/smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst b/smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst
new file mode 100644
index 0000000..a3c448f
--- /dev/null
+++ b/smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst
@@ -0,0 +1,22 @@
+cc\smtweb\system\canal\CanalController.class
+cc\smtweb\system\canal\CanalConstants.class
+cc\smtweb\system\canal\InstanceConfig.class
+cc\smtweb\system\canal\CanalController$1.class
+cc\smtweb\system\canal\monitor\SpringInstanceConfigMonitor.class
+cc\smtweb\system\canal\CanalLauncher.class
+cc\smtweb\system\canal\monitor\SpringInstanceConfigMonitor$FileInfo.class
+cc\smtweb\system\canal\client\CanalClient.class
+cc\smtweb\system\canal\CanalController$3.class
+cc\smtweb\system\canal\admin\CanalAdminController.class
+cc\smtweb\system\canal\CanalLauncher$1.class
+cc\smtweb\system\canal\monitor\InstanceConfigMonitor.class
+cc\smtweb\system\canal\CanalStarter.class
+cc\smtweb\system\canal\CanalApplication.class
+cc\smtweb\system\canal\CanalController$1$1.class
+cc\smtweb\system\canal\monitor\InstanceAction.class
+cc\smtweb\system\canal\client\CanalClient$1.class
+cc\smtweb\system\canal\CanalController$2.class
+cc\smtweb\system\canal\monitor\SpringInstanceConfigMonitor$InstanceConfigFiles.class
+cc\smtweb\system\canal\client\CanalVO.class
+cc\smtweb\system\canal\InstanceConfig$InstanceMode.class
+cc\smtweb\system\canal\monitor\ManagerInstanceConfigMonitor.class
diff --git a/smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst b/smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst
new file mode 100644
index 0000000..f87c00f
--- /dev/null
+++ b/smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst
@@ -0,0 +1,13 @@
+E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\CanalConstants.java
+E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\CanalStarter.java
+E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\InstanceConfig.java
+E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\CanalApplication.java
+E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\CanalLauncher.java
+E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\client\CanalClient.java
+E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\monitor\InstanceAction.java
+E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\monitor\InstanceConfigMonitor.java
+E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\CanalController.java
+E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\monitor\ManagerInstanceConfigMonitor.java
+E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\admin\CanalAdminController.java
+E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\monitor\SpringInstanceConfigMonitor.java
+E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\client\CanalVO.java
diff --git a/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadWorker.java b/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadWorker.java
index bf79f7e..8d84183 100644
--- a/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadWorker.java
+++ b/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadWorker.java
@@ -6,20 +6,14 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class SysThreadWorker implements Runnable {
- private String loginPartyId;
-
- private String loginOwnerPartyId;
private String bizName;
- protected SysThreadWorker(String loginPartyId, String loginOwnerPartyId) {
- this.loginPartyId = loginPartyId;
- this.loginOwnerPartyId = loginOwnerPartyId;
+ protected SysThreadWorker() {
+
}
- protected SysThreadWorker(String loginPartyId, String loginOwnerPartyId, String bizName) {
- this.loginPartyId = loginPartyId;
- this.loginOwnerPartyId = loginOwnerPartyId;
+ protected SysThreadWorker(String bizName) {
this.bizName = StringUtil.checkNull(bizName);
}
@@ -27,8 +21,6 @@ public abstract class SysThreadWorker implements Runnable {
@Override
public void run() {
try {
-// SysParams.setLoginOwnerPartyId(loginOwnerPartyId);
-// SysParams.setLoginPartyId(loginPartyId);
localWork();
} catch (Exception e) {
log.error("DfpPool handle error:", e);