From 8ab1bde422e0ff236becfb5572983b2951fe0ac5 Mon Sep 17 00:00:00 2001 From: yaoq Date: Wed, 14 Sep 2022 10:17:11 +0800 Subject: [PATCH] =?UTF-8?q?=E9=9B=86=E6=88=90Canal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- smtweb-framework/canal/client/pom.xml | 210 +++ .../smtweb/system/canal/client/ClientConsts.java | 76 ++ .../smtweb/system/canal/client/ClientInstance.java | 290 +++++ .../system/canal/client/ClientStartedListener.java | 32 + .../cc/smtweb/system/canal/client/ClientVO.java | 56 + .../client/src/main/resources/client.properties | 14 + .../canal/client/target/classes/client.properties | 14 + smtweb-framework/canal/server/pom.xml | 227 ++++ .../canal/server/src/main/assembly/dev.xml | 54 + .../canal/server/src/main/assembly/release.xml | 58 + .../canal/server/src/main/bin/restart.sh | 15 + .../canal/server/src/main/bin/startup.bat | 27 + .../canal/server/src/main/bin/startup.sh | 126 ++ smtweb-framework/canal/server/src/main/bin/stop.sh | 65 + .../system/canal/server/CanalApplication.java | 13 + .../smtweb/system/canal/server/CanalConstants.java | 97 ++ .../system/canal/server/CanalController.java | 606 +++++++++ .../smtweb/system/canal/server/CanalLauncher.java | 141 ++ .../system/canal/server/CanalStartedListener.java | 39 + .../smtweb/system/canal/server/CanalStarter.java | 169 +++ .../smtweb/system/canal/server/InstanceConfig.java | 93 ++ .../canal/server/admin/CanalAdminController.java | 256 ++++ .../canal/server/monitor/InstanceAction.java | 30 + .../server/monitor/InstanceConfigMonitor.java | 16 + .../monitor/ManagerInstanceConfigMonitor.java | 184 +++ .../monitor/SpringInstanceConfigMonitor.java | 297 +++++ .../server/src/main/resources/canal.properties | 182 +++ .../src/main/resources/canal_local.properties | 12 + .../src/main/resources/example/instance.properties | 59 + .../server/src/main/resources/logback.xml.bak | 117 ++ .../resources/metrics/Canal_instances_tmpl.json | 1375 ++++++++++++++++++++ .../src/main/resources/spring/base-instance.xml | 39 + .../src/main/resources/spring/default-instance.xml | 211 +++ .../src/main/resources/spring/file-instance.xml | 197 +++ .../src/main/resources/spring/group-instance.xml | 292 +++++ .../src/main/resources/spring/memory-instance.xml | 185 +++ .../src/main/resources/spring/tsdb/h2-tsdb.xml | 61 + .../src/main/resources/spring/tsdb/mysql-tsdb.xml | 63 + .../spring/tsdb/sql-map/sqlmap-config.xml | 14 + .../spring/tsdb/sql-map/sqlmap_history.xml | 44 + .../spring/tsdb/sql-map/sqlmap_snapshot.xml | 48 + .../resources/spring/tsdb/sql/create_table.sql | 39 + .../canal/server/target/classes/canal.properties | 182 +++ .../server/target/classes/canal_local.properties | 12 + .../target/classes/example/instance.properties | 59 + .../canal/server/target/classes/logback.xml.bak | 117 ++ .../classes/metrics/Canal_instances_tmpl.json | 1375 ++++++++++++++++++++ .../server/target/classes/spring/base-instance.xml | 39 + .../target/classes/spring/default-instance.xml | 211 +++ .../server/target/classes/spring/file-instance.xml | 197 +++ .../target/classes/spring/group-instance.xml | 292 +++++ .../target/classes/spring/memory-instance.xml | 185 +++ .../server/target/classes/spring/tsdb/h2-tsdb.xml | 61 + .../target/classes/spring/tsdb/mysql-tsdb.xml | 63 + .../classes/spring/tsdb/sql-map/sqlmap-config.xml | 14 + .../classes/spring/tsdb/sql-map/sqlmap_history.xml | 44 + .../spring/tsdb/sql-map/sqlmap_snapshot.xml | 48 + .../classes/spring/tsdb/sql/create_table.sql | 39 + .../compile/default-compile/createdFiles.lst | 22 + .../compile/default-compile/inputFiles.lst | 13 + .../framework/core/systask/SysThreadWorker.java | 14 +- 61 files changed, 9119 insertions(+), 11 deletions(-) create mode 100644 smtweb-framework/canal/client/pom.xml create mode 100644 smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientConsts.java create mode 100644 smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientInstance.java create mode 100644 smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientStartedListener.java create mode 100644 smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientVO.java create mode 100644 smtweb-framework/canal/client/src/main/resources/client.properties create mode 100644 smtweb-framework/canal/client/target/classes/client.properties create mode 100644 smtweb-framework/canal/server/pom.xml create mode 100644 smtweb-framework/canal/server/src/main/assembly/dev.xml create mode 100644 smtweb-framework/canal/server/src/main/assembly/release.xml create mode 100644 smtweb-framework/canal/server/src/main/bin/restart.sh create mode 100644 smtweb-framework/canal/server/src/main/bin/startup.bat create mode 100644 smtweb-framework/canal/server/src/main/bin/startup.sh create mode 100644 smtweb-framework/canal/server/src/main/bin/stop.sh create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalApplication.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalConstants.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalController.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalLauncher.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStartedListener.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStarter.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/InstanceConfig.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/admin/CanalAdminController.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceAction.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceConfigMonitor.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/ManagerInstanceConfigMonitor.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/SpringInstanceConfigMonitor.java create mode 100644 smtweb-framework/canal/server/src/main/resources/canal.properties create mode 100644 smtweb-framework/canal/server/src/main/resources/canal_local.properties create mode 100644 smtweb-framework/canal/server/src/main/resources/example/instance.properties create mode 100644 smtweb-framework/canal/server/src/main/resources/logback.xml.bak create mode 100644 smtweb-framework/canal/server/src/main/resources/metrics/Canal_instances_tmpl.json create mode 100644 smtweb-framework/canal/server/src/main/resources/spring/base-instance.xml create mode 100644 smtweb-framework/canal/server/src/main/resources/spring/default-instance.xml create mode 100644 smtweb-framework/canal/server/src/main/resources/spring/file-instance.xml create mode 100644 smtweb-framework/canal/server/src/main/resources/spring/group-instance.xml create mode 100644 smtweb-framework/canal/server/src/main/resources/spring/memory-instance.xml create mode 100644 smtweb-framework/canal/server/src/main/resources/spring/tsdb/h2-tsdb.xml create mode 100644 smtweb-framework/canal/server/src/main/resources/spring/tsdb/mysql-tsdb.xml create mode 100644 smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap-config.xml create mode 100644 smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml create mode 100644 smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml create mode 100644 smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql/create_table.sql create mode 100644 smtweb-framework/canal/server/target/classes/canal.properties create mode 100644 smtweb-framework/canal/server/target/classes/canal_local.properties create mode 100644 smtweb-framework/canal/server/target/classes/example/instance.properties create mode 100644 smtweb-framework/canal/server/target/classes/logback.xml.bak create mode 100644 smtweb-framework/canal/server/target/classes/metrics/Canal_instances_tmpl.json create mode 100644 smtweb-framework/canal/server/target/classes/spring/base-instance.xml create mode 100644 smtweb-framework/canal/server/target/classes/spring/default-instance.xml create mode 100644 smtweb-framework/canal/server/target/classes/spring/file-instance.xml create mode 100644 smtweb-framework/canal/server/target/classes/spring/group-instance.xml create mode 100644 smtweb-framework/canal/server/target/classes/spring/memory-instance.xml create mode 100644 smtweb-framework/canal/server/target/classes/spring/tsdb/h2-tsdb.xml create mode 100644 smtweb-framework/canal/server/target/classes/spring/tsdb/mysql-tsdb.xml create mode 100644 smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap-config.xml create mode 100644 smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_history.xml create mode 100644 smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_snapshot.xml create mode 100644 smtweb-framework/canal/server/target/classes/spring/tsdb/sql/create_table.sql create mode 100644 smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst create mode 100644 smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst diff --git a/smtweb-framework/canal/client/pom.xml b/smtweb-framework/canal/client/pom.xml new file mode 100644 index 0000000..09c9462 --- /dev/null +++ b/smtweb-framework/canal/client/pom.xml @@ -0,0 +1,210 @@ + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 2.6.9 + + + + cc.smtweb + canal.client + 1.1.6 + + + + com.alibaba.otter + canal.client + 1.1.6 + + + com.alibaba.otter + canal.protocol + 1.1.6 + + + com.alibaba + druid + 1.2.11 + + + + cc.smtweb + sw-system-bpm + 3.1.0-SNAPSHOT + + + + mysql + mysql-connector-java + + + org.apache.ddlutils + ddlutils + 1.0 + + + commons-beanutils + commons-beanutils-core + + + commons-lang + commons-lang + + + commons-dbcp + commons-dbcp + + + commons-pool + commons-pool + + + commons-logging + commons-logging-api + + + dom4j + dom4j + + + stax + stax-api + + + commons-collections + commons-collections + + + commons-digester + commons-digester + + + commons-betwixt + commons-betwixt + + + + + org.apache.commons + commons-pool2 + 2.5.0 + + + commons-beanutils + commons-beanutils + 1.8.2 + + + org.apache.commons + commons-lang3 + 3.7 + + + commons-collections + commons-collections + 3.2 + + + + + junit + junit + test + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.5 + + + true + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + 2.2.1 + + + assemble + + single + + package + + + + false + false + + + + + + + + dev + + true + + env + !release + + + + + + + maven-assembly-plugin + + + + ${basedir}/src/main/assembly/dev.xml + + canal-example + ${project.build.directory} + + + + + + + + + release + + + env + release + + + + + + + maven-assembly-plugin + + + + ${basedir}/src/main/assembly/release.xml + + + ${project.artifactId}-1.1.6 + + ${project.parent.build.directory} + + + + + + + diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientConsts.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientConsts.java new file mode 100644 index 0000000..c238ae9 --- /dev/null +++ b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientConsts.java @@ -0,0 +1,76 @@ +package cc.smtweb.system.canal.client; + +import org.apache.commons.lang.StringUtils; + +import java.util.Properties; + +/** + * 启动常用变量 + * + * @author jianghang 2012-11-8 下午03:15:55 + * @version 1.0.0 + */ +public class ClientConsts { + + public static final String ROOT = "canal.client"; + public static final String SIZE = ROOT + "." + "size"; + public static final String INSTANCE = ROOT + "." + "instance"; + public static final String IP = ROOT + "." + "ip"; + public static final String PORT = ROOT + "." + "port"; + public static final String USERNAME = ROOT + "." + "username"; + public static final String PASSWORD = ROOT + "." + "password"; + public static final String FILTER = ROOT + "." + "filter"; + + + public static String getIp(Properties properties){ + return getProperty(properties,IP,"127.0.0.1"); + } + + public static int getPort(Properties properties){ + return Integer.parseInt(getProperty(properties,PORT,"11111")); + } + + public static int getBatchSize(Properties properties){ + return Integer.parseInt(getProperty(properties,SIZE,"50")); + } + + public static String getInstance(Properties properties){ + return getProperty(properties,INSTANCE,"example"); + } + + public static String getUsername(Properties properties){ + return getProperty(properties,USERNAME,""); + } + + public static String getPassword(Properties properties){ + return getProperty(properties,PASSWORD,""); + } + + public static String getFilter(Properties properties){ + return getProperty(properties,FILTER,".*\\..*"); + } + + public static String getProperty(Properties properties, String key, String defaultValue) { + String value = getProperty(properties, key); + if (StringUtils.isEmpty(value)) { + return defaultValue; + } else { + return value; + } + } + + public static String getProperty(Properties properties, String key) { + key = StringUtils.trim(key); + String value = System.getProperty(key); + + if (value == null) { + value = System.getenv(key); + } + + if (value == null) { + value = properties.getProperty(key); + } + + return StringUtils.trim(value); + } +} diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientInstance.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientInstance.java new file mode 100644 index 0000000..a4415d1 --- /dev/null +++ b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientInstance.java @@ -0,0 +1,290 @@ +package cc.smtweb.system.canal.client; + +import cc.smtweb.framework.core.util.CommUtil; +import cc.smtweb.framework.core.util.JsonUtil; +import com.alibaba.otter.canal.client.CanalConnector; +import com.alibaba.otter.canal.client.CanalConnectors; +import com.alibaba.otter.canal.protocol.CanalEntry; +import com.alibaba.otter.canal.protocol.Message; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.Assert; + +import java.io.FileInputStream; +import java.net.InetSocketAddress; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @Author yaoq + * @Date 2022年09月13日 18:02 + * @Description + */ +public class ClientInstance { + private static final String CLASSPATH_URL_PREFIX = "classpath:"; + private static final Logger logger = LoggerFactory.getLogger(ClientInstance.class); + protected volatile boolean running = false; + protected Thread thread = null; + protected CanalConnector connector; + protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { + + public void uncaughtException(Thread t, Throwable e) { + logger.error("解析出错了哦", e); + + } + }; + + private static ClientInstance instance = null; + private static Properties properties = null; + + static { + instance = new ClientInstance(); + } + + public static ClientInstance getInstance() { + return instance; + } + + private void loadProperties() { + try { + String conf = System.getProperty("client.conf", "classpath:client.properties"); + properties = new Properties(); + if (conf.startsWith(CLASSPATH_URL_PREFIX)) { + conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX); + properties.load(ClientInstance.class.getClassLoader().getResourceAsStream(conf)); + } else { + properties.load(new FileInputStream(conf)); + } + } catch (Exception e) { + logger.error("加载canal客户端配置参数失败！", e); + } + } + + private ClientInstance() { + loadProperties(); + connector = CanalConnectors.newSingleConnector(new + InetSocketAddress(ClientConsts.getIp(properties), + ClientConsts.getPort(properties)), ClientConsts.getInstance(properties), ClientConsts.getUsername(properties), ClientConsts.getPassword(properties)); + connector.connect(); + connector.subscribe(ClientConsts.getFilter(properties)); + connector.rollback(); + + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + logger.info("## stop the canal client"); + ClientInstance.this.stop(); + } catch (Throwable e) { + logger.warn("##something goes wrong when stopping canal:", e); + } finally { + logger.info("## canal client is down."); + } + })); + } + + //开始 + public void start() { + Assert.notNull(connector, "connector is null"); + thread = new Thread(() -> process()); + + thread.setUncaughtExceptionHandler(handler); + running = true; + thread.start(); + } + + //停止 + public void stop() { + if (!running) { + return; + } + running = false; + if (thread != null) { + try { + thread.join(); + } catch (InterruptedException e) { + // ignore + } + } + } + + protected void process() { + while (running) { + try { + connector.disconnect(); + connector.connect(); + connector.unsubscribe(); + connector.subscribe(ClientConsts.getFilter(properties)); + while (running) { + Message message = connector.getWithoutAck(ClientConsts.getBatchSize(properties) * 1024); // 获取指定数量的数据 + long batchId = message.getId(); + int size = message.getEntries().size(); + if (batchId == -1 || size == 0) { + Thread.sleep(3000L);//无数据时，睡眠。 + continue; + } else { + dataHandle(message.getEntries()); + } + if (batchId != -1) { + connector.ack(batchId); // 提交确认 + } + } + } catch (Throwable e) { + logger.error("process error!", e); + try { + Thread.sleep(1000L); + } catch (InterruptedException e1) { + // ignore + } + if (!isReConnection()) { + connector.rollback(); // 处理失败, 回滚数据 + } + } finally { + connector.disconnect(); + } + } + } + + private boolean isReConnection() { + if (!connector.checkValid()) return false; + int count = 1; + int rate = 1; + while (connector.checkValid()) { + rate = (count + 100) / 100; + try { + logger.debug("canal服务连接中断，第【" + count + "】次重连开始"); + connector.disconnect(); + connector.connect(); + logger.debug("canal服务连接中断，第【" + count + "】次重连成功"); + break; + } catch (Throwable e2) { + if (count > 99) { + logger.error("canal服务连接异常！", "重新连接canal服务端已失败【" + count + "】次，请检查canal服务是否启动，请及时启动canal服务端，或联系研发人员！"); + } + logger.debug("canal服务连接中断，第【" + count + "】次重连失败：" + CommUtil.getOrigMsg(e2)); + logger.debug("canal服务连接中断，" + (3 * rate) + "分钟后进行第【" + (count + 1) + "】连接测试！"); + try { + Thread.sleep(180 * rate * 1000L); + } catch (InterruptedException e1) { + // ignore + } + } + count++; + } + return true; + } + + /** + * 数据处理 + * + * @param entrys + */ + private void dataHandle(List entrys) throws Exception { + for (CanalEntry.Entry entry : entrys) { + if (CanalEntry.EntryType.ROWDATA == entry.getEntryType()) { + CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); + CanalEntry.EventType eventType = rowChange.getEventType(); + if (eventType == CanalEntry.EventType.DELETE) { + saveDeleteSql(entry); + } else if (eventType == CanalEntry.EventType.UPDATE) { + saveUpdateSql(entry); + } else if (eventType == CanalEntry.EventType.INSERT) { + saveInsertSql(entry); + } + } + } + } + + /** + * 保存更新语句 + * + * @param entry + */ + private void saveUpdateSql(CanalEntry.Entry entry) { + try { + CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); + List rowDatasList = rowChange.getRowDatasList(); + for (CanalEntry.RowData rowData : rowDatasList) { + List newColumnList = rowData.getAfterColumnsList(); + Map data = new HashMap<>(); + newColumnList.stream().forEach(item -> { + if (StringUtils.isNotEmpty(item.getValue())) { + data.put(lineToHump(item.getName()), item.getValue()); + } + }); + List oldColumnList = rowData.getBeforeColumnsList(); + ClientVO update = ClientVO.ok(entry.getHeader().getTableName(), "UPDATE", data, oldColumnList.get(0).getValue()); + System.out.println("更新返回 ： " + JsonUtil.encodeString(update)); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 保存删除语句 + * + * @param entry + */ + private void saveDeleteSql(CanalEntry.Entry entry) { + try { + CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); + List rowDatasList = rowChange.getRowDatasList(); + for (CanalEntry.RowData rowData : rowDatasList) { + List oldColumnList = rowData.getBeforeColumnsList(); + ClientVO delete = ClientVO.ok(entry.getHeader().getTableName(), "DELETE", null, oldColumnList.get(0).getValue()); + System.out.println("删除返回 ： " + JsonUtil.encodeString(delete)); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 保存插入语句 + * + * @param entry + */ + private void saveInsertSql(CanalEntry.Entry entry) { + try { + CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); + List rowDatasList = rowChange.getRowDatasList(); + for (CanalEntry.RowData rowData : rowDatasList) { + List columnList = rowData.getAfterColumnsList(); + Map data = new HashMap<>(); + columnList.stream().forEach(item -> { + if (StringUtils.isNotEmpty(item.getValue())) { + data.put(lineToHump(item.getName()), item.getValue()); + } + }); + ClientVO insert = ClientVO.ok(entry.getHeader().getTableName(), "INSERT", data, null); + System.out.println("插入返回 ： " + JsonUtil.encodeString(insert)); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 下划线转驼峰 + * + * @param str + */ + private static Pattern linePattern = Pattern.compile("_(\\w)"); + + public static String lineToHump(String str) { + str = str.toLowerCase(); + Matcher matcher = linePattern.matcher(str); + StringBuffer sb = new StringBuffer(); + while (matcher.find()) { + matcher.appendReplacement(sb, matcher.group(1).toUpperCase()); + } + matcher.appendTail(sb); + return sb.toString(); + } +} + + diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientStartedListener.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientStartedListener.java new file mode 100644 index 0000000..5dad1df --- /dev/null +++ b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientStartedListener.java @@ -0,0 +1,32 @@ +package cc.smtweb.system.canal.client; + +import cc.smtweb.framework.core.annotation.SwStartListener; +import cc.smtweb.framework.core.common.SwConsts; +import cc.smtweb.framework.core.mvc.controller.IStartListener; +import lombok.extern.slf4j.Slf4j; + +/** + * @Author yaoq + * @Date 2022年09月06日 10:31 + * @Description + */ +@Slf4j +@SwStartListener +public class ClientStartedListener implements IStartListener { + + @Override + public int order() { + return SwConsts.DEFAULT_ORDER + 3; + } + + + @Override + public void init() { + + } + + @Override + public void run() { + ClientInstance.getInstance().start(); + } +} diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientVO.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientVO.java new file mode 100644 index 0000000..02cbb46 --- /dev/null +++ b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientVO.java @@ -0,0 +1,56 @@ +package cc.smtweb.system.canal.client; + +import java.util.Map; + +/** + * @Author yaoq + * @Date 2022年09月13日 18:20 + * @Description + */ +public class ClientVO { + private String tableName; // 表名 + private String type; // 类型（更新、删除、插入） + private Map data; // 数据JSON 自己转对应表格实体类 + private String id; // 更新或删除都是根据ID来 + + public static ClientVO ok(String tableName, String type, Map data, String id){ + ClientVO canalVO=new ClientVO(); + canalVO.setId(id); + canalVO.setTableName(tableName); + canalVO.setType(type); + canalVO.setData(data); + return canalVO; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Map getData() { + return data; + } + + public void setData(Map data) { + this.data = data; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } +} diff --git a/smtweb-framework/canal/client/src/main/resources/client.properties b/smtweb-framework/canal/client/src/main/resources/client.properties new file mode 100644 index 0000000..6314051 --- /dev/null +++ b/smtweb-framework/canal/client/src/main/resources/client.properties @@ -0,0 +1,14 @@ +#canal ??????? +# ??????? ??MB +canal.server.size=50 +# ?????? +canal.server.instance=example +canal.server.ip=127.0.0.1 +canal.server.port=11111 +# ????????? ?????? +canal.server.username= +canal.server.password= +# ??????????.*\\..*??????scmz\\..* +canal.server.filter=.*\\..* + + diff --git a/smtweb-framework/canal/client/target/classes/client.properties b/smtweb-framework/canal/client/target/classes/client.properties new file mode 100644 index 0000000..6314051 --- /dev/null +++ b/smtweb-framework/canal/client/target/classes/client.properties @@ -0,0 +1,14 @@ +#canal ??????? +# ??????? ??MB +canal.server.size=50 +# ?????? +canal.server.instance=example +canal.server.ip=127.0.0.1 +canal.server.port=11111 +# ????????? ?????? +canal.server.username= +canal.server.password= +# ??????????.*\\..*??????scmz\\..* +canal.server.filter=.*\\..* + + diff --git a/smtweb-framework/canal/server/pom.xml b/smtweb-framework/canal/server/pom.xml new file mode 100644 index 0000000..89b0415 --- /dev/null +++ b/smtweb-framework/canal/server/pom.xml @@ -0,0 +1,227 @@ + + 4.0.0 + + + org.springframework.boot + spring-boot-starter-parent + 2.6.9 + + + + cc.smtweb + canal.server + 1.1.6 + + + + + com.alibaba.otter + canal.protocol + 1.1.6 + + + com.alibaba.otter + canal.client + 1.1.6 + + + + com.alibaba.otter + canal.server + 1.1.6 + + + + + com.alibaba.otter + canal.prometheus + 1.1.6 + runtime + + + + com.alibaba.otter + connector.kafka + 1.1.6 + + + * + * + + + jar-with-dependencies + provided + + + com.alibaba.otter + connector.rocketmq + 1.1.6 + + + * + * + + + jar-with-dependencies + provided + + + com.alibaba.otter + connector.rabbitmq + 1.1.6 + + + * + * + + + jar-with-dependencies + provided + + + com.alibaba.otter + connector.pulsarmq + 1.1.6 + + + * + * + + + jar-with-dependencies + provided + + + org.springframework.boot + spring-boot-autoconfigure + + + cc.smtweb + sw-framework-core + 3.1.0-SNAPSHOT + compile + + + + cc.smtweb + sw-system-bpm + 3.1.0-SNAPSHOT + compile + + + + + + + maven-jar-plugin + + + true + + + **/logback.xml + **/canal.properties + **/spring/** + **/example/** + **/mq.yml + + + + + org.apache.maven.plugins + maven-dependency-plugin + 2.10 + + + copy-dependencies-to-canal-deployer + package + + copy-dependencies + + + jar-with-dependencies + ${project.basedir}/target/canal/plugin + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + 2.2.1 + + + assemble + + single + + package + + + + false + false + + + + + + + + dev + + true + + env + !release + + + + + + + maven-assembly-plugin + + + + ${basedir}/src/main/assembly/dev.xml + + canal + ${project.build.directory} + + + + + + + + + release + + + env + release + + + + + + + maven-assembly-plugin + + + + ${basedir}/src/main/assembly/release.xml + + + ${project.artifactId}-1.1.6 + + ${project.parent.build.directory} + + + + + + + diff --git a/smtweb-framework/canal/server/src/main/assembly/dev.xml b/smtweb-framework/canal/server/src/main/assembly/dev.xml new file mode 100644 index 0000000..3c66c4c --- /dev/null +++ b/smtweb-framework/canal/server/src/main/assembly/dev.xml @@ -0,0 +1,54 @@ + + dist + + dir + + false + + + . + / + + README* + + + + ./src/main/bin + bin + + **/* + + 0755 + + + ./src/main/conf + /conf + + **/* + + + + ./src/main/resources + /conf + + **/* + + + + target + logs + + **/* + + + + + + lib + + junit:junit + + + + diff --git a/smtweb-framework/canal/server/src/main/assembly/release.xml b/smtweb-framework/canal/server/src/main/assembly/release.xml new file mode 100644 index 0000000..601328e --- /dev/null +++ b/smtweb-framework/canal/server/src/main/assembly/release.xml @@ -0,0 +1,58 @@ + + dist + + tar.gz + + false + + + . + / + + README* + + + + ./src/main/bin + bin + + **/* + + 0755 + + + ./src/main/conf + /conf + + **/* + + + + ./src/main/resources + /conf + + **/* + + + + target + logs + + **/* + + + + ${project.basedir}/target/canal/plugin + /plugin/ + + + + + lib + + junit:junit + + + + diff --git a/smtweb-framework/canal/server/src/main/bin/restart.sh b/smtweb-framework/canal/server/src/main/bin/restart.sh new file mode 100644 index 0000000..3623edd --- /dev/null +++ b/smtweb-framework/canal/server/src/main/bin/restart.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +args=$@ + +case $(uname) in +Linux) + bin_abs_path=$(readlink -f $(dirname $0)) + ;; +*) + bin_abs_path=$(cd $(dirname $0) ||exit ; pwd) + ;; +esac + +sh "$bin_abs_path"/stop.sh $args +sh "$bin_abs_path"/startup.sh $args diff --git a/smtweb-framework/canal/server/src/main/bin/startup.bat b/smtweb-framework/canal/server/src/main/bin/startup.bat new file mode 100644 index 0000000..0cad8cb --- /dev/null +++ b/smtweb-framework/canal/server/src/main/bin/startup.bat @@ -0,0 +1,27 @@ +@echo off +@if not "%ECHO%" == "" echo %ECHO% +@if "%OS%" == "Windows_NT" setlocal + +set ENV_PATH=.\ +if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0% + +set conf_dir=%ENV_PATH%\..\conf +set canal_conf=%conf_dir%\canal.properties +@rem set canal_conf=%conf_dir%\canal_local.properties +if "%1" == "local" set canal_conf=%conf_dir%\canal_local.properties +set logback_configurationFile=%conf_dir%\logback.xml + +set CLASSPATH=%conf_dir% +set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH% + +set JAVA_MEM_OPTS= -Xms128m -Xmx512m -XX:PermSize=128m +set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8 +set JAVA_DEBUG_OPT= -server -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=9099,server=y,suspend=n +set CANAL_OPTS= -DappName=otter-canal -Dlogback.configurationFile="%logback_configurationFile%" -Dcanal.conf="%canal_conf%" + +set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS% + +set CMD_STR= java %JAVA_OPTS% -classpath "%CLASSPATH%" java %JAVA_OPTS% -classpath "%CLASSPATH%" CanalLauncher +echo start cmd : %CMD_STR% + +java %JAVA_OPTS% -classpath "%CLASSPATH%" CanalLauncher diff --git a/smtweb-framework/canal/server/src/main/bin/startup.sh b/smtweb-framework/canal/server/src/main/bin/startup.sh new file mode 100644 index 0000000..b8c0ed3 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/bin/startup.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +current_path=`pwd` +case "`uname`" in + Linux) + bin_abs_path=$(readlink -f $(dirname $0)) + ;; + *) + bin_abs_path=`cd $(dirname $0); pwd` + ;; +esac +base=${bin_abs_path}/.. +canal_conf=$base/conf/canal.properties +canal_local_conf=$base/conf/canal_local.properties +logback_configurationFile=$base/conf/logback.xml +export LANG=en_US.UTF-8 +export BASE=$base + +if [ -f $base/bin/canal.pid ] ; then + echo "found canal.pid , Please run stop.sh first ,then startup.sh" 2>&2 + exit 1 +fi + +if [ ! -d $base/logs/canal ] ; then + mkdir -p $base/logs/canal +fi + +## set java path +if [ -z "$JAVA" ] ; then + JAVA=$(which java) +fi + +ALIBABA_JAVA="/usr/alibaba/java/bin/java" +TAOBAO_JAVA="/opt/taobao/java/bin/java" +if [ -z "$JAVA" ]; then + if [ -f $ALIBABA_JAVA ] ; then + JAVA=$ALIBABA_JAVA + elif [ -f $TAOBAO_JAVA ] ; then + JAVA=$TAOBAO_JAVA + else + echo "Cannot find a Java JDK. Please set either set JAVA or put java (>=1.5) in your PATH." 2>&2 + exit 1 + fi +fi + +case "$#" +in +0 ) + ;; +1 ) + var=$* + if [ "$var" = "local" ]; then + canal_conf=$canal_local_conf + else + if [ -f $var ] ; then + canal_conf=$var + else + echo "THE PARAMETER IS NOT CORRECT.PLEASE CHECK AGAIN." + exit + fi + fi;; +2 ) + var=$1 + if [ "$var" = "local" ]; then + canal_conf=$canal_local_conf + else + if [ -f $var ] ; then + canal_conf=$var + else + if [ "$1" = "debug" ]; then + DEBUG_PORT=$2 + DEBUG_SUSPEND="n" + JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND" + fi + fi + fi;; +* ) + echo "THE PARAMETERS MUST BE TWO OR LESS.PLEASE CHECK AGAIN." + exit;; +esac + +JavaVersion=`$JAVA -version 2>&1 |awk 'NR==1{ gsub(/"/,""); print $3 }' | awk -F '.' '{print $1}'` +str=`file -L $JAVA | grep 64-bit` +JAVA_OPTS="$JAVA_OPTS -Xss256k -XX:+AggressiveOpts -XX:-UseBiasedLocking -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$base/logs" + +if [ $JavaVersion -ge 11 ] ; then + #JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$base_log/gc.log:time " + JAVA_OPTS="$JAVA_OPTS" +else + #JAVA_OPTS="$JAVA_OPTS -Xloggc:$base/logs/canal/gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime" + JAVA_OPTS="$JAVA_OPTS -XX:+UseFastAccessorMethods -XX:+PrintAdaptiveSizePolicy -XX:+PrintTenuringDistribution" +fi + +if [ -n "$str" ]; then + # JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS" + # For G1 + JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS" +else + JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m $JAVA_OPTS" +fi + +JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8" +CANAL_OPTS="-DappName=otter-canal -Dlogback.configurationFile=$logback_configurationFile -Dcanal.conf=$canal_conf" + +if [ -e $canal_conf -a -e $logback_configurationFile ] +then + + for i in $base/lib/*; + do CLASSPATH=$i:"$CLASSPATH"; + done + CLASSPATH="$base/conf:$CLASSPATH"; + + echo "cd to $bin_abs_path for workaround relative path" + cd $bin_abs_path + + echo LOG CONFIGURATION : $logback_configurationFile + echo canal conf : $canal_conf + echo CLASSPATH :$CLASSPATH + $JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.deployer.CanalLauncher 1>>$base/logs/canal/canal_stdout.log 2>&1 & + echo $! > $base/bin/canal.pid + + echo "cd to $current_path for continue" + cd $current_path +else + echo "canal conf("$canal_conf") OR log configration file($logback_configurationFile) is not exist,please create then first!" +fi diff --git a/smtweb-framework/canal/server/src/main/bin/stop.sh b/smtweb-framework/canal/server/src/main/bin/stop.sh new file mode 100644 index 0000000..f398749 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/bin/stop.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +cygwin=false; +linux=false; +case "`uname`" in + CYGWIN*) + cygwin=true + ;; + Linux*) + linux=true + ;; +esac + +get_pid() { + STR=$1 + PID=$2 + if $cygwin; then + JAVA_CMD="$JAVA_HOME\bin\java" + JAVA_CMD=`cygpath --path --unix $JAVA_CMD` + JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'` + else + if $linux; then + if [ ! -z "$PID" ]; then + JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` + else + JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'` + fi + else + if [ ! -z "$PID" ]; then + JAVA_PID=`ps aux |grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` + else + JAVA_PID=`ps aux |grep "$STR"|grep -v grep|awk '{print $2}'` + fi + fi + fi + echo $JAVA_PID; +} + +base=`dirname $0`/.. +pidfile=$base/bin/canal.pid +if [ ! -f "$pidfile" ];then + echo "canal is not running. exists" + exit +fi + +pid=`cat $pidfile` +if [ "$pid" == "" ] ; then + pid=`get_pid "appName=otter-canal"` +fi + +echo -e "`hostname`: stopping canal $pid ... " +kill $pid + +LOOPS=0 +while (true); +do + gpid=`get_pid "appName=otter-canal" "$pid"` + if [ "$gpid" == "" ] ; then + echo "Oook! cost:$LOOPS" + `rm $pidfile` + break; + fi + let LOOPS=LOOPS+1 + sleep 1 +done \ No newline at end of file diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalApplication.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalApplication.java new file mode 100644 index 0000000..7a644ac --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalApplication.java @@ -0,0 +1,13 @@ +package cc.smtweb.system.canal.server; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class CanalApplication { + + public static void main(String[] args) { + SpringApplication.run(CanalApplication.class, args); + } + +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalConstants.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalConstants.java new file mode 100644 index 0000000..d9a6a48 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalConstants.java @@ -0,0 +1,97 @@ +package cc.smtweb.system.canal.server; + +import java.text.MessageFormat; + +/** + * 启动常用变量 + * + * @author jianghang 2012-11-8 下午03:15:55 + * @version 1.0.0 + */ +public class CanalConstants { + + public static final String MDC_DESTINATION = "destination"; + public static final String ROOT = "canal"; + public static final String CANAL_ID = ROOT + "." + "id"; + public static final String CANAL_IP = ROOT + "." + "ip"; + public static final String CANAL_REGISTER_IP = ROOT + "." + "register.ip"; + public static final String CANAL_PORT = ROOT + "." + "port"; + public static final String CANAL_USER = ROOT + "." + "user"; + public static final String CANAL_PASSWD = ROOT + "." + "passwd"; + public static final String CANAL_METRICS_PULL_PORT = ROOT + "." + "metrics.pull.port"; + public static final String CANAL_ADMIN_MANAGER = ROOT + "." + "admin.manager"; + public static final String CANAL_ADMIN_PORT = ROOT + "." + "admin.port"; + public static final String CANAL_ADMIN_USER = ROOT + "." + "admin.user"; + public static final String CANAL_ADMIN_PASSWD = ROOT + "." + "admin.passwd"; + public static final String CANAL_ADMIN_AUTO_REGISTER = ROOT + "." + "admin.register.auto"; + public static final String CANAL_ADMIN_AUTO_CLUSTER = ROOT + "." + "admin.register.cluster"; + public static final String CANAL_ADMIN_REGISTER_NAME = ROOT + "." + "admin.register.name"; + public static final String CANAL_ZKSERVERS = ROOT + "." + "zkServers"; + public static final String CANAL_WITHOUT_NETTY = ROOT + "." + "withoutNetty"; + + public static final String CANAL_DESTINATIONS = ROOT + "." + "destinations"; + public static final String CANAL_AUTO_SCAN = ROOT + "." + "auto.scan"; + public static final String CANAL_AUTO_SCAN_INTERVAL = ROOT + "." + "auto.scan.interval"; + public static final String CANAL_CONF_DIR = ROOT + "." + "conf.dir"; + public static final String CANAL_SERVER_MODE = ROOT + "." + "serverMode"; + + public static final String CANAL_DESTINATION_SPLIT = ","; + public static final String GLOBAL_NAME = "global"; + + public static final String INSTANCE_MODE_TEMPLATE = ROOT + "." + "instance.{0}.mode"; + public static final String INSTANCE_LAZY_TEMPLATE = ROOT + "." + "instance.{0}.lazy"; + public static final String INSTANCE_MANAGER_ADDRESS_TEMPLATE = ROOT + "." + "instance.{0}.manager.address"; + public static final String INSTANCE_SPRING_XML_TEMPLATE = ROOT + "." + "instance.{0}.spring.xml"; + + public static final String CANAL_DESTINATION_PROPERTY = ROOT + ".instance.destination"; + + public static final String CANAL_SOCKETCHANNEL = ROOT + "." + "socketChannel"; + + public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey"; + public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey"; + +// public static final String CANAL_MQ_SERVERS = ROOT + "." + "mq.servers"; +// public static final String CANAL_MQ_RETRIES = ROOT + "." + "mq.retries"; +// public static final String CANAL_MQ_BATCHSIZE = ROOT + "." + "mq.batchSize"; +// public static final String CANAL_MQ_LINGERMS = ROOT + "." + "mq.lingerMs"; +// public static final String CANAL_MQ_MAXREQUESTSIZE = ROOT + "." + "mq.maxRequestSize"; +// public static final String CANAL_MQ_BUFFERMEMORY = ROOT + "." + "mq.bufferMemory"; +// public static final String CANAL_MQ_CANALBATCHSIZE = ROOT + "." + "mq.canalBatchSize"; +// public static final String CANAL_MQ_CANALGETTIMEOUT = ROOT + "." + "mq.canalGetTimeout"; +// public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage"; +// public static final String CANAL_MQ_PARALLELTHREADSIZE = ROOT + "." + "mq.parallelThreadSize"; +// public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType"; +// public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks"; +// public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction"; +// public static final String CANAL_MQ_PRODUCERGROUP = ROOT + "." + "mq.producerGroup"; +// public static final String CANAL_MQ_PROPERTIES = ROOT + "." + "mq.properties"; +// public static final String CANAL_MQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "mq.enableMessageTrace"; +// public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.accessChannel"; +// public static final String CANAL_MQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "mq.customizedTraceTopic"; +// public static final String CANAL_MQ_NAMESPACE = ROOT + "." + "mq.namespace"; +// public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable"; +// public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH = ROOT + "." + "mq.kafka.kerberos.krb5FilePath"; +// public static final String CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH = ROOT + "." + "mq.kafka.kerberos.jaasFilePath"; +// public static final String CANAL_MQ_USERNAME = ROOT + "." + "mq.username"; +// public static final String CANAL_MQ_PASSWORD = ROOT + "." + "mq.password"; +// public static final String CANAL_MQ_VHOST = ROOT + "." + "mq.vhost"; +// public static final String CANAL_MQ_ALIYUN_UID = ROOT + "." + "mq.aliyunuid"; +// public static final String CANAL_MQ_EXCHANGE = ROOT + "." + "mq.exchange"; +// public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash"; + + public static String getInstanceModeKey(String destination) { + return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination); + } + + public static String getInstanceManagerAddressKey(String destination) { + return MessageFormat.format(INSTANCE_MANAGER_ADDRESS_TEMPLATE, destination); + } + + public static String getInstancSpringXmlKey(String destination) { + return MessageFormat.format(INSTANCE_SPRING_XML_TEMPLATE, destination); + } + + public static String getInstancLazyKey(String destination) { + return MessageFormat.format(INSTANCE_LAZY_TEMPLATE, destination); + } +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalController.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalController.java new file mode 100644 index 0000000..75f2339 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalController.java @@ -0,0 +1,606 @@ +package cc.smtweb.system.canal.server; + +import java.util.Map; +import java.util.Properties; + +import cc.smtweb.system.canal.server.monitor.InstanceAction; +import cc.smtweb.system.canal.server.monitor.InstanceConfigMonitor; +import cc.smtweb.system.canal.server.monitor.ManagerInstanceConfigMonitor; +import cc.smtweb.system.canal.server.monitor.SpringInstanceConfigMonitor; +import org.I0Itec.zkclient.IZkStateListener; +import org.I0Itec.zkclient.exception.ZkNoNodeException; +import org.I0Itec.zkclient.exception.ZkNodeExistsException; +import org.apache.commons.lang.BooleanUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.zookeeper.Watcher.Event.KeeperState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; + +import com.alibaba.otter.canal.common.utils.AddressUtils; +import com.alibaba.otter.canal.common.zookeeper.ZkClientx; +import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; +import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningData; +import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningListener; +import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitor; +import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitors; +import cc.smtweb.system.canal.server.InstanceConfig.InstanceMode; +import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator; +import com.alibaba.otter.canal.instance.manager.PlainCanalInstanceGenerator; +import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient; +import com.alibaba.otter.canal.instance.spring.SpringCanalInstanceGenerator; +import com.alibaba.otter.canal.server.CanalMQStarter; +import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded; +import com.alibaba.otter.canal.server.exception.CanalServerException; +import com.alibaba.otter.canal.server.netty.CanalServerWithNetty; +import com.google.common.base.Function; +import com.google.common.collect.MapMaker; +import com.google.common.collect.MigrateMap; + +/** + * canal调度控制器 + * + * @author jianghang 2012-11-8 下午12:03:11 + * @version 1.0.0 + */ +public class CanalController { + + private static final Logger logger = LoggerFactory.getLogger(CanalController.class); + private String ip; + private String registerIp; + private int port; + private int adminPort; + // 默认使用spring的方式载入 + private Map instanceConfigs; + private InstanceConfig globalInstanceConfig; + private Map managerClients; + // 监听instance config的变化 + private boolean autoScan = true; + private InstanceAction defaultAction; + private Map instanceConfigMonitors; + private CanalServerWithEmbedded embededCanalServer; + private CanalServerWithNetty canalServer; + + private CanalInstanceGenerator instanceGenerator; + private ZkClientx zkclientx; + + private CanalMQStarter canalMQStarter; + private String adminUser; + private String adminPasswd; + + public CanalController(){ + this(System.getProperties()); + } + + public CanalController(final Properties properties){ + managerClients = MigrateMap.makeComputingMap(this::getManagerClient); + + // 初始化全局参数设置 + globalInstanceConfig = initGlobalConfig(properties); + instanceConfigs = new MapMaker().makeMap(); + // 初始化instance config + initInstanceConfig(properties); + + // init socketChannel + String socketChannel = getProperty(properties, CanalConstants.CANAL_SOCKETCHANNEL); + if (StringUtils.isNotEmpty(socketChannel)) { + System.setProperty(CanalConstants.CANAL_SOCKETCHANNEL, socketChannel); + } + + // 兼容1.1.0版本的ak/sk参数名 + String accesskey = getProperty(properties, "canal.instance.rds.accesskey"); + String secretkey = getProperty(properties, "canal.instance.rds.secretkey"); + if (StringUtils.isNotEmpty(accesskey)) { + System.setProperty(CanalConstants.CANAL_ALIYUN_ACCESSKEY, accesskey); + } + if (StringUtils.isNotEmpty(secretkey)) { + System.setProperty(CanalConstants.CANAL_ALIYUN_SECRETKEY, secretkey); + } + + // 准备canal server + ip = getProperty(properties, CanalConstants.CANAL_IP); + registerIp = getProperty(properties, CanalConstants.CANAL_REGISTER_IP); + port = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_PORT, "11111")); + adminPort = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_ADMIN_PORT, "11110")); + embededCanalServer = CanalServerWithEmbedded.instance(); + embededCanalServer.setCanalInstanceGenerator(instanceGenerator);// 设置自定义的instanceGenerator + int metricsPort = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_METRICS_PULL_PORT, "11112")); + embededCanalServer.setMetricsPort(metricsPort); + + this.adminUser = getProperty(properties, CanalConstants.CANAL_ADMIN_USER); + this.adminPasswd = getProperty(properties, CanalConstants.CANAL_ADMIN_PASSWD); + embededCanalServer.setUser(getProperty(properties, CanalConstants.CANAL_USER)); + embededCanalServer.setPasswd(getProperty(properties, CanalConstants.CANAL_PASSWD)); + + String canalWithoutNetty = getProperty(properties, CanalConstants.CANAL_WITHOUT_NETTY); + if (canalWithoutNetty == null || "false".equals(canalWithoutNetty)) { + canalServer = CanalServerWithNetty.instance(); + canalServer.setIp(ip); + canalServer.setPort(port); + } + + // 处理下ip为空，默认使用hostIp暴露到zk中 + if (StringUtils.isEmpty(ip) && StringUtils.isEmpty(registerIp)) { + ip = registerIp = AddressUtils.getHostIp(); + } + + if (StringUtils.isEmpty(ip)) { + ip = AddressUtils.getHostIp(); + } + + if (StringUtils.isEmpty(registerIp)) { + registerIp = ip; // 兼容以前配置 + } + final String zkServers = getProperty(properties, CanalConstants.CANAL_ZKSERVERS); + if (StringUtils.isNotEmpty(zkServers)) { + zkclientx = ZkClientx.getZkClient(zkServers); + // 初始化系统目录 + zkclientx.createPersistent(ZookeeperPathUtils.DESTINATION_ROOT_NODE, true); + zkclientx.createPersistent(ZookeeperPathUtils.CANAL_CLUSTER_ROOT_NODE, true); + } + + final ServerRunningData serverData = new ServerRunningData(registerIp + ":" + port); + ServerRunningMonitors.setServerData(serverData); + ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap((Function) destination -> { + ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData); + runningMonitor.setDestination(destination); + runningMonitor.setListener(new ServerRunningListener() { + + public void processActiveEnter() { + try { + MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); + embededCanalServer.start(destination); + if (canalMQStarter != null) { + canalMQStarter.startDestination(destination); + } + } finally { + MDC.remove(CanalConstants.MDC_DESTINATION); + } + } + + public void processActiveExit() { + try { + MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); + if (canalMQStarter != null) { + canalMQStarter.stopDestination(destination); + } + embededCanalServer.stop(destination); + } finally { + MDC.remove(CanalConstants.MDC_DESTINATION); + } + } + + public void processStart() { + try { + if (zkclientx != null) { + final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, + registerIp + ":" + port); + initCid(path); + zkclientx.subscribeStateChanges(new IZkStateListener() { + + public void handleStateChanged(KeeperState state) throws Exception { + + } + + public void handleNewSession() throws Exception { + initCid(path); + } + + @Override + public void handleSessionEstablishmentError(Throwable error) throws Exception { + logger.error("failed to connect to zookeeper", error); + } + }); + } + } finally { + MDC.remove(CanalConstants.MDC_DESTINATION); + } + } + + public void processStop() { + try { + MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); + if (zkclientx != null) { + final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, + registerIp + ":" + port); + releaseCid(path); + } + } finally { + MDC.remove(CanalConstants.MDC_DESTINATION); + } + } + + }); + if (zkclientx != null) { + runningMonitor.setZkClient(zkclientx); + } + // 触发创建一下cid节点 + runningMonitor.init(); + return runningMonitor; + })); + + // 初始化monitor机制 + autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN)); + if (autoScan) { + defaultAction = new InstanceAction() { + + public void start(String destination) { + InstanceConfig config = instanceConfigs.get(destination); + if (config == null) { + // 重新读取一下instance config + config = parseInstanceConfig(properties, destination); + instanceConfigs.put(destination, config); + } + + if (!embededCanalServer.isStart(destination)) { + // HA机制启动 + ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination); + if (!config.getLazy() && !runningMonitor.isStart()) { + runningMonitor.start(); + } + } + + logger.info("auto notify start {} successful.", destination); + } + + public void stop(String destination) { + // 此处的stop，代表强制退出，非HA机制，所以需要退出HA的monitor和配置信息 + InstanceConfig config = instanceConfigs.remove(destination); + if (config != null) { + embededCanalServer.stop(destination); + ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination); + if (runningMonitor.isStart()) { + runningMonitor.stop(); + } + } + + logger.info("auto notify stop {} successful.", destination); + } + + public void reload(String destination) { + // 目前任何配置变化，直接重启，简单处理 + stop(destination); + start(destination); + + logger.info("auto notify reload {} successful.", destination); + } + + @Override + public void release(String destination) { + // 此处的release，代表强制释放，主要针对HA机制释放运行，让给其他机器抢占 + InstanceConfig config = instanceConfigs.get(destination); + if (config != null) { + ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination); + if (runningMonitor.isStart()) { + boolean release = runningMonitor.release(); + if (!release) { + // 如果是单机模式,则直接清除配置 + instanceConfigs.remove(destination); + // 停掉服务 + runningMonitor.stop(); + if (instanceConfigMonitors.containsKey(InstanceConfig.InstanceMode.MANAGER)) { + ManagerInstanceConfigMonitor monitor = (ManagerInstanceConfigMonitor) instanceConfigMonitors.get(InstanceConfig.InstanceMode.MANAGER); + Map instanceActions = monitor.getActions(); + if (instanceActions.containsKey(destination)) { + // 清除内存中的autoScan cache + monitor.release(destination); + } + } + } + } + } + + logger.info("auto notify release {} successful.", destination); + } + }; + + instanceConfigMonitors = MigrateMap.makeComputingMap(mode -> { + int scanInterval = Integer.valueOf(getProperty(properties, + CanalConstants.CANAL_AUTO_SCAN_INTERVAL, + "5")); + + if (mode.isSpring()) { + SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor(); + monitor.setScanIntervalInSecond(scanInterval); + monitor.setDefaultAction(defaultAction); + // 设置conf目录，默认是user.dir + conf目录组成 + String rootDir = getProperty(properties, CanalConstants.CANAL_CONF_DIR); + if (StringUtils.isEmpty(rootDir)) { + rootDir = "../conf"; + } + + if (StringUtils.equals("otter-canal", System.getProperty("appName"))) { + monitor.setRootConf(rootDir); + } else { + // eclipse debug模式 + monitor.setRootConf("src/main/resources/"); + } + return monitor; + } else if (mode.isManager()) { + ManagerInstanceConfigMonitor monitor = new ManagerInstanceConfigMonitor(); + monitor.setScanIntervalInSecond(scanInterval); + monitor.setDefaultAction(defaultAction); + String managerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER); + monitor.setConfigClient(getManagerClient(managerAddress)); + return monitor; + } else { + throw new UnsupportedOperationException("unknow mode :" + mode + " for monitor"); + } + }); + } + } + + private InstanceConfig initGlobalConfig(Properties properties) { + String adminManagerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER); + InstanceConfig globalConfig = new InstanceConfig(); + String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(CanalConstants.GLOBAL_NAME)); + if (StringUtils.isNotEmpty(adminManagerAddress)) { + // 如果指定了manager地址,则强制适用manager + globalConfig.setMode(InstanceMode.MANAGER); + } else if (StringUtils.isNotEmpty(modeStr)) { + globalConfig.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr))); + } + + String lazyStr = getProperty(properties, CanalConstants.getInstancLazyKey(CanalConstants.GLOBAL_NAME)); + if (StringUtils.isNotEmpty(lazyStr)) { + globalConfig.setLazy(Boolean.valueOf(lazyStr)); + } + + String managerAddress = getProperty(properties, + CanalConstants.getInstanceManagerAddressKey(CanalConstants.GLOBAL_NAME)); + if (StringUtils.isNotEmpty(managerAddress)) { + if (StringUtils.equals(managerAddress, "${canal.admin.manager}")) { + managerAddress = adminManagerAddress; + } + + globalConfig.setManagerAddress(managerAddress); + } + + String springXml = getProperty(properties, CanalConstants.getInstancSpringXmlKey(CanalConstants.GLOBAL_NAME)); + if (StringUtils.isNotEmpty(springXml)) { + globalConfig.setSpringXml(springXml); + } + + instanceGenerator = destination -> { + InstanceConfig config = instanceConfigs.get(destination); + if (config == null) { + throw new CanalServerException("can't find destination:" + destination); + } + + if (config.getMode().isManager()) { + PlainCanalInstanceGenerator instanceGenerator = new PlainCanalInstanceGenerator(properties); + instanceGenerator.setCanalConfigClient(managerClients.get(config.getManagerAddress())); + instanceGenerator.setSpringXml(config.getSpringXml()); + return instanceGenerator.generate(destination); + } else if (config.getMode().isSpring()) { + SpringCanalInstanceGenerator instanceGenerator = new SpringCanalInstanceGenerator(); + instanceGenerator.setSpringXml(config.getSpringXml()); + return instanceGenerator.generate(destination); + } else { + throw new UnsupportedOperationException("unknow mode :" + config.getMode()); + } + + }; + + return globalConfig; + } + + private PlainCanalConfigClient getManagerClient(String managerAddress) { + return new PlainCanalConfigClient(managerAddress, this.adminUser, this.adminPasswd, this.registerIp, adminPort); + } + + private void initInstanceConfig(Properties properties) { + String destinationStr = getProperty(properties, CanalConstants.CANAL_DESTINATIONS); + String[] destinations = StringUtils.split(destinationStr, CanalConstants.CANAL_DESTINATION_SPLIT); + + for (String destination : destinations) { + InstanceConfig config = parseInstanceConfig(properties, destination); + InstanceConfig oldConfig = instanceConfigs.put(destination, config); + + if (oldConfig != null) { + logger.warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config); + } + } + } + + private InstanceConfig parseInstanceConfig(Properties properties, String destination) { + String adminManagerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER); + InstanceConfig config = new InstanceConfig(globalInstanceConfig); + String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(destination)); + if (StringUtils.isNotEmpty(adminManagerAddress)) { + // 如果指定了manager地址,则强制适用manager + config.setMode(InstanceMode.MANAGER); + } else if (StringUtils.isNotEmpty(modeStr)) { + config.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr))); + } + + String lazyStr = getProperty(properties, CanalConstants.getInstancLazyKey(destination)); + if (!StringUtils.isEmpty(lazyStr)) { + config.setLazy(Boolean.valueOf(lazyStr)); + } + + if (config.getMode().isManager()) { + String managerAddress = getProperty(properties, CanalConstants.getInstanceManagerAddressKey(destination)); + if (StringUtils.isNotEmpty(managerAddress)) { + if (StringUtils.equals(managerAddress, "${canal.admin.manager}")) { + managerAddress = adminManagerAddress; + } + config.setManagerAddress(managerAddress); + } + } + + String springXml = getProperty(properties, CanalConstants.getInstancSpringXmlKey(destination)); + if (StringUtils.isNotEmpty(springXml)) { + config.setSpringXml(springXml); + } + + return config; + } + + public static String getProperty(Properties properties, String key, String defaultValue) { + String value = getProperty(properties, key); + if (StringUtils.isEmpty(value)) { + return defaultValue; + } else { + return value; + } + } + + public static String getProperty(Properties properties, String key) { + key = StringUtils.trim(key); + String value = System.getProperty(key); + + if (value == null) { + value = System.getenv(key); + } + + if (value == null) { + value = properties.getProperty(key); + } + + return StringUtils.trim(value); + } + + public void start() throws Throwable { + logger.info("## start the canal server[{}({}):{}]", ip, registerIp, port); + // 创建整个canal的工作节点 + final String path = ZookeeperPathUtils.getCanalClusterNode(registerIp + ":" + port); + initCid(path); + if (zkclientx != null) { + this.zkclientx.subscribeStateChanges(new IZkStateListener() { + + public void handleStateChanged(KeeperState state) throws Exception { + + } + + public void handleNewSession() throws Exception { + initCid(path); + } + + @Override + public void handleSessionEstablishmentError(Throwable error) throws Exception { + logger.error("failed to connect to zookeeper", error); + } + }); + } + // 优先启动embeded服务 + embededCanalServer.start(); + // 尝试启动一下非lazy状态的通道 + for (Map.Entry entry : instanceConfigs.entrySet()) { + final String destination = entry.getKey(); + InstanceConfig config = entry.getValue(); + // 创建destination的工作节点 + if (!embededCanalServer.isStart(destination)) { + // HA机制启动 + ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination); + if (!config.getLazy() && !runningMonitor.isStart()) { + runningMonitor.start(); + } + } + + if (autoScan) { + instanceConfigMonitors.get(config.getMode()).register(destination, defaultAction); + } + } + + if (autoScan) { + instanceConfigMonitors.get(globalInstanceConfig.getMode()).start(); + for (InstanceConfigMonitor monitor : instanceConfigMonitors.values()) { + if (!monitor.isStart()) { + monitor.start(); + } + } + } + + // 启动网络接口 + if (canalServer != null) { + canalServer.start(); + } + } + + public void stop() throws Throwable { + + if (canalServer != null) { + canalServer.stop(); + } + + if (autoScan) { + for (InstanceConfigMonitor monitor : instanceConfigMonitors.values()) { + if (monitor.isStart()) { + monitor.stop(); + } + } + } + + for (ServerRunningMonitor runningMonitor : ServerRunningMonitors.getRunningMonitors().values()) { + if (runningMonitor.isStart()) { + runningMonitor.stop(); + } + } + + // 释放canal的工作节点 + releaseCid(ZookeeperPathUtils.getCanalClusterNode(registerIp + ":" + port)); + logger.info("## stop the canal server[{}({}):{}]", ip, registerIp, port); + + if (zkclientx != null) { + zkclientx.close(); + } + + // 关闭时清理缓存 + if (instanceConfigs != null) { + instanceConfigs.clear(); + } + if (managerClients != null) { + managerClients.clear(); + } + if (instanceConfigMonitors != null) { + instanceConfigMonitors.clear(); + } + + ZkClientx.clearClients(); + } + + private void initCid(String path) { + // logger.info("## init the canalId = {}", cid); + // 初始化系统目录 + if (zkclientx != null) { + try { + zkclientx.createEphemeral(path); + } catch (ZkNoNodeException e) { + // 如果父目录不存在，则创建 + String parentDir = path.substring(0, path.lastIndexOf('/')); + zkclientx.createPersistent(parentDir, true); + zkclientx.createEphemeral(path); + } catch (ZkNodeExistsException e) { + // ignore + // 因为第一次启动时创建了cid,但在stop/start的时可能会关闭和新建,允许出现NodeExists问题s + } + + } + } + + private void releaseCid(String path) { + // logger.info("## release the canalId = {}", cid); + // 初始化系统目录 + if (zkclientx != null) { + zkclientx.delete(path); + } + } + + public CanalMQStarter getCanalMQStarter() { + return canalMQStarter; + } + + public void setCanalMQStarter(CanalMQStarter canalMQStarter) { + this.canalMQStarter = canalMQStarter; + } + + public Map getInstanceConfigMonitors() { + return instanceConfigMonitors; + } + + public Map getInstanceConfigs() { + return instanceConfigs; + } + +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalLauncher.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalLauncher.java new file mode 100644 index 0000000..e84b6f0 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalLauncher.java @@ -0,0 +1,141 @@ +package cc.smtweb.system.canal.server; + +import com.alibaba.otter.canal.common.utils.AddressUtils; +import com.alibaba.otter.canal.common.utils.NamedThreadFactory; +import com.alibaba.otter.canal.instance.manager.plain.PlainCanal; +import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient; +import org.apache.commons.lang.BooleanUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileInputStream; +import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * canal独立版本启动的入口类 + * + * @author jianghang 2012-11-6 下午05:20:49 + * @version 1.0.0 + */ +public class CanalLauncher { + + private static final String CLASSPATH_URL_PREFIX = "classpath:"; + private static final Logger logger = LoggerFactory.getLogger(CanalLauncher.class); + public static final CountDownLatch runningLatch = new CountDownLatch(1); + private static ScheduledExecutorService executor = Executors.newScheduledThreadPool(1, + new NamedThreadFactory("canal-server-scan")); + + public static void main(String[] args) { + startServer(); + } + + public static void startServer() { + try { + logger.info("## canal sever start begin"); + logger.info("## set default uncaught exception handler"); + setGlobalUncaughtExceptionHandler(); + + // 支持rocketmq client 配置日志路径 + System.setProperty("rocketmq.client.logUseSlf4j", "true"); + + logger.info("## load canal configurations"); + String conf = System.getProperty("canal.conf", "classpath:canal.properties"); + Properties properties = new Properties(); + if (conf.startsWith(CLASSPATH_URL_PREFIX)) { + conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX); + properties.load(CanalLauncher.class.getClassLoader().getResourceAsStream(conf)); + } else { + properties.load(new FileInputStream(conf)); + } + + final CanalStarter canalStater = new CanalStarter(properties); + String managerAddress = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER); + if (StringUtils.isNotEmpty(managerAddress)) { + String user = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_USER); + String passwd = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PASSWD); + String adminPort = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PORT, "11110"); + boolean autoRegister = BooleanUtils.toBoolean(CanalController.getProperty(properties, + CanalConstants.CANAL_ADMIN_AUTO_REGISTER)); + String autoCluster = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_AUTO_CLUSTER); + String name = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_REGISTER_NAME); + if (StringUtils.isEmpty(name)) { + name = AddressUtils.getHostName(); + } + + String registerIp = CanalController.getProperty(properties, CanalConstants.CANAL_REGISTER_IP); + if (StringUtils.isEmpty(registerIp)) { + registerIp = AddressUtils.getHostIp(); + } + final PlainCanalConfigClient configClient = new PlainCanalConfigClient(managerAddress, + user, + passwd, + registerIp, + Integer.parseInt(adminPort), + autoRegister, + autoCluster, + name); + PlainCanal canalConfig = configClient.findServer(null); + if (canalConfig == null) { + throw new IllegalArgumentException("managerAddress:" + managerAddress + + " can't not found config for [" + registerIp + ":" + adminPort + + "]"); + } + Properties managerProperties = canalConfig.getProperties(); + // merge local + managerProperties.putAll(properties); + int scanIntervalInSecond = Integer.valueOf(CanalController.getProperty(managerProperties, + CanalConstants.CANAL_AUTO_SCAN_INTERVAL, + "5")); + executor.scheduleWithFixedDelay(new Runnable() { + + private PlainCanal lastCanalConfig; + + public void run() { + try { + if (lastCanalConfig == null) { + lastCanalConfig = configClient.findServer(null); + } else { + PlainCanal newCanalConfig = configClient.findServer(lastCanalConfig.getMd5()); + if (newCanalConfig != null) { + // 远程配置canal.properties修改重新加载整个应用 + canalStater.stop(); + Properties managerProperties = newCanalConfig.getProperties(); + // merge local + managerProperties.putAll(properties); + canalStater.setProperties(managerProperties); + canalStater.start(); + + lastCanalConfig = newCanalConfig; + } + } + + } catch (Throwable e) { + logger.error("scan failed", e); + } + } + + }, 0, scanIntervalInSecond, TimeUnit.SECONDS); + canalStater.setProperties(managerProperties); + } else { + canalStater.setProperties(properties); + } + + canalStater.start(); + logger.info("## canal server start success"); + runningLatch.await(); + executor.shutdownNow(); + } catch (Throwable e) { + logger.error("## Something goes wrong when starting up the canal Server:", e); + } + } + + private static void setGlobalUncaughtExceptionHandler() { + Thread.setDefaultUncaughtExceptionHandler((t, e) -> logger.error("UnCaughtException", e)); + } + +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStartedListener.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStartedListener.java new file mode 100644 index 0000000..4530945 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStartedListener.java @@ -0,0 +1,39 @@ +package cc.smtweb.system.canal.server; + +import cc.smtweb.framework.core.annotation.SwStartListener; +import cc.smtweb.framework.core.common.SwConsts; +import cc.smtweb.framework.core.mvc.controller.IStartListener; +import cc.smtweb.framework.core.systask.SysThreadPool; +import cc.smtweb.framework.core.systask.SysThreadWorker; +import lombok.extern.slf4j.Slf4j; + +/** + * @Author yaoq + * @Date 2022年09月06日 10:31 + * @Description + */ +@Slf4j +@SwStartListener +public class CanalStartedListener implements IStartListener { + + @Override + public int order() { + return SwConsts.DEFAULT_ORDER + 2; + } + + + @Override + public void init() { + + } + + @Override + public void run() { + SysThreadPool.getInstance().addTask(new SysThreadWorker("canal server") { + @Override + public void localWork() throws Exception { + CanalLauncher.startServer(); + } + }); + } +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStarter.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStarter.java new file mode 100644 index 0000000..58b6621 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStarter.java @@ -0,0 +1,169 @@ +package cc.smtweb.system.canal.server; + +import java.util.Properties; + +import com.alibaba.otter.canal.connector.core.config.MQProperties; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.admin.netty.CanalAdminWithNetty; +import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer; +import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader; +import cc.smtweb.system.canal.server.admin.CanalAdminController; +import com.alibaba.otter.canal.server.CanalMQStarter; + +/** + * Canal server 启动类 + * + * @author rewerma 2020-01-27 + * @version 1.0.2 + */ +public class CanalStarter { + + private static final Logger logger = LoggerFactory.getLogger(CanalStarter.class); + + private static final String CONNECTOR_SPI_DIR = "/plugin"; + private static final String CONNECTOR_STANDBY_SPI_DIR = "/canal/plugin"; + + private CanalController controller = null; + private CanalMQProducer canalMQProducer = null; + private Thread shutdownThread = null; + private CanalMQStarter canalMQStarter = null; + private volatile Properties properties; + private volatile boolean running = false; + + private CanalAdminWithNetty canalAdmin; + + public CanalStarter(Properties properties){ + this.properties = properties; + } + + public boolean isRunning() { + return running; + } + + public Properties getProperties() { + return properties; + } + + public void setProperties(Properties properties) { + this.properties = properties; + } + + public CanalController getController() { + return controller; + } + + /** + * 启动方法 + * + * @throws Throwable + */ + public synchronized void start() throws Throwable { + String serverMode = CanalController.getProperty(properties, CanalConstants.CANAL_SERVER_MODE); + if (!"tcp".equalsIgnoreCase(serverMode)) { + ExtensionLoader loader = ExtensionLoader.getExtensionLoader(CanalMQProducer.class); + canalMQProducer = loader + .getExtension(serverMode.toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR); + if (canalMQProducer != null) { + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(canalMQProducer.getClass().getClassLoader()); + canalMQProducer.init(properties); + Thread.currentThread().setContextClassLoader(cl); + } + } + + if (canalMQProducer != null) { + MQProperties mqProperties = canalMQProducer.getMqProperties(); + // disable netty + System.setProperty(CanalConstants.CANAL_WITHOUT_NETTY, "true"); + if (mqProperties.isFlatMessage()) { + // 设置为raw避免ByteString->Entry的二次解析 + System.setProperty("canal.instance.memory.rawEntry", "false"); + } + } + + logger.info("## start the canal server."); + controller = new CanalController(properties); + controller.start(); + logger.info("## the canal server is running now ......"); + shutdownThread = new Thread(() -> { + try { + logger.info("## stop the canal server"); + controller.stop(); + CanalLauncher.runningLatch.countDown(); + } catch (Throwable e) { + logger.warn("##something goes wrong when stopping canal Server:", e); + } finally { + logger.info("## canal server is down."); + } + }); + Runtime.getRuntime().addShutdownHook(shutdownThread); + + if (canalMQProducer != null) { + canalMQStarter = new CanalMQStarter(canalMQProducer); + String destinations = CanalController.getProperty(properties, CanalConstants.CANAL_DESTINATIONS); + canalMQStarter.start(destinations); + controller.setCanalMQStarter(canalMQStarter); + } + + // start canalAdmin + String port = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PORT); + if (canalAdmin == null && StringUtils.isNotEmpty(port)) { + String user = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_USER); + String passwd = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PASSWD); + CanalAdminController canalAdmin = new CanalAdminController(this); + canalAdmin.setUser(user); + canalAdmin.setPasswd(passwd); + + String ip = CanalController.getProperty(properties, CanalConstants.CANAL_IP); + + logger.debug("canal admin port:{}, canal admin user:{}, canal admin password: {}, canal ip:{}", + port, + user, + passwd, + ip); + + CanalAdminWithNetty canalAdminWithNetty = CanalAdminWithNetty.instance(); + canalAdminWithNetty.setCanalAdmin(canalAdmin); + canalAdminWithNetty.setPort(Integer.parseInt(port)); + canalAdminWithNetty.setIp(ip); + canalAdminWithNetty.start(); + this.canalAdmin = canalAdminWithNetty; + } + + running = true; + } + + public synchronized void stop() throws Throwable { + stop(false); + } + + /** + * 销毁方法，远程配置变更时调用 + * + * @throws Throwable + */ + public synchronized void stop(boolean stopByAdmin) throws Throwable { + if (!stopByAdmin && canalAdmin != null) { + canalAdmin.stop(); + canalAdmin = null; + } + + if (controller != null) { + controller.stop(); + controller = null; + } + if (shutdownThread != null) { + Runtime.getRuntime().removeShutdownHook(shutdownThread); + shutdownThread = null; + } + if (canalMQProducer != null && canalMQStarter != null) { + canalMQStarter.destroy(); + canalMQStarter = null; + canalMQProducer = null; + } + running = false; + } +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/InstanceConfig.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/InstanceConfig.java new file mode 100644 index 0000000..6775ca9 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/InstanceConfig.java @@ -0,0 +1,93 @@ +package cc.smtweb.system.canal.server; + +import org.apache.commons.lang.builder.ToStringBuilder; + +import com.alibaba.otter.canal.common.utils.CanalToStringStyle; + +/** + * 启动的相关配置 + * + * @author jianghang 2012-11-8 下午02:50:54 + * @version 1.0.0 + */ +public class InstanceConfig { + + private InstanceConfig globalConfig; + private InstanceMode mode; + private Boolean lazy; + private String managerAddress; + private String springXml; + + public InstanceConfig(){ + + } + + public InstanceConfig(InstanceConfig globalConfig){ + this.globalConfig = globalConfig; + } + + public static enum InstanceMode { + SPRING, MANAGER; + + public boolean isSpring() { + return this == InstanceMode.SPRING; + } + + public boolean isManager() { + return this == InstanceMode.MANAGER; + } + } + + public Boolean getLazy() { + if (lazy == null && globalConfig != null) { + return globalConfig.getLazy(); + } else { + return lazy; + } + } + + public void setLazy(Boolean lazy) { + this.lazy = lazy; + } + + public InstanceMode getMode() { + if (mode == null && globalConfig != null) { + return globalConfig.getMode(); + } else { + return mode; + } + } + + public void setMode(InstanceMode mode) { + this.mode = mode; + } + + public String getManagerAddress() { + if (managerAddress == null && globalConfig != null) { + return globalConfig.getManagerAddress(); + } else { + return managerAddress; + } + } + + public void setManagerAddress(String managerAddress) { + this.managerAddress = managerAddress; + } + + public String getSpringXml() { + if (springXml == null && globalConfig != null) { + return globalConfig.getSpringXml(); + } else { + return springXml; + } + } + + public void setSpringXml(String springXml) { + this.springXml = springXml; + } + + public String toString() { + return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); + } + +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/admin/CanalAdminController.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/admin/CanalAdminController.java new file mode 100644 index 0000000..d504621 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/admin/CanalAdminController.java @@ -0,0 +1,256 @@ +package cc.smtweb.system.canal.server.admin; + +import java.io.File; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import cc.smtweb.system.canal.server.CanalStarter; +import cc.smtweb.system.canal.server.monitor.InstanceConfigMonitor; +import cc.smtweb.system.canal.server.monitor.ManagerInstanceConfigMonitor; +import cc.smtweb.system.canal.server.monitor.SpringInstanceConfigMonitor; +import cc.smtweb.system.canal.server.monitor.InstanceAction; +import org.apache.commons.io.filefilter.TrueFileFilter; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.admin.CanalAdmin; +import com.alibaba.otter.canal.common.utils.FileUtils; +import cc.smtweb.system.canal.server.InstanceConfig; +import com.alibaba.otter.canal.instance.core.CanalInstance; +import com.alibaba.otter.canal.protocol.SecurityUtil; +import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded; +import com.google.common.base.Joiner; + +/** + * 提供canal admin的管理操作 + * + * @author agapple 2019年8月24日 下午11:39:01 + * @since 1.1.4 + */ +public class CanalAdminController implements CanalAdmin { + + private static final Logger logger = LoggerFactory.getLogger(CanalAdminController.class); + private String user; + private String passwd; + private CanalStarter canalStater; + + public CanalAdminController(CanalStarter canalStater){ + this.canalStater = canalStater; + } + + @Override + public boolean check() { + return canalStater.isRunning(); + } + + @Override + public synchronized boolean start() { + try { + if (!canalStater.isRunning()) { + canalStater.start(); + return true; + } + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } + return false; + } + + @Override + public synchronized boolean stop() { + try { + if (canalStater.isRunning()) { + canalStater.stop(true); + return true; + } + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } + return false; + } + + @Override + public synchronized boolean restart() { + stop(); + return start(); + } + + @Override + public boolean auth(String user, String passwd, byte[] seed) { + // 如果user/passwd密码为空,则任何用户账户都能登录 + if ((StringUtils.isEmpty(this.user) || StringUtils.equals(this.user, user))) { + if (StringUtils.isEmpty(this.passwd)) { + return true; + } else if (StringUtils.isEmpty(passwd)) { + // 如果server密码有配置,客户端密码为空,则拒绝 + return false; + } + + try { + byte[] passForClient = SecurityUtil.hexStr2Bytes(passwd); + return SecurityUtil.scrambleServerAuth(passForClient, SecurityUtil.hexStr2Bytes(this.passwd), seed); + } catch (NoSuchAlgorithmException e) { + return false; + } + } + + return false; + } + + @Override + public String getRunningInstances() { + try { + Map instances = CanalServerWithEmbedded.instance().getCanalInstances(); + List runningInstances = new ArrayList<>(); + instances.forEach((destination, instance) -> { + if (instance.isStart()) { + runningInstances.add(destination); + } + }); + + return Joiner.on(",").join(runningInstances); + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } + return ""; + } + + @Override + public boolean checkInstance(String destination) { + Map instances = CanalServerWithEmbedded.instance().getCanalInstances(); + if (instances == null || !instances.containsKey(destination)) { + return false; + } else { + CanalInstance instance = instances.get(destination); + return instance.isStart(); + } + } + + @Override + public boolean startInstance(String destination) { + try { + InstanceAction instanceAction = getInstanceAction(destination); + if (instanceAction != null) { + instanceAction.start(destination); + return true; + } + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } + return false; + } + + @Override + public boolean stopInstance(String destination) { + try { + InstanceAction instanceAction = getInstanceAction(destination); + if (instanceAction != null) { + instanceAction.stop(destination); + return true; + } + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } + return false; + } + + @Override + public boolean releaseInstance(String destination) { + try { + InstanceAction instanceAction = getInstanceAction(destination); + if (instanceAction != null) { + instanceAction.release(destination); + return true; + } + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } + return false; + } + + @Override + public boolean restartInstance(String destination) { + try { + InstanceAction instanceAction = getInstanceAction(destination); + if (instanceAction != null) { + instanceAction.reload(destination); + return true; + } + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } + return false; + } + + @Override + public String listCanalLog() { + Collection files = org.apache.commons.io.FileUtils.listFiles(new File("../logs/canal/"), + TrueFileFilter.TRUE, + TrueFileFilter.TRUE); + List names = files.stream().map(File::getName).collect(Collectors.toList()); + return Joiner.on(",").join(names); + } + + @Override + public String canalLog(int lines) { + return FileUtils.readFileFromOffset("../logs/canal/canal.log", lines, "UTF-8"); + } + + @Override + public String listInstanceLog(String destination) { + Collection files = org.apache.commons.io.FileUtils.listFiles(new File("../logs/" + destination + "/"), + TrueFileFilter.TRUE, + TrueFileFilter.TRUE); + List names = files.stream().map(File::getName).collect(Collectors.toList()); + return Joiner.on(",").join(names); + } + + @Override + public String instanceLog(String destination, String fileName, int lines) { + if (StringUtils.isEmpty(fileName)) { + fileName = destination + ".log"; + } + return FileUtils.readFileFromOffset("../logs/" + destination + "/" + fileName, lines, "UTF-8"); + } + + private InstanceAction getInstanceAction(String destination) { + Map monitors = canalStater.getController() + .getInstanceConfigMonitors(); + + InstanceAction instanceAction = null; + if (monitors.containsKey(InstanceConfig.InstanceMode.SPRING)) { + SpringInstanceConfigMonitor monitor = (SpringInstanceConfigMonitor) monitors.get(InstanceConfig.InstanceMode.SPRING); + Map instanceActions = monitor.getActions(); + instanceAction = instanceActions.get(destination); + } + + if (instanceAction != null) { + return instanceAction; + } + + if (monitors.containsKey(InstanceConfig.InstanceMode.MANAGER)) { + ManagerInstanceConfigMonitor monitor = (ManagerInstanceConfigMonitor) monitors.get(InstanceConfig.InstanceMode.MANAGER); + Map instanceActions = monitor.getActions(); + instanceAction = instanceActions.get(destination); + } + return instanceAction; + } + + public void setUser(String user) { + this.user = user; + } + + public void setPasswd(String passwd) { + this.passwd = passwd; + } + + public void setCanalStater(CanalStarter canalStater) { + this.canalStater = canalStater; + } + +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceAction.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceAction.java new file mode 100644 index 0000000..f1ed700 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceAction.java @@ -0,0 +1,30 @@ +package cc.smtweb.system.canal.server.monitor; + +/** + * config配置变化 + * + * @author jianghang 2013-2-18 下午01:19:29 + * @version 1.0.1 + */ +public interface InstanceAction { + + /** + * 启动destination + */ + void start(String destination); + + /** + * 主动释放destination运行 + */ + void release(String destination); + + /** + * 停止destination + */ + void stop(String destination); + + /** + * 重载destination，可能需要stop,start操作，或者只是更新下内存配置 + */ + void reload(String destination); +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceConfigMonitor.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceConfigMonitor.java new file mode 100644 index 0000000..702a2b9 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceConfigMonitor.java @@ -0,0 +1,16 @@ +package cc.smtweb.system.canal.server.monitor; + +import com.alibaba.otter.canal.common.CanalLifeCycle; + +/** + * 监听instance file的文件变化，触发instance start/stop等操作 + * + * @author jianghang 2013-2-6 下午06:19:56 + * @version 1.0.1 + */ +public interface InstanceConfigMonitor extends CanalLifeCycle { + + void register(String destination, InstanceAction action); + + void unregister(String destination); +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/ManagerInstanceConfigMonitor.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/ManagerInstanceConfigMonitor.java new file mode 100644 index 0000000..d0aa043 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/ManagerInstanceConfigMonitor.java @@ -0,0 +1,184 @@ +package cc.smtweb.system.canal.server.monitor; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; +import com.alibaba.otter.canal.common.CanalLifeCycle; +import com.alibaba.otter.canal.common.utils.NamedThreadFactory; +import com.alibaba.otter.canal.instance.manager.plain.PlainCanal; +import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient; +import com.google.common.collect.Lists; +import com.google.common.collect.MapMaker; +import com.google.common.collect.MigrateMap; + +/** + * 基于manager配置的实现 + * + * @author agapple 2019年8月26日 下午10:00:20 + * @since 1.1.4 + */ +public class ManagerInstanceConfigMonitor extends AbstractCanalLifeCycle implements InstanceConfigMonitor, CanalLifeCycle { + + private static final Logger logger = LoggerFactory.getLogger(ManagerInstanceConfigMonitor.class); + private long scanIntervalInSecond = 5; + private InstanceAction defaultAction = null; + private Map actions = new MapMaker().makeMap(); + private Map configs = MigrateMap.makeComputingMap(destination -> new PlainCanal()); + private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1, + new NamedThreadFactory("canal-instance-scan")); + + private volatile boolean isFirst = true; + private PlainCanalConfigClient configClient; + + public void start() { + super.start(); + executor.scheduleWithFixedDelay(() -> { + try { + scan(); + if (isFirst) { + isFirst = false; + } + } catch (Throwable e) { + logger.error("scan failed", e); + } + }, 0, scanIntervalInSecond, TimeUnit.SECONDS); + } + + public void stop() { + super.stop(); + executor.shutdownNow(); + actions.clear(); + } + + public void register(String destination, InstanceAction action) { + if (action != null) { + actions.put(destination, action); + } else { + actions.put(destination, defaultAction); + } + } + + public void unregister(String destination) { + actions.remove(destination); + } + + private void scan() { + String instances = configClient.findInstances(null); + if (instances == null) { + return; + } + + final List is = Lists.newArrayList(StringUtils.split(instances, ',')); + List start = new ArrayList<>(); + List stop = new ArrayList<>(); + List restart = new ArrayList<>(); + for (String instance : is) { + if (!configs.containsKey(instance)) { + PlainCanal newPlainCanal = configClient.findInstance(instance, null); + if (newPlainCanal != null) { + configs.put(instance, newPlainCanal); + start.add(instance); + } + } else { + PlainCanal plainCanal = configs.get(instance); + PlainCanal newPlainCanal = configClient.findInstance(instance, plainCanal.getMd5()); + if (newPlainCanal != null) { + // 配置有变化 + restart.add(instance); + configs.put(instance, newPlainCanal); + } + } + } + + configs.forEach((instance, plainCanal) -> { + if (!is.contains(instance)) { + stop.add(instance); + } + }); + + stop.forEach(instance -> { + notifyStop(instance); + }); + + restart.forEach(instance -> { + notifyReload(instance); + }); + + start.forEach(instance -> { + notifyStart(instance); + }); + + } + + private void notifyStart(String destination) { + try { + defaultAction.start(destination); + actions.put(destination, defaultAction); + // 启动成功后记录配置文件信息 + } catch (Throwable e) { + logger.error(String.format("scan add found[%s] but start failed", destination), e); + } + } + + private void notifyStop(String destination) { + InstanceAction action = actions.remove(destination); + if (action != null) { + try { + action.stop(destination); + configs.remove(destination); + } catch (Throwable e) { + logger.error(String.format("scan delete found[%s] but stop failed", destination), e); + actions.put(destination, action);// 再重新加回去，下一次scan时再执行删除 + } + } + } + + private void notifyReload(String destination) { + InstanceAction action = actions.get(destination); + if (action != null) { + try { + action.reload(destination); + } catch (Throwable e) { + logger.error(String.format("scan reload found[%s] but reload failed", destination), e); + } + } + } + + public void release(String destination) { + InstanceAction action = actions.remove(destination); + if (action != null) { + try { + configs.remove(destination); + } catch (Throwable e) { + logger.error(String.format("scan delete found[%s] but stop failed", destination), e); + actions.put(destination, action);// 再重新加回去，下一次scan时再执行删除 + } + } + } + + public void setDefaultAction(InstanceAction defaultAction) { + this.defaultAction = defaultAction; + } + + public void setScanIntervalInSecond(long scanIntervalInSecond) { + this.scanIntervalInSecond = scanIntervalInSecond; + } + + public void setConfigClient(PlainCanalConfigClient configClient) { + this.configClient = configClient; + } + + public Map getActions() { + return actions; + } + +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/SpringInstanceConfigMonitor.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/SpringInstanceConfigMonitor.java new file mode 100644 index 0000000..83a2b13 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/SpringInstanceConfigMonitor.java @@ -0,0 +1,297 @@ +package cc.smtweb.system.canal.server.monitor; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; + +import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; +import com.alibaba.otter.canal.common.CanalLifeCycle; +import com.alibaba.otter.canal.common.utils.NamedThreadFactory; +import com.google.common.collect.MapMaker; +import com.google.common.collect.MigrateMap; + +/** + * 监听基于spring配置的instance变化 + * + * @author jianghang 2013-2-6 下午06:23:55 + * @version 1.0.1 + */ +public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implements InstanceConfigMonitor, CanalLifeCycle { + + private static final Logger logger = LoggerFactory.getLogger(SpringInstanceConfigMonitor.class); + private String rootConf; + // 扫描周期，单位秒 + private long scanIntervalInSecond = 5; + private InstanceAction defaultAction = null; + private Map actions = new MapMaker().makeMap(); + private Map lastFiles = MigrateMap.makeComputingMap(InstanceConfigFiles::new); + private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1, + new NamedThreadFactory("canal-instance-scan")); + + private volatile boolean isFirst = true; + + public Map getActions() { + return actions; + } + + public void start() { + super.start(); + Assert.notNull(rootConf, "root conf dir is null!"); + + executor.scheduleWithFixedDelay(() -> { + try { + scan(); + if (isFirst) { + isFirst = false; + } + } catch (Throwable e) { + logger.error("scan failed", e); + } + }, 0, scanIntervalInSecond, TimeUnit.SECONDS); + } + + public void stop() { + super.stop(); + executor.shutdownNow(); + actions.clear(); + lastFiles.clear(); + } + + public void register(String destination, InstanceAction action) { + if (action != null) { + actions.put(destination, action); + } else { + actions.put(destination, defaultAction); + } + } + + public void unregister(String destination) { + actions.remove(destination); + } + + public void setRootConf(String rootConf) { + this.rootConf = rootConf; + } + + private void scan() { + File rootdir = new File(rootConf); + if (!rootdir.exists()) { + return; + } + + File[] instanceDirs = rootdir.listFiles(pathname -> { + String filename = pathname.getName(); + return pathname.isDirectory() && !"spring".equalsIgnoreCase(filename); + }); + + // 扫描目录的新增 + Set currentInstanceNames = new HashSet<>(); + + // 判断目录内文件的变化 + for (File instanceDir : instanceDirs) { + String destination = instanceDir.getName(); + currentInstanceNames.add(destination); + File[] instanceConfigs = instanceDir.listFiles((dir, name) -> { + // return !StringUtils.endsWithIgnoreCase(name, ".dat"); + // 限制一下，只针对instance.properties文件,避免因为.svn或者其他生成的临时文件导致出现reload + return StringUtils.equalsIgnoreCase(name, "instance.properties"); + }); + + if (!actions.containsKey(destination) && instanceConfigs.length > 0) { + // 存在合法的instance.properties，并且第一次添加时，进行启动操作 + notifyStart(instanceDir, destination, instanceConfigs); + } else if (actions.containsKey(destination)) { + // 历史已经启动过 + if (instanceConfigs.length == 0) { // 如果不存在合法的instance.properties + notifyStop(destination); + } else { + InstanceConfigFiles lastFile = lastFiles.get(destination); + // 历史启动过 所以配置文件信息必然存在 + if (!isFirst && CollectionUtils.isEmpty(lastFile.getInstanceFiles())) { + logger.error("[{}] is started, but not found instance file info.", destination); + } + + boolean hasChanged = judgeFileChanged(instanceConfigs, lastFile.getInstanceFiles()); + // 通知变化 + if (hasChanged) { + notifyReload(destination); + } + + if (hasChanged || CollectionUtils.isEmpty(lastFile.getInstanceFiles())) { + // 更新内容 + List newFileInfo = new ArrayList<>(); + for (File instanceConfig : instanceConfigs) { + newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified())); + } + + lastFile.setInstanceFiles(newFileInfo); + } + } + } + + } + + // 判断目录是否删除 + Set deleteInstanceNames = new HashSet<>(); + for (String destination : actions.keySet()) { + if (!currentInstanceNames.contains(destination)) { + deleteInstanceNames.add(destination); + } + } + for (String deleteInstanceName : deleteInstanceNames) { + notifyStop(deleteInstanceName); + } + } + + private void notifyStart(File instanceDir, String destination, File[] instanceConfigs) { + try { + defaultAction.start(destination); + actions.put(destination, defaultAction); + + // 启动成功后记录配置文件信息 + InstanceConfigFiles lastFile = lastFiles.get(destination); + List newFileInfo = new ArrayList<>(); + for (File instanceConfig : instanceConfigs) { + newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified())); + } + lastFile.setInstanceFiles(newFileInfo); + } catch (Throwable e) { + logger.error(String.format("scan add found[%s] but start failed", destination), e); + } + } + + private void notifyStop(String destination) { + InstanceAction action = actions.remove(destination); + if (action != null) { + try { + action.stop(destination); + lastFiles.remove(destination); + } catch (Throwable e) { + logger.error(String.format("scan delete found[%s] but stop failed", destination), e); + actions.put(destination, action);// 再重新加回去，下一次scan时再执行删除 + } + } + } + + private void notifyReload(String destination) { + InstanceAction action = actions.get(destination); + if (action != null) { + try { + action.reload(destination); + } catch (Throwable e) { + logger.error(String.format("scan reload found[%s] but reload failed", destination), e); + } + } + } + + private boolean judgeFileChanged(File[] instanceConfigs, List fileInfos) { + boolean hasChanged = false; + for (File instanceConfig : instanceConfigs) { + for (FileInfo fileInfo : fileInfos) { + if (instanceConfig.getName().equals(fileInfo.getName())) { + hasChanged |= (instanceConfig.lastModified() != fileInfo.getLastModified()); + if (hasChanged) { + return hasChanged; + } + } + } + } + + return hasChanged; + } + + public void setDefaultAction(InstanceAction defaultAction) { + this.defaultAction = defaultAction; + } + + public void setScanIntervalInSecond(long scanIntervalInSecond) { + this.scanIntervalInSecond = scanIntervalInSecond; + } + + public static class InstanceConfigFiles { + + private String destination; // instance + // name + private List springFile = new ArrayList<>(); // spring的instance + // xml + private FileInfo rootFile; // canal.properties + private List instanceFiles = new ArrayList<>(); // instance对应的配置 + + public InstanceConfigFiles(String destination){ + this.destination = destination; + } + + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + + public List getSpringFile() { + return springFile; + } + + public void setSpringFile(List springFile) { + this.springFile = springFile; + } + + public FileInfo getRootFile() { + return rootFile; + } + + public void setRootFile(FileInfo rootFile) { + this.rootFile = rootFile; + } + + public List getInstanceFiles() { + return instanceFiles; + } + + public void setInstanceFiles(List instanceFiles) { + this.instanceFiles = instanceFiles; + } + + } + + public static class FileInfo { + + private String name; + private long lastModified = 0; + + public FileInfo(String name, long lastModified){ + this.name = name; + this.lastModified = lastModified; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public long getLastModified() { + return lastModified; + } + + public void setLastModified(long lastModified) { + this.lastModified = lastModified; + } + + } +} diff --git a/smtweb-framework/canal/server/src/main/resources/canal.properties b/smtweb-framework/canal/server/src/main/resources/canal.properties new file mode 100644 index 0000000..82992bc --- /dev/null +++ b/smtweb-framework/canal/server/src/main/resources/canal.properties @@ -0,0 +1,182 @@ +################################################# +######### common argument ############# +################################################# +# tcp bind ip +canal.ip = +# register ip to zookeeper +canal.register.ip = +canal.port = 11111 +canal.metrics.pull.port = 11112 +# canal instance user/passwd +# canal.user = canal +# canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458 + +# canal admin config +#canal.admin.manager = 127.0.0.1:8089 +canal.admin.port = 11110 +canal.admin.user = admin +canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441 +# admin auto register +#canal.admin.register.auto = true +#canal.admin.register.cluster = +#canal.admin.register.name = + +canal.zkServers = +# flush data to zk +canal.zookeeper.flush.period = 1000 +canal.withoutNetty = false +# tcp, kafka, rocketMQ, rabbitMQ, pulsarMQ +canal.serverMode = tcp +# flush meta cursor/parse position to file +canal.file.data.dir = ${canal.conf.dir} +canal.file.flush.period = 1000 +## memory store RingBuffer size, should be Math.pow(2,n) +canal.instance.memory.buffer.size = 16384 +## memory store RingBuffer used memory unit size , default 1kb +canal.instance.memory.buffer.memunit = 1024 +## meory store gets mode used MEMSIZE or ITEMSIZE +canal.instance.memory.batch.mode = MEMSIZE +canal.instance.memory.rawEntry = true + +## detecing config +canal.instance.detecting.enable = false +#canal.instance.detecting.sql = insert into retl.xdual values(1,now()) on duplicate key update x=now() +canal.instance.detecting.sql = select 1 +canal.instance.detecting.interval.time = 3 +canal.instance.detecting.retry.threshold = 3 +canal.instance.detecting.heartbeatHaEnable = false + +# support maximum transaction size, more than the size of the transaction will be cut into multiple transactions delivery +canal.instance.transaction.size = 1024 +# mysql fallback connected to new master should fallback times +canal.instance.fallbackIntervalInSeconds = 60 + +# network config +canal.instance.network.receiveBufferSize = 16384 +canal.instance.network.sendBufferSize = 16384 +canal.instance.network.soTimeout = 30 + +# binlog filter config +canal.instance.filter.druid.ddl = true +canal.instance.filter.query.dcl = false +canal.instance.filter.query.dml = false +canal.instance.filter.query.ddl = false +canal.instance.filter.table.error = false +canal.instance.filter.rows = false +canal.instance.filter.transaction.entry = false +canal.instance.filter.dml.insert = false +canal.instance.filter.dml.update = false +canal.instance.filter.dml.delete = false + +# binlog format/image check +canal.instance.binlog.format = ROW,STATEMENT,MIXED +canal.instance.binlog.image = FULL,MINIMAL,NOBLOB + +# binlog ddl isolation +canal.instance.get.ddl.isolation = false + +# parallel parser config +canal.instance.parser.parallel = true +## concurrent thread number, default 60% available processors, suggest not to exceed Runtime.getRuntime().availableProcessors() +#canal.instance.parser.parallelThreadSize = 16 +## disruptor ringbuffer size, must be power of 2 +canal.instance.parser.parallelBufferSize = 256 + +# table meta tsdb info +canal.instance.tsdb.enable = true +canal.instance.tsdb.dir = ${canal.file.data.dir:../conf}/${canal.instance.destination:} +canal.instance.tsdb.url = jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL; +canal.instance.tsdb.dbUsername = canal +canal.instance.tsdb.dbPassword = canal +# dump snapshot interval, default 24 hour +canal.instance.tsdb.snapshot.interval = 24 +# purge snapshot expire , default 360 hour(15 days) +canal.instance.tsdb.snapshot.expire = 360 + +################################################# +######### destinations ############# +################################################# +canal.destinations = example +# conf root dir +canal.conf.dir = ../conf +# auto scan instance dir add/remove and start/stop instance +canal.auto.scan = true +canal.auto.scan.interval = 5 +# set this value to 'true' means that when binlog pos not found, skip to latest. +# WARN: pls keep 'false' in production env, or if you know what you want. +canal.auto.reset.latest.pos.mode = false + +canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml +#canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml + +canal.instance.global.mode = spring +canal.instance.global.lazy = false +canal.instance.global.manager.address = ${canal.admin.manager} +#canal.instance.global.spring.xml = classpath:spring/memory-instance.xml +canal.instance.global.spring.xml = classpath:spring/file-instance.xml +#canal.instance.global.spring.xml = classpath:spring/default-instance.xml + +################################################## +######### MQ Properties ############# +################################################## +# aliyun ak/sk , support rds/mq +canal.aliyun.accessKey = +canal.aliyun.secretKey = +canal.aliyun.uid= + +canal.mq.flatMessage = true +canal.mq.canalBatchSize = 50 +canal.mq.canalGetTimeout = 100 +# Set this value to "cloud", if you want open message trace feature in aliyun. +canal.mq.accessChannel = local + +canal.mq.database.hash = true +canal.mq.send.thread.size = 30 +canal.mq.build.thread.size = 8 + +################################################## +######### Kafka ############# +################################################## +kafka.bootstrap.servers = 127.0.0.1:9092 +kafka.acks = all +kafka.compression.type = none +kafka.batch.size = 16384 +kafka.linger.ms = 1 +kafka.max.request.size = 1048576 +kafka.buffer.memory = 33554432 +kafka.max.in.flight.requests.per.connection = 1 +kafka.retries = 0 + +kafka.kerberos.enable = false +kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf" +kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf" + +################################################## +######### RocketMQ ############# +################################################## +rocketmq.producer.group = test +rocketmq.enable.message.trace = false +rocketmq.customized.trace.topic = +rocketmq.namespace = +rocketmq.namesrv.addr = 127.0.0.1:9876 +rocketmq.retry.times.when.send.failed = 0 +rocketmq.vip.channel.enabled = false +rocketmq.tag = + +################################################## +######### RabbitMQ ############# +################################################## +rabbitmq.host = +rabbitmq.virtual.host = +rabbitmq.exchange = +rabbitmq.username = +rabbitmq.password = +rabbitmq.deliveryMode = + + +################################################## +######### Pulsar ############# +################################################## +pulsarmq.serverUrl = +pulsarmq.roleToken = +pulsarmq.topicTenantPrefix = \ No newline at end of file diff --git a/smtweb-framework/canal/server/src/main/resources/canal_local.properties b/smtweb-framework/canal/server/src/main/resources/canal_local.properties new file mode 100644 index 0000000..f92b8f3 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/resources/canal_local.properties @@ -0,0 +1,12 @@ +# register ip +canal.register.ip = + +# canal admin config +canal.admin.manager = 127.0.0.1:8089 +canal.admin.port = 11110 +canal.admin.user = admin +canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441 +# admin auto register +canal.admin.register.auto = true +canal.admin.register.cluster = +canal.admin.register.name = \ No newline at end of file diff --git a/smtweb-framework/canal/server/src/main/resources/example/instance.properties b/smtweb-framework/canal/server/src/main/resources/example/instance.properties new file mode 100644 index 0000000..7a1f752 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/resources/example/instance.properties @@ -0,0 +1,59 @@ +################################################# +## mysql serverId , v1.0.26+ will autoGen +canal.instance.mysql.slaveId=2 + +# enable gtid use true/false +canal.instance.gtidon=false + +# position info +canal.instance.master.address=172.28.123.25:3306 +#canal.instance.master.journal.name= +#canal.instance.master.position= +#canal.instance.master.timestamp= +#canal.instance.master.gtid= + +# rds oss binlog +canal.instance.rds.accesskey= +canal.instance.rds.secretkey= +canal.instance.rds.instanceId= + +# table meta tsdb info +canal.instance.tsdb.enable=true +#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb +#canal.instance.tsdb.dbUsername=canal +#canal.instance.tsdb.dbPassword=canal + +#canal.instance.standby.address = +#canal.instance.standby.journal.name = +#canal.instance.standby.position = +#canal.instance.standby.timestamp = +#canal.instance.standby.gtid= + +# username/password +canal.instance.dbUsername=root +canal.instance.dbPassword=root +canal.instance.connectionCharset = UTF-8 +# enable druid Decrypt database password +canal.instance.enableDruid=false +#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ== + +# table regex +canal.instance.filter.regex=scmz\\..* +# table black regex +canal.instance.filter.black.regex=mysql\\.slave_.* +# table field filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2) +#canal.instance.filter.field=test1.t_product:id/subject/keywords,test2.t_company:id/name/contact/ch +# table field black filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2) +#canal.instance.filter.black.field=test1.t_product:subject/product_image,test2.t_company:id/name/contact/ch + +# mq config +canal.mq.topic=example +# dynamic topic route by schema or table regex +#canal.mq.dynamicTopic=mytest1.user,topic2:mytest2\\..*,.*\\..* +canal.mq.partition=0 +# hash partition config +#canal.mq.enableDynamicQueuePartition=false +#canal.mq.partitionsNum=3 +#canal.mq.dynamicTopicPartitionNum=test.*:4,mycanal:6 +#canal.mq.partitionHash=test.table:id^name,.*\\..* +################################################# diff --git a/smtweb-framework/canal/server/src/main/resources/logback.xml.bak b/smtweb-framework/canal/server/src/main/resources/logback.xml.bak new file mode 100644 index 0000000..682e80f --- /dev/null +++ b/smtweb-framework/canal/server/src/main/resources/logback.xml.bak @@ -0,0 +1,117 @@ + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n + + + + + + + destination + canal + + + + ../logs/${destination}/${destination}.log + + + ../logs/${destination}/%d{yyyy-MM-dd}/${destination}-%d{yyyy-MM-dd}-%i.log.gz + + + 512MB + + 60 + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n + + + + + + + + + destination + canal + + + + ../logs/${destination}/meta.log + + + ../logs/${destination}/%d{yyyy-MM-dd}/meta-%d{yyyy-MM-dd}-%i.log.gz + + + 32MB + + 60 + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} - %msg%n + + + + + + + + ../logs/canal/rocketmq_client.log + + + ../logs/canal/%d{yyyy-MM-dd}/rocketmq_client-%d{yyyy-MM-dd}-%i.log.gz + + + 512MB + + 60 + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/smtweb-framework/canal/server/src/main/resources/metrics/Canal_instances_tmpl.json b/smtweb-framework/canal/server/src/main/resources/metrics/Canal_instances_tmpl.json new file mode 100644 index 0000000..89fd332 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/resources/metrics/Canal_instances_tmpl.json @@ -0,0 +1,1375 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.2.2" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1536989235272, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 30, + "panels": [], + "title": "Instance status", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Canal instance 基本信息。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 500, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "canal_instance{destination=~\"$destination\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "Destination: {{destination}}", + "refId": "A" + }, + { + "expr": "canal_instance_parser_mode{destination=~\"$destination\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "Parallel parser: {{parallel}}", + "refId": "B" + }, + { + "expr": "canal_instance_store{destination=~\"$destination\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "Batch mode: {{batchMode}}", + "refId": "C" + }, + { + "expr": "canal_instance_store{destination=~\"$destination\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "Buffer size: {{size}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Basic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": true, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": false, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "inbound": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Canal instance 网络带宽占用。\ninbound: 读取MySQL binlog.\noutbound: 对Client端传输格式化binlog.", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"0\"}[2m]) / 1024", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "inbound", + "refId": "A" + }, + { + "expr": "rate(canal_instance_client_bytes{destination=~\"$destination\"}[2m]) / 1024", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "outbound", + "refId": "B" + }, + { + "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"1\"}[2m]) / 1024", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "inbound-1", + "refId": "C" + }, + { + "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"2\"}[2m]) / 1024", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "inbound-2", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network bandwith", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "KBs", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "ack": "#f29191", + "get": "#cca300", + "put": "#1f78c1" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "master: Canal server相对于MySQL master的延时。通过master heartbeat机制能刷新idle状态下的延时。\nput: store put操作的时间点为基准。\nget: client get操作的时间点为基准。\nack: client ack操作的时间点为基准。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "canal_instance_traffic_delay{destination=~\"$destination\"} / 1000", + "format": "time_series", + "hide": false, + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "master", + "refId": "D" + }, + { + "expr": "canal_instance_put_delay{destination=~\"$destination\"} / 1000", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "put", + "refId": "A" + }, + { + "expr": "canal_instance_get_delay{destination=~\"$destination\"} / 1000", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "get", + "refId": "B" + }, + { + "expr": "canal_instance_ack_delay{destination=~\"$destination\"} / 1000", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "ack", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Delay", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "sink线程blocking占比；dump线程blocking占比(仅parallel mode)。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "hideTimeOverride": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"0\"}[2m]), 1000) / 10", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "dump", + "refId": "B" + }, + { + "expr": "clamp_max(rate(canal_instance_sink_blocking_time{destination=~\"$destination\"}[2m]), 1000) / 10", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "sink", + "refId": "A" + }, + { + "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"1\"}[2m]), 1000) / 10", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "dump-1", + "refId": "C" + }, + { + "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"2\"}[2m]), 1000) / 10", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "dump-2", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Blocking", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 32, + "panels": [], + "title": "Throughput", + "type": "row" + }, + { + "aliasColors": { + "rowDatas": "#7eb26d", + "tableRows": "#c15c17" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Instance处理binlog的TPS(以master变更行数table rows为基准计算)。\nput: put操作TPS。\nget: get操作TPS。\nack: ack操作TPS。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(canal_instance_put_rows{destination=~\"$destination\"}[2m])", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "put", + "refId": "A" + }, + { + "expr": "rate(canal_instance_get_rows{destination=~\"$destination\"}[2m])", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "get", + "refId": "B" + }, + { + "expr": "rate(canal_instance_ack_rows{destination=~\"$destination\"}[2m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "ack", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TPS(table rows)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "transactions": "#f9ba8f" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Canal instance 处理binlog的TPS，以MySQL transaction为单位计算。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(canal_instance_transactions{destination=~\"$destination\"}[2m])", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "transactions", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TPS(MySQL transaction)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 34, + "panels": [], + "title": "Client", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Canal instance接收到的请求统计，结果按packet type分类。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 13 + }, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "canal_instance_client_packets{destination=~\"$destination\"}", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{packetType}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "client 请求的GET与ACK包的QPS。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 13 + }, + "id": 38, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(canal_instance_client_packets{destination=~\"$destination\",packetType=\"GET\"}[2m])", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "GET", + "refId": "A" + }, + { + "expr": "rate(canal_instance_client_packets{destination=~\"$destination\",packetType=\"CLIENTACK\"}[2m])", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "ACK", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client QPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "server响应GET请求，但返回空包的占比。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 13 + }, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(canal_instance_client_empty_batches{destination=~\"$destination\"}[2m])", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "empty", + "refId": "A" + }, + { + "expr": "rate(canal_instance_client_packets{destination=~\"$destination\", packetType=\"GET\"}[2m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "nonempty", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Empty packets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "wps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Canal client 请求响应时间的概况。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 18, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "25.0", + "yaxis": 1 + }, + { + "alias": "100.0", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(canal_instance_client_request_latency_bucket{destination=~\"$destination\"}[2m])", + "format": "time_series", + "hide": false, + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{le}}ms", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Response time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 36, + "panels": [], + "title": "Store", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Canal instance ringbuffer内未释放的events数量。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 19 + }, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "canal_instance_store_produce_seq{destination=~\"$destination\"} - canal_instance_store_consume_seq{destination=~\"$destination\"}", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "events", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Store remain events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Canal instance ringbuffer 内未释放events占用内存。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 19 + }, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(canal_instance_store_produce_mem{destination=~\"$destination\"} - canal_instance_store_consume_mem{destination=~\"$destination\"}) / 1024", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "memsize", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Store remain mem", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "deckbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 16, + "style": "dark", + "tags": [ + "canal" + ], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "label": "datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "destination", + "multi": false, + "name": "destination", + "options": [], + "query": "label_values(canal_instance, destination)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Canal instances", + "uid": "8vh8NGpiz", + "version": 103 +} \ No newline at end of file diff --git a/smtweb-framework/canal/server/src/main/resources/spring/base-instance.xml b/smtweb-framework/canal/server/src/main/resources/spring/base-instance.xml new file mode 100644 index 0000000..a89b740 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/resources/spring/base-instance.xml @@ -0,0 +1,39 @@ + + + + + + + + + + classpath:canal.properties + classpath:${canal.instance.destination:}/instance.properties + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/smtweb-framework/canal/server/src/main/resources/spring/default-instance.xml b/smtweb-framework/canal/server/src/main/resources/spring/default-instance.xml new file mode 100644 index 0000000..9907ae8 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/resources/spring/default-instance.xml @@ -0,0 +1,211 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${canal.zkServers:127.0.0.1:2181} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/smtweb-framework/canal/server/src/main/resources/spring/file-instance.xml b/smtweb-framework/canal/server/src/main/resources/spring/file-instance.xml new file mode 100644 index 0000000..800f981 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/resources/spring/file-instance.xml @@ -0,0 +1,197 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/smtweb-framework/canal/server/src/main/resources/spring/group-instance.xml b/smtweb-framework/canal/server/src/main/resources/spring/group-instance.xml new file mode 100644 index 0000000..b0b887f --- /dev/null +++ b/smtweb-framework/canal/server/src/main/resources/spring/group-instance.xml @@ -0,0 +1,292 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/smtweb-framework/canal/server/src/main/resources/spring/memory-instance.xml b/smtweb-framework/canal/server/src/main/resources/spring/memory-instance.xml new file mode 100644 index 0000000..a7dc634 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/resources/spring/memory-instance.xml @@ -0,0 +1,185 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/smtweb-framework/canal/server/src/main/resources/spring/tsdb/h2-tsdb.xml b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/h2-tsdb.xml new file mode 100644 index 0000000..29c7a24 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/h2-tsdb.xml @@ -0,0 +1,61 @@ + + + + + + + + + + classpath:canal.properties + classpath:${canal.instance.destination:}/instance.properties + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/smtweb-framework/canal/server/src/main/resources/spring/tsdb/mysql-tsdb.xml b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/mysql-tsdb.xml new file mode 100644 index 0000000..72d82bf --- /dev/null +++ b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/mysql-tsdb.xml @@ -0,0 +1,63 @@ + + + + + + + + + + classpath:canal.properties + classpath:${canal.instance.destination:}/instance.properties + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap-config.xml b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap-config.xml new file mode 100644 index 0000000..c27d463 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap-config.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + diff --git a/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml new file mode 100644 index 0000000..206c712 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + insert into meta_history () + values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#{destination},#{binlogFile},#{binlogOffest},#{binlogMasterId},#{binlogTimestamp},#{useSchema},#{sqlSchema},#{sqlTable},#{sqlText},#{sqlType},#{extra}) + + + + delete from meta_history + where destination=#{destination} + + + + + + + \ No newline at end of file diff --git a/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml new file mode 100644 index 0000000..a4d1466 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + insert into meta_snapshot () + values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#{destination},#{binlogFile},#{binlogOffest},#{binlogMasterId},#{binlogTimestamp},#{data},#{extra}) + + + + update meta_snapshot set gmt_modified=now(), + binlog_file=#{binlogFile},binlog_offest=#{binlogOffest},binlog_master_id=#{binlogMasterId},binlog_timestamp=#{binlogTimestamp},data=#{data},extra=#{extra} + where destination=#{destination} and binlog_timestamp=0 + + + + delete from meta_snapshot + where destination=#{destination} + + + + 0 + ]]> + + \ No newline at end of file diff --git a/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql/create_table.sql b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql/create_table.sql new file mode 100644 index 0000000..8cb82c8 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql/create_table.sql @@ -0,0 +1,39 @@ +CREATE TABLE IF NOT EXISTS `meta_snapshot` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', + `gmt_create` datetime NOT NULL COMMENT '创建时间', + `gmt_modified` datetime NOT NULL COMMENT '修改时间', + `destination` varchar(128) DEFAULT NULL COMMENT '通道名称', + `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名', + `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量', + `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id', + `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳', + `data` longtext DEFAULT NULL COMMENT '表结构数据', + `extra` text DEFAULT NULL COMMENT '额外的扩展信息', + PRIMARY KEY (`id`), + UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`), + KEY `destination` (`destination`), + KEY `destination_timestamp` (`destination`,`binlog_timestamp`), + KEY `gmt_modified` (`gmt_modified`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='表结构记录表快照表'; + +CREATE TABLE IF NOT EXISTS `meta_history` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', + `gmt_create` datetime NOT NULL COMMENT '创建时间', + `gmt_modified` datetime NOT NULL COMMENT '修改时间', + `destination` varchar(128) DEFAULT NULL COMMENT '通道名称', + `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名', + `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量', + `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id', + `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳', + `use_schema` varchar(1024) DEFAULT NULL COMMENT '执行sql时对应的schema', + `sql_schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema', + `sql_table` varchar(1024) DEFAULT NULL COMMENT '对应的table', + `sql_text` longtext DEFAULT NULL COMMENT '执行的sql', + `sql_type` varchar(256) DEFAULT NULL COMMENT 'sql类型', + `extra` text DEFAULT NULL COMMENT '额外的扩展信息', + PRIMARY KEY (`id`), + UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`), + KEY `destination` (`destination`), + KEY `destination_timestamp` (`destination`,`binlog_timestamp`), + KEY `gmt_modified` (`gmt_modified`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='表结构变化明细表'; \ No newline at end of file diff --git a/smtweb-framework/canal/server/target/classes/canal.properties b/smtweb-framework/canal/server/target/classes/canal.properties new file mode 100644 index 0000000..82992bc --- /dev/null +++ b/smtweb-framework/canal/server/target/classes/canal.properties @@ -0,0 +1,182 @@ +################################################# +######### common argument ############# +################################################# +# tcp bind ip +canal.ip = +# register ip to zookeeper +canal.register.ip = +canal.port = 11111 +canal.metrics.pull.port = 11112 +# canal instance user/passwd +# canal.user = canal +# canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458 + +# canal admin config +#canal.admin.manager = 127.0.0.1:8089 +canal.admin.port = 11110 +canal.admin.user = admin +canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441 +# admin auto register +#canal.admin.register.auto = true +#canal.admin.register.cluster = +#canal.admin.register.name = + +canal.zkServers = +# flush data to zk +canal.zookeeper.flush.period = 1000 +canal.withoutNetty = false +# tcp, kafka, rocketMQ, rabbitMQ, pulsarMQ +canal.serverMode = tcp +# flush meta cursor/parse position to file +canal.file.data.dir = ${canal.conf.dir} +canal.file.flush.period = 1000 +## memory store RingBuffer size, should be Math.pow(2,n) +canal.instance.memory.buffer.size = 16384 +## memory store RingBuffer used memory unit size , default 1kb +canal.instance.memory.buffer.memunit = 1024 +## meory store gets mode used MEMSIZE or ITEMSIZE +canal.instance.memory.batch.mode = MEMSIZE +canal.instance.memory.rawEntry = true + +## detecing config +canal.instance.detecting.enable = false +#canal.instance.detecting.sql = insert into retl.xdual values(1,now()) on duplicate key update x=now() +canal.instance.detecting.sql = select 1 +canal.instance.detecting.interval.time = 3 +canal.instance.detecting.retry.threshold = 3 +canal.instance.detecting.heartbeatHaEnable = false + +# support maximum transaction size, more than the size of the transaction will be cut into multiple transactions delivery +canal.instance.transaction.size = 1024 +# mysql fallback connected to new master should fallback times +canal.instance.fallbackIntervalInSeconds = 60 + +# network config +canal.instance.network.receiveBufferSize = 16384 +canal.instance.network.sendBufferSize = 16384 +canal.instance.network.soTimeout = 30 + +# binlog filter config +canal.instance.filter.druid.ddl = true +canal.instance.filter.query.dcl = false +canal.instance.filter.query.dml = false +canal.instance.filter.query.ddl = false +canal.instance.filter.table.error = false +canal.instance.filter.rows = false +canal.instance.filter.transaction.entry = false +canal.instance.filter.dml.insert = false +canal.instance.filter.dml.update = false +canal.instance.filter.dml.delete = false + +# binlog format/image check +canal.instance.binlog.format = ROW,STATEMENT,MIXED +canal.instance.binlog.image = FULL,MINIMAL,NOBLOB + +# binlog ddl isolation +canal.instance.get.ddl.isolation = false + +# parallel parser config +canal.instance.parser.parallel = true +## concurrent thread number, default 60% available processors, suggest not to exceed Runtime.getRuntime().availableProcessors() +#canal.instance.parser.parallelThreadSize = 16 +## disruptor ringbuffer size, must be power of 2 +canal.instance.parser.parallelBufferSize = 256 + +# table meta tsdb info +canal.instance.tsdb.enable = true +canal.instance.tsdb.dir = ${canal.file.data.dir:../conf}/${canal.instance.destination:} +canal.instance.tsdb.url = jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL; +canal.instance.tsdb.dbUsername = canal +canal.instance.tsdb.dbPassword = canal +# dump snapshot interval, default 24 hour +canal.instance.tsdb.snapshot.interval = 24 +# purge snapshot expire , default 360 hour(15 days) +canal.instance.tsdb.snapshot.expire = 360 + +################################################# +######### destinations ############# +################################################# +canal.destinations = example +# conf root dir +canal.conf.dir = ../conf +# auto scan instance dir add/remove and start/stop instance +canal.auto.scan = true +canal.auto.scan.interval = 5 +# set this value to 'true' means that when binlog pos not found, skip to latest. +# WARN: pls keep 'false' in production env, or if you know what you want. +canal.auto.reset.latest.pos.mode = false + +canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml +#canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml + +canal.instance.global.mode = spring +canal.instance.global.lazy = false +canal.instance.global.manager.address = ${canal.admin.manager} +#canal.instance.global.spring.xml = classpath:spring/memory-instance.xml +canal.instance.global.spring.xml = classpath:spring/file-instance.xml +#canal.instance.global.spring.xml = classpath:spring/default-instance.xml + +################################################## +######### MQ Properties ############# +################################################## +# aliyun ak/sk , support rds/mq +canal.aliyun.accessKey = +canal.aliyun.secretKey = +canal.aliyun.uid= + +canal.mq.flatMessage = true +canal.mq.canalBatchSize = 50 +canal.mq.canalGetTimeout = 100 +# Set this value to "cloud", if you want open message trace feature in aliyun. +canal.mq.accessChannel = local + +canal.mq.database.hash = true +canal.mq.send.thread.size = 30 +canal.mq.build.thread.size = 8 + +################################################## +######### Kafka ############# +################################################## +kafka.bootstrap.servers = 127.0.0.1:9092 +kafka.acks = all +kafka.compression.type = none +kafka.batch.size = 16384 +kafka.linger.ms = 1 +kafka.max.request.size = 1048576 +kafka.buffer.memory = 33554432 +kafka.max.in.flight.requests.per.connection = 1 +kafka.retries = 0 + +kafka.kerberos.enable = false +kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf" +kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf" + +################################################## +######### RocketMQ ############# +################################################## +rocketmq.producer.group = test +rocketmq.enable.message.trace = false +rocketmq.customized.trace.topic = +rocketmq.namespace = +rocketmq.namesrv.addr = 127.0.0.1:9876 +rocketmq.retry.times.when.send.failed = 0 +rocketmq.vip.channel.enabled = false +rocketmq.tag = + +################################################## +######### RabbitMQ ############# +################################################## +rabbitmq.host = +rabbitmq.virtual.host = +rabbitmq.exchange = +rabbitmq.username = +rabbitmq.password = +rabbitmq.deliveryMode = + + +################################################## +######### Pulsar ############# +################################################## +pulsarmq.serverUrl = +pulsarmq.roleToken = +pulsarmq.topicTenantPrefix = \ No newline at end of file diff --git a/smtweb-framework/canal/server/target/classes/canal_local.properties b/smtweb-framework/canal/server/target/classes/canal_local.properties new file mode 100644 index 0000000..f92b8f3 --- /dev/null +++ b/smtweb-framework/canal/server/target/classes/canal_local.properties @@ -0,0 +1,12 @@ +# register ip +canal.register.ip = + +# canal admin config +canal.admin.manager = 127.0.0.1:8089 +canal.admin.port = 11110 +canal.admin.user = admin +canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441 +# admin auto register +canal.admin.register.auto = true +canal.admin.register.cluster = +canal.admin.register.name = \ No newline at end of file diff --git a/smtweb-framework/canal/server/target/classes/example/instance.properties b/smtweb-framework/canal/server/target/classes/example/instance.properties new file mode 100644 index 0000000..7a1f752 --- /dev/null +++ b/smtweb-framework/canal/server/target/classes/example/instance.properties @@ -0,0 +1,59 @@ +################################################# +## mysql serverId , v1.0.26+ will autoGen +canal.instance.mysql.slaveId=2 + +# enable gtid use true/false +canal.instance.gtidon=false + +# position info +canal.instance.master.address=172.28.123.25:3306 +#canal.instance.master.journal.name= +#canal.instance.master.position= +#canal.instance.master.timestamp= +#canal.instance.master.gtid= + +# rds oss binlog +canal.instance.rds.accesskey= +canal.instance.rds.secretkey= +canal.instance.rds.instanceId= + +# table meta tsdb info +canal.instance.tsdb.enable=true +#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb +#canal.instance.tsdb.dbUsername=canal +#canal.instance.tsdb.dbPassword=canal + +#canal.instance.standby.address = +#canal.instance.standby.journal.name = +#canal.instance.standby.position = +#canal.instance.standby.timestamp = +#canal.instance.standby.gtid= + +# username/password +canal.instance.dbUsername=root +canal.instance.dbPassword=root +canal.instance.connectionCharset = UTF-8 +# enable druid Decrypt database password +canal.instance.enableDruid=false +#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ== + +# table regex +canal.instance.filter.regex=scmz\\..* +# table black regex +canal.instance.filter.black.regex=mysql\\.slave_.* +# table field filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2) +#canal.instance.filter.field=test1.t_product:id/subject/keywords,test2.t_company:id/name/contact/ch +# table field black filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2) +#canal.instance.filter.black.field=test1.t_product:subject/product_image,test2.t_company:id/name/contact/ch + +# mq config +canal.mq.topic=example +# dynamic topic route by schema or table regex +#canal.mq.dynamicTopic=mytest1.user,topic2:mytest2\\..*,.*\\..* +canal.mq.partition=0 +# hash partition config +#canal.mq.enableDynamicQueuePartition=false +#canal.mq.partitionsNum=3 +#canal.mq.dynamicTopicPartitionNum=test.*:4,mycanal:6 +#canal.mq.partitionHash=test.table:id^name,.*\\..* +################################################# diff --git a/smtweb-framework/canal/server/target/classes/logback.xml.bak b/smtweb-framework/canal/server/target/classes/logback.xml.bak new file mode 100644 index 0000000..682e80f --- /dev/null +++ b/smtweb-framework/canal/server/target/classes/logback.xml.bak @@ -0,0 +1,117 @@ + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n + + + + + + + destination + canal + + + + ../logs/${destination}/${destination}.log + + + ../logs/${destination}/%d{yyyy-MM-dd}/${destination}-%d{yyyy-MM-dd}-%i.log.gz + + + 512MB + + 60 + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n + + + + + + + + + destination + canal + + + + ../logs/${destination}/meta.log + + + ../logs/${destination}/%d{yyyy-MM-dd}/meta-%d{yyyy-MM-dd}-%i.log.gz + + + 32MB + + 60 + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} - %msg%n + + + + + + + + ../logs/canal/rocketmq_client.log + + + ../logs/canal/%d{yyyy-MM-dd}/rocketmq_client-%d{yyyy-MM-dd}-%i.log.gz + + + 512MB + + 60 + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/smtweb-framework/canal/server/target/classes/metrics/Canal_instances_tmpl.json b/smtweb-framework/canal/server/target/classes/metrics/Canal_instances_tmpl.json new file mode 100644 index 0000000..89fd332 --- /dev/null +++ b/smtweb-framework/canal/server/target/classes/metrics/Canal_instances_tmpl.json @@ -0,0 +1,1375 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.2.2" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1536989235272, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 30, + "panels": [], + "title": "Instance status", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Canal instance 基本信息。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 500, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "canal_instance{destination=~\"$destination\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "Destination: {{destination}}", + "refId": "A" + }, + { + "expr": "canal_instance_parser_mode{destination=~\"$destination\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "Parallel parser: {{parallel}}", + "refId": "B" + }, + { + "expr": "canal_instance_store{destination=~\"$destination\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "Batch mode: {{batchMode}}", + "refId": "C" + }, + { + "expr": "canal_instance_store{destination=~\"$destination\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "Buffer size: {{size}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Basic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": true, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": false, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "inbound": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Canal instance 网络带宽占用。\ninbound: 读取MySQL binlog.\noutbound: 对Client端传输格式化binlog.", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"0\"}[2m]) / 1024", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "inbound", + "refId": "A" + }, + { + "expr": "rate(canal_instance_client_bytes{destination=~\"$destination\"}[2m]) / 1024", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "outbound", + "refId": "B" + }, + { + "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"1\"}[2m]) / 1024", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "inbound-1", + "refId": "C" + }, + { + "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"2\"}[2m]) / 1024", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "inbound-2", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network bandwith", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "KBs", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "ack": "#f29191", + "get": "#cca300", + "put": "#1f78c1" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "master: Canal server相对于MySQL master的延时。通过master heartbeat机制能刷新idle状态下的延时。\nput: store put操作的时间点为基准。\nget: client get操作的时间点为基准。\nack: client ack操作的时间点为基准。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "canal_instance_traffic_delay{destination=~\"$destination\"} / 1000", + "format": "time_series", + "hide": false, + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "master", + "refId": "D" + }, + { + "expr": "canal_instance_put_delay{destination=~\"$destination\"} / 1000", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "put", + "refId": "A" + }, + { + "expr": "canal_instance_get_delay{destination=~\"$destination\"} / 1000", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "get", + "refId": "B" + }, + { + "expr": "canal_instance_ack_delay{destination=~\"$destination\"} / 1000", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "ack", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Delay", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "sink线程blocking占比；dump线程blocking占比(仅parallel mode)。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "hideTimeOverride": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"0\"}[2m]), 1000) / 10", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "dump", + "refId": "B" + }, + { + "expr": "clamp_max(rate(canal_instance_sink_blocking_time{destination=~\"$destination\"}[2m]), 1000) / 10", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "sink", + "refId": "A" + }, + { + "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"1\"}[2m]), 1000) / 10", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "dump-1", + "refId": "C" + }, + { + "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"2\"}[2m]), 1000) / 10", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "dump-2", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Blocking", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 32, + "panels": [], + "title": "Throughput", + "type": "row" + }, + { + "aliasColors": { + "rowDatas": "#7eb26d", + "tableRows": "#c15c17" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Instance处理binlog的TPS(以master变更行数table rows为基准计算)。\nput: put操作TPS。\nget: get操作TPS。\nack: ack操作TPS。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(canal_instance_put_rows{destination=~\"$destination\"}[2m])", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "put", + "refId": "A" + }, + { + "expr": "rate(canal_instance_get_rows{destination=~\"$destination\"}[2m])", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "get", + "refId": "B" + }, + { + "expr": "rate(canal_instance_ack_rows{destination=~\"$destination\"}[2m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "ack", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TPS(table rows)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "transactions": "#f9ba8f" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Canal instance 处理binlog的TPS，以MySQL transaction为单位计算。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(canal_instance_transactions{destination=~\"$destination\"}[2m])", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "transactions", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TPS(MySQL transaction)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 34, + "panels": [], + "title": "Client", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Canal instance接收到的请求统计，结果按packet type分类。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 13 + }, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "canal_instance_client_packets{destination=~\"$destination\"}", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{packetType}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "client 请求的GET与ACK包的QPS。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 13 + }, + "id": 38, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(canal_instance_client_packets{destination=~\"$destination\",packetType=\"GET\"}[2m])", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "GET", + "refId": "A" + }, + { + "expr": "rate(canal_instance_client_packets{destination=~\"$destination\",packetType=\"CLIENTACK\"}[2m])", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "ACK", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client QPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "server响应GET请求，但返回空包的占比。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 13 + }, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(canal_instance_client_empty_batches{destination=~\"$destination\"}[2m])", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "empty", + "refId": "A" + }, + { + "expr": "rate(canal_instance_client_packets{destination=~\"$destination\", packetType=\"GET\"}[2m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "nonempty", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Empty packets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "wps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Canal client 请求响应时间的概况。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 18, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "25.0", + "yaxis": 1 + }, + { + "alias": "100.0", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(canal_instance_client_request_latency_bucket{destination=~\"$destination\"}[2m])", + "format": "time_series", + "hide": false, + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{le}}ms", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Response time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 36, + "panels": [], + "title": "Store", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Canal instance ringbuffer内未释放的events数量。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 19 + }, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "canal_instance_store_produce_seq{destination=~\"$destination\"} - canal_instance_store_consume_seq{destination=~\"$destination\"}", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "events", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Store remain events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Canal instance ringbuffer 内未释放events占用内存。", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 19 + }, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(canal_instance_store_produce_mem{destination=~\"$destination\"} - canal_instance_store_consume_mem{destination=~\"$destination\"}) / 1024", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "memsize", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Store remain mem", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "deckbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 16, + "style": "dark", + "tags": [ + "canal" + ], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "label": "datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "destination", + "multi": false, + "name": "destination", + "options": [], + "query": "label_values(canal_instance, destination)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Canal instances", + "uid": "8vh8NGpiz", + "version": 103 +} \ No newline at end of file diff --git a/smtweb-framework/canal/server/target/classes/spring/base-instance.xml b/smtweb-framework/canal/server/target/classes/spring/base-instance.xml new file mode 100644 index 0000000..a89b740 --- /dev/null +++ b/smtweb-framework/canal/server/target/classes/spring/base-instance.xml @@ -0,0 +1,39 @@ + + + + + + + + + + classpath:canal.properties + classpath:${canal.instance.destination:}/instance.properties + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/smtweb-framework/canal/server/target/classes/spring/default-instance.xml b/smtweb-framework/canal/server/target/classes/spring/default-instance.xml new file mode 100644 index 0000000..9907ae8 --- /dev/null +++ b/smtweb-framework/canal/server/target/classes/spring/default-instance.xml @@ -0,0 +1,211 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${canal.zkServers:127.0.0.1:2181} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/smtweb-framework/canal/server/target/classes/spring/file-instance.xml b/smtweb-framework/canal/server/target/classes/spring/file-instance.xml new file mode 100644 index 0000000..800f981 --- /dev/null +++ b/smtweb-framework/canal/server/target/classes/spring/file-instance.xml @@ -0,0 +1,197 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/smtweb-framework/canal/server/target/classes/spring/group-instance.xml b/smtweb-framework/canal/server/target/classes/spring/group-instance.xml new file mode 100644 index 0000000..b0b887f --- /dev/null +++ b/smtweb-framework/canal/server/target/classes/spring/group-instance.xml @@ -0,0 +1,292 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/smtweb-framework/canal/server/target/classes/spring/memory-instance.xml b/smtweb-framework/canal/server/target/classes/spring/memory-instance.xml new file mode 100644 index 0000000..a7dc634 --- /dev/null +++ b/smtweb-framework/canal/server/target/classes/spring/memory-instance.xml @@ -0,0 +1,185 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/smtweb-framework/canal/server/target/classes/spring/tsdb/h2-tsdb.xml b/smtweb-framework/canal/server/target/classes/spring/tsdb/h2-tsdb.xml new file mode 100644 index 0000000..29c7a24 --- /dev/null +++ b/smtweb-framework/canal/server/target/classes/spring/tsdb/h2-tsdb.xml @@ -0,0 +1,61 @@ + + + + + + + + + + classpath:canal.properties + classpath:${canal.instance.destination:}/instance.properties + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/smtweb-framework/canal/server/target/classes/spring/tsdb/mysql-tsdb.xml b/smtweb-framework/canal/server/target/classes/spring/tsdb/mysql-tsdb.xml new file mode 100644 index 0000000..72d82bf --- /dev/null +++ b/smtweb-framework/canal/server/target/classes/spring/tsdb/mysql-tsdb.xml @@ -0,0 +1,63 @@ + + + + + + + + + + classpath:canal.properties + classpath:${canal.instance.destination:}/instance.properties + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap-config.xml b/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap-config.xml new file mode 100644 index 0000000..c27d463 --- /dev/null +++ b/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap-config.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + diff --git a/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_history.xml b/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_history.xml new file mode 100644 index 0000000..206c712 --- /dev/null +++ b/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_history.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + insert into meta_history () + values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#{destination},#{binlogFile},#{binlogOffest},#{binlogMasterId},#{binlogTimestamp},#{useSchema},#{sqlSchema},#{sqlTable},#{sqlText},#{sqlType},#{extra}) + + + + delete from meta_history + where destination=#{destination} + + + + + + + \ No newline at end of file diff --git a/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_snapshot.xml b/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_snapshot.xml new file mode 100644 index 0000000..a4d1466 --- /dev/null +++ b/smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_snapshot.xml @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + insert into meta_snapshot () + values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#{destination},#{binlogFile},#{binlogOffest},#{binlogMasterId},#{binlogTimestamp},#{data},#{extra}) + + + + update meta_snapshot set gmt_modified=now(), + binlog_file=#{binlogFile},binlog_offest=#{binlogOffest},binlog_master_id=#{binlogMasterId},binlog_timestamp=#{binlogTimestamp},data=#{data},extra=#{extra} + where destination=#{destination} and binlog_timestamp=0 + + + + delete from meta_snapshot + where destination=#{destination} + + + + 0 + ]]> + + \ No newline at end of file diff --git a/smtweb-framework/canal/server/target/classes/spring/tsdb/sql/create_table.sql b/smtweb-framework/canal/server/target/classes/spring/tsdb/sql/create_table.sql new file mode 100644 index 0000000..8cb82c8 --- /dev/null +++ b/smtweb-framework/canal/server/target/classes/spring/tsdb/sql/create_table.sql @@ -0,0 +1,39 @@ +CREATE TABLE IF NOT EXISTS `meta_snapshot` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', + `gmt_create` datetime NOT NULL COMMENT '创建时间', + `gmt_modified` datetime NOT NULL COMMENT '修改时间', + `destination` varchar(128) DEFAULT NULL COMMENT '通道名称', + `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名', + `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量', + `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id', + `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳', + `data` longtext DEFAULT NULL COMMENT '表结构数据', + `extra` text DEFAULT NULL COMMENT '额外的扩展信息', + PRIMARY KEY (`id`), + UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`), + KEY `destination` (`destination`), + KEY `destination_timestamp` (`destination`,`binlog_timestamp`), + KEY `gmt_modified` (`gmt_modified`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='表结构记录表快照表'; + +CREATE TABLE IF NOT EXISTS `meta_history` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', + `gmt_create` datetime NOT NULL COMMENT '创建时间', + `gmt_modified` datetime NOT NULL COMMENT '修改时间', + `destination` varchar(128) DEFAULT NULL COMMENT '通道名称', + `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名', + `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量', + `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id', + `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳', + `use_schema` varchar(1024) DEFAULT NULL COMMENT '执行sql时对应的schema', + `sql_schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema', + `sql_table` varchar(1024) DEFAULT NULL COMMENT '对应的table', + `sql_text` longtext DEFAULT NULL COMMENT '执行的sql', + `sql_type` varchar(256) DEFAULT NULL COMMENT 'sql类型', + `extra` text DEFAULT NULL COMMENT '额外的扩展信息', + PRIMARY KEY (`id`), + UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`), + KEY `destination` (`destination`), + KEY `destination_timestamp` (`destination`,`binlog_timestamp`), + KEY `gmt_modified` (`gmt_modified`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='表结构变化明细表'; \ No newline at end of file diff --git a/smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst b/smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst new file mode 100644 index 0000000..a3c448f --- /dev/null +++ b/smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst @@ -0,0 +1,22 @@ +cc\smtweb\system\canal\CanalController.class +cc\smtweb\system\canal\CanalConstants.class +cc\smtweb\system\canal\InstanceConfig.class +cc\smtweb\system\canal\CanalController$1.class +cc\smtweb\system\canal\monitor\SpringInstanceConfigMonitor.class +cc\smtweb\system\canal\CanalLauncher.class +cc\smtweb\system\canal\monitor\SpringInstanceConfigMonitor$FileInfo.class +cc\smtweb\system\canal\client\CanalClient.class +cc\smtweb\system\canal\CanalController$3.class +cc\smtweb\system\canal\admin\CanalAdminController.class +cc\smtweb\system\canal\CanalLauncher$1.class +cc\smtweb\system\canal\monitor\InstanceConfigMonitor.class +cc\smtweb\system\canal\CanalStarter.class +cc\smtweb\system\canal\CanalApplication.class +cc\smtweb\system\canal\CanalController$1$1.class +cc\smtweb\system\canal\monitor\InstanceAction.class +cc\smtweb\system\canal\client\CanalClient$1.class +cc\smtweb\system\canal\CanalController$2.class +cc\smtweb\system\canal\monitor\SpringInstanceConfigMonitor$InstanceConfigFiles.class +cc\smtweb\system\canal\client\CanalVO.class +cc\smtweb\system\canal\InstanceConfig$InstanceMode.class +cc\smtweb\system\canal\monitor\ManagerInstanceConfigMonitor.class diff --git a/smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst b/smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst new file mode 100644 index 0000000..f87c00f --- /dev/null +++ b/smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst @@ -0,0 +1,13 @@ +E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\CanalConstants.java +E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\CanalStarter.java +E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\InstanceConfig.java +E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\CanalApplication.java +E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\CanalLauncher.java +E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\client\CanalClient.java +E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\monitor\InstanceAction.java +E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\monitor\InstanceConfigMonitor.java +E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\CanalController.java +E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\monitor\ManagerInstanceConfigMonitor.java +E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\admin\CanalAdminController.java +E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\monitor\SpringInstanceConfigMonitor.java +E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\client\CanalVO.java diff --git a/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadWorker.java b/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadWorker.java index bf79f7e..8d84183 100644 --- a/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadWorker.java +++ b/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadWorker.java @@ -6,20 +6,14 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public abstract class SysThreadWorker implements Runnable { - private String loginPartyId; - - private String loginOwnerPartyId; private String bizName; - protected SysThreadWorker(String loginPartyId, String loginOwnerPartyId) { - this.loginPartyId = loginPartyId; - this.loginOwnerPartyId = loginOwnerPartyId; + protected SysThreadWorker() { + } - protected SysThreadWorker(String loginPartyId, String loginOwnerPartyId, String bizName) { - this.loginPartyId = loginPartyId; - this.loginOwnerPartyId = loginOwnerPartyId; + protected SysThreadWorker(String bizName) { this.bizName = StringUtil.checkNull(bizName); } @@ -27,8 +21,6 @@ public abstract class SysThreadWorker implements Runnable { @Override public void run() { try { -// SysParams.setLoginOwnerPartyId(loginOwnerPartyId); -// SysParams.setLoginPartyId(loginPartyId); localWork(); } catch (Exception e) { log.error("DfpPool handle error：", e);