From d7aad13dcc49b3402864c683989b24392d2061fc Mon Sep 17 00:00:00 2001 From: yaoq Date: Wed, 14 Sep 2022 16:15:37 +0800 Subject: [PATCH] =?UTF-8?q?canal=20=E9=9B=86=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- smtweb-framework/bpm/pom.xml | 4 +- .../smtweb/system/bpm/web/BpmStartedListener.java | 4 +- smtweb-framework/canal/client/canal.example.iml | 168 ------ smtweb-framework/canal/client/pom.xml | 2 +- .../canal/client/src/main/assembly/dev.xml | 54 ++ .../canal/client/src/main/assembly/release.xml | 54 ++ .../canal/client/src/main/bin/restart.sh | 15 + .../canal/client/src/main/bin/startup.bat | 23 + .../canal/client/src/main/bin/startup.sh | 105 ++++ smtweb-framework/canal/client/src/main/bin/stop.sh | 53 ++ .../smtweb/system/canal/client/ClientConsts.java | 76 --- .../smtweb/system/canal/client/ClientInstance.java | 290 ---------- .../system/canal/client/ClientStartedListener.java | 32 -- .../cc/smtweb/system/canal/client/ClientVO.java | 56 -- .../system/canal/example/ClientApplication.java | 13 + .../system/canal/example/ClientConfiguration.java | 21 + .../smtweb/system/canal/example/ClientConsts.java | 76 +++ .../system/canal/example/ClientInstance.java | 290 ++++++++++ .../system/canal/example/ClientLauncher.java | 14 + .../canal/example/ClientStartedListener.java | 37 ++ .../cc/smtweb/system/canal/example/ClientVO.java | 56 ++ .../src/main/resources/META-INF/spring.factories | 2 + .../client/src/main/resources/client.properties | 10 +- smtweb-framework/canal/server/canal.server.iml | 198 ------- smtweb-framework/canal/server/pom.xml | 31 +- .../canal/server/src/main/bin/startup.bat | 4 +- .../canal/server/src/main/bin/startup.sh | 34 +- .../system/canal/deployer/CanalApplication.java | 13 + .../system/canal/deployer/CanalConfiguration.java | 21 + .../system/canal/deployer/CanalConstants.java | 97 ++++ .../system/canal/deployer/CanalController.java | 606 +++++++++++++++++++++ .../system/canal/deployer/CanalLauncher.java | 141 +++++ .../canal/deployer/CanalStartedListener.java | 39 ++ .../smtweb/system/canal/deployer/CanalStarter.java | 169 ++++++ .../system/canal/deployer/InstanceConfig.java | 93 ++++ .../canal/deployer/admin/CanalAdminController.java | 256 +++++++++ .../canal/deployer/monitor/InstanceAction.java | 30 + .../deployer/monitor/InstanceConfigMonitor.java | 16 + .../monitor/ManagerInstanceConfigMonitor.java | 184 +++++++ .../monitor/SpringInstanceConfigMonitor.java | 297 ++++++++++ .../system/canal/server/CanalApplication.java | 13 - .../smtweb/system/canal/server/CanalConstants.java | 97 ---- .../system/canal/server/CanalController.java | 606 --------------------- .../smtweb/system/canal/server/CanalLauncher.java | 141 ----- .../system/canal/server/CanalStartedListener.java | 39 -- .../smtweb/system/canal/server/CanalStarter.java | 169 ------ .../smtweb/system/canal/server/InstanceConfig.java | 93 ---- .../canal/server/admin/CanalAdminController.java | 256 --------- .../canal/server/monitor/InstanceAction.java | 30 - .../server/monitor/InstanceConfigMonitor.java | 16 - .../monitor/ManagerInstanceConfigMonitor.java | 184 ------- .../monitor/SpringInstanceConfigMonitor.java | 297 ---------- .../src/main/resources/META-INF/spring.factories | 2 + .../server/src/main/resources/canal.properties | 16 +- .../src/main/resources/example/instance.properties | 7 +- .../framework/core/systask/SysServiceFactory.java | 4 +- .../framework/core/systask/SysThreadPool.java | 8 +- .../framework/core/systask/SysThreadWorker.java | 2 +- 58 files changed, 2839 insertions(+), 2825 deletions(-) delete mode 100644 smtweb-framework/canal/client/canal.example.iml create mode 100644 smtweb-framework/canal/client/src/main/assembly/dev.xml create mode 100644 smtweb-framework/canal/client/src/main/assembly/release.xml create mode 100644 smtweb-framework/canal/client/src/main/bin/restart.sh create mode 100644 smtweb-framework/canal/client/src/main/bin/startup.bat create mode 100644 smtweb-framework/canal/client/src/main/bin/startup.sh create mode 100644 smtweb-framework/canal/client/src/main/bin/stop.sh delete mode 100644 smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientConsts.java delete mode 100644 smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientInstance.java delete mode 100644 smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientStartedListener.java delete mode 100644 smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientVO.java create mode 100644 smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientApplication.java create mode 100644 smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientConfiguration.java create mode 100644 smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientConsts.java create mode 100644 smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientInstance.java create mode 100644 smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientLauncher.java create mode 100644 smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientStartedListener.java create mode 100644 smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientVO.java create mode 100644 smtweb-framework/canal/client/src/main/resources/META-INF/spring.factories delete mode 100644 smtweb-framework/canal/server/canal.server.iml create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalApplication.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalConfiguration.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalConstants.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalController.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalLauncher.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalStartedListener.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalStarter.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/InstanceConfig.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/admin/CanalAdminController.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/monitor/InstanceAction.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/monitor/InstanceConfigMonitor.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/monitor/ManagerInstanceConfigMonitor.java create mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/monitor/SpringInstanceConfigMonitor.java delete mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalApplication.java delete mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalConstants.java delete mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalController.java delete mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalLauncher.java delete mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStartedListener.java delete mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStarter.java delete mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/InstanceConfig.java delete mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/admin/CanalAdminController.java delete mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceAction.java delete mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceConfigMonitor.java delete mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/ManagerInstanceConfigMonitor.java delete mode 100644 smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/SpringInstanceConfigMonitor.java create mode 100644 smtweb-framework/canal/server/src/main/resources/META-INF/spring.factories diff --git a/smtweb-framework/bpm/pom.xml b/smtweb-framework/bpm/pom.xml index 2fd5116..ada91ac 100644 --- a/smtweb-framework/bpm/pom.xml +++ b/smtweb-framework/bpm/pom.xml @@ -51,12 +51,12 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml - 2.11.0 + 2.13.4 com.fasterxml.jackson.dataformat jackson-dataformat-xml - 2.11.0 + 2.13.4 diff --git a/smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/BpmStartedListener.java b/smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/BpmStartedListener.java index 88be51b..046ba99 100644 --- a/smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/BpmStartedListener.java +++ b/smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/BpmStartedListener.java @@ -8,6 +8,7 @@ import cc.smtweb.framework.core.db.vo.ModelCatalog; import cc.smtweb.framework.core.mvc.controller.IStartListener; import cc.smtweb.framework.core.mvc.service.TreeHelper; import cc.smtweb.framework.core.systask.SysServiceFactory; +import cc.smtweb.framework.core.systask.SysThreadPool; import cc.smtweb.system.bpm.web.design.db.ModelCatalogTreeHelper; import cc.smtweb.system.bpm.web.sys.base.job.JobUtils; import cc.smtweb.system.bpm.web.sys.oneTimeService.OneTimeServiceFactory; @@ -27,7 +28,7 @@ public class BpmStartedListener implements IStartListener { SwConsts.SysParam.RUN_PROJECTS = "bpm"; SysServiceFactory.getInstance().reg(new OneTimeTaskCleanService()); TreeHelper.regTreeHelper(ModelCatalog.ENTITY_NAME, ModelCatalogTreeHelper.class); - + } @Override @@ -44,5 +45,6 @@ public class BpmStartedListener implements IStartListener { public void close() { OneTimeServiceFactory.getInstance().stop(); JobUtils.getInstance().stop(); + SysThreadPool.getInstance().stop(); } } diff --git a/smtweb-framework/canal/client/canal.example.iml b/smtweb-framework/canal/client/canal.example.iml deleted file mode 100644 index 7d2fbb7..0000000 --- a/smtweb-framework/canal/client/canal.example.iml +++ /dev/null @@ -1,168 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/smtweb-framework/canal/client/pom.xml b/smtweb-framework/canal/client/pom.xml index 09c9462..47b1bb1 100644 --- a/smtweb-framework/canal/client/pom.xml +++ b/smtweb-framework/canal/client/pom.xml @@ -8,7 +8,7 @@ cc.smtweb - canal.client + canal.example 1.1.6 diff --git a/smtweb-framework/canal/client/src/main/assembly/dev.xml b/smtweb-framework/canal/client/src/main/assembly/dev.xml new file mode 100644 index 0000000..3c66c4c --- /dev/null +++ b/smtweb-framework/canal/client/src/main/assembly/dev.xml @@ -0,0 +1,54 @@ + + dist + + dir + + false + + + . + / + + README* + + + + ./src/main/bin + bin + + **/* + + 0755 + + + ./src/main/conf + /conf + + **/* + + + + ./src/main/resources + /conf + + **/* + + + + target + logs + + **/* + + + + + + lib + + junit:junit + + + + diff --git a/smtweb-framework/canal/client/src/main/assembly/release.xml b/smtweb-framework/canal/client/src/main/assembly/release.xml new file mode 100644 index 0000000..6a1f9ef --- /dev/null +++ b/smtweb-framework/canal/client/src/main/assembly/release.xml @@ -0,0 +1,54 @@ + + dist + + tar.gz + + false + + + . + / + + README* + + + + ./src/main/bin + bin + + **/* + + 0755 + + + ./src/main/conf + /conf + + **/* + + + + ./src/main/resources + /conf + + **/* + + + + target + logs + + **/* + + + + + + lib + + junit:junit + + + + diff --git a/smtweb-framework/canal/client/src/main/bin/restart.sh b/smtweb-framework/canal/client/src/main/bin/restart.sh new file mode 100644 index 0000000..3623edd --- /dev/null +++ b/smtweb-framework/canal/client/src/main/bin/restart.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +args=$@ + +case $(uname) in +Linux) + bin_abs_path=$(readlink -f $(dirname $0)) + ;; +*) + bin_abs_path=$(cd $(dirname $0) ||exit ; pwd) + ;; +esac + +sh "$bin_abs_path"/stop.sh $args +sh "$bin_abs_path"/startup.sh $args diff --git a/smtweb-framework/canal/client/src/main/bin/startup.bat b/smtweb-framework/canal/client/src/main/bin/startup.bat new file mode 100644 index 0000000..3072543 --- /dev/null +++ b/smtweb-framework/canal/client/src/main/bin/startup.bat @@ -0,0 +1,23 @@ +@echo off +@if not "%ECHO%" == "" echo %ECHO% +@if "%OS%" == "Windows_NT" setlocal + +set ENV_PATH=.\ +if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0% + +set conf_dir=%ENV_PATH%\..\conf +set logback_configurationFile=%conf_dir%\logback.xml +set client_mode=Simple +if not "%1" == "" set client_mode=%1 + +set CLASSPATH=%conf_dir% +set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH% + +set JAVA_MEM_OPTS= -Xms128m -Xmx512m -XX:PermSize=128m +set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8 +set JAVA_DEBUG_OPT= -server -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=9199,server=y,suspend=n +set CANAL_OPTS= -DappName=otter-canal-example -Dlogback.configurationFile="%logback_configurationFile%" + +set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS% + +java %JAVA_OPTS% -classpath "%CLASSPATH%" cc.smtweb.system.canal.example.ClientLauncher diff --git a/smtweb-framework/canal/client/src/main/bin/startup.sh b/smtweb-framework/canal/client/src/main/bin/startup.sh new file mode 100644 index 0000000..a64ecee --- /dev/null +++ b/smtweb-framework/canal/client/src/main/bin/startup.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +current_path=`pwd` +case "`uname`" in + Linux) + bin_abs_path=$(readlink -f $(dirname $0)) + ;; + *) + bin_abs_path=`cd $(dirname $0); pwd` + ;; +esac +base=${bin_abs_path}/.. +client_mode="Simple" +logback_configurationFile=$base/conf/logback.xml +export LANG=en_US.UTF-8 +export BASE=$base + +if [ -f $base/bin/canal.pid ] ; then + echo "found canal.pid , Please run stop.sh first ,then startup.sh" 2>&2 + exit 1 +fi + +## set java path +if [ -z "$JAVA" ] ; then + JAVA=$(which java) +fi + +ALIBABA_JAVA="/usr/alibaba/java/bin/java" +TAOBAO_JAVA="/opt/taobao/java/bin/java" +if [ -z "$JAVA" ]; then + if [ -f $ALIBABA_JAVA ] ; then + JAVA=$ALIBABA_JAVA + elif [ -f $TAOBAO_JAVA ] ; then + JAVA=$TAOBAO_JAVA + else + echo "Cannot find a Java JDK. Please set either set JAVA or put java (>=1.5) in your PATH." 2>&2 + exit 1 + fi +fi + +case "$#" +in +0 ) + ;; +1 ) + client_mode=$* + ;; +2 ) + if [ "$1" = "debug" ]; then + DEBUG_PORT=$2 + DEBUG_SUSPEND="y" + JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND" + else + client_mode=$1 + fi;; +* ) + echo "THE PARAMETERS MUST BE TWO OR LESS.PLEASE CHECK AGAIN." + exit;; +esac + +JavaVersion=`$JAVA -version 2>&1 |awk 'NR==1{ gsub(/"/,""); print $3 }' | awk -F '.' '{print $1}'` +str=`file -L $JAVA | grep 64-bit` + +JAVA_OPTS="$JAVA_OPTS -Xss256k -XX:+AggressiveOpts -XX:-UseBiasedLocking -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$base/logs" +if [ $JavaVersion -ge 11 ] ; then + #JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$base_log/gc.log:time " + JAVA_OPTS="$JAVA_OPTS" +else + #JAVA_OPTS="$JAVA_OPTS -Xloggc:$base/logs/canal/gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime" + JAVA_OPTS="$JAVA_OPTS -XX:+UseFastAccessorMethods -XX:+PrintAdaptiveSizePolicy -XX:+PrintTenuringDistribution" +fi + +if [ -n "$str" ]; then + # JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS" + # For G1 + JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS" +else + JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m $JAVA_OPTS" +fi + +JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8" +CANAL_OPTS="-DappName=otter-canal-example -Dlogback.configurationFile=$logback_configurationFile" + +if [ -e $logback_configurationFile ] +then + + for i in $base/lib/*; + do CLASSPATH=$i:"$CLASSPATH"; + done + CLASSPATH="$base/conf:$CLASSPATH"; + + echo "cd to $bin_abs_path for workaround relative path" + cd $bin_abs_path + + echo LOG CONFIGURATION : $logback_configurationFile + echo client mode : $client_mode + echo CLASSPATH :$CLASSPATH + $JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH cc.smtweb.system.canal.example.ClientLauncher 1>>$base/bin/nohup.out 2>&1 & + + echo $! > $base/bin/canal.pid + echo "cd to $current_path for continue" + cd $current_path +else + echo "client mode("$client_mode") OR log configration file($logback_configurationFile) is not exist,please create then first!" +fi diff --git a/smtweb-framework/canal/client/src/main/bin/stop.sh b/smtweb-framework/canal/client/src/main/bin/stop.sh new file mode 100644 index 0000000..b8c997f --- /dev/null +++ b/smtweb-framework/canal/client/src/main/bin/stop.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +cygwin=false; +case "`uname`" in + CYGWIN*) + cygwin=true + ;; +esac + +get_pid() { + STR=$1 + PID=$2 + if $cygwin; then + JAVA_CMD="$JAVA_HOME\bin\java" + JAVA_CMD=`cygpath --path --unix $JAVA_CMD` + JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'` + else + if [ ! -z "$PID" ]; then + JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` + else + JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'` + fi + fi + echo $JAVA_PID; +} + +base=`dirname $0`/.. +pidfile=$base/bin/canal.pid +if [ ! -f "$pidfile" ];then + echo "canal is not running. exists" + exit +fi + +pid=`cat $pidfile` +if [ "$pid" == "" ] ; then + pid=`get_pid "appName=otter-canal-example"` +fi + +echo -e "`hostname`: stopping canal $pid ... " +kill $pid + +LOOPS=0 +while (true); +do + gpid=`get_pid "appName=otter-canal-example" "$pid"` + if [ "$gpid" == "" ] ; then + echo "Oook! cost:$LOOPS" + `rm $pidfile` + break; + fi + let LOOPS=LOOPS+1 + sleep 1 +done \ No newline at end of file diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientConsts.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientConsts.java deleted file mode 100644 index c238ae9..0000000 --- a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientConsts.java +++ /dev/null @@ -1,76 +0,0 @@ -package cc.smtweb.system.canal.client; - -import org.apache.commons.lang.StringUtils; - -import java.util.Properties; - -/** - * 启动常用变量 - * - * @author jianghang 2012-11-8 下午03:15:55 - * @version 1.0.0 - */ -public class ClientConsts { - - public static final String ROOT = "canal.client"; - public static final String SIZE = ROOT + "." + "size"; - public static final String INSTANCE = ROOT + "." + "instance"; - public static final String IP = ROOT + "." + "ip"; - public static final String PORT = ROOT + "." + "port"; - public static final String USERNAME = ROOT + "." + "username"; - public static final String PASSWORD = ROOT + "." + "password"; - public static final String FILTER = ROOT + "." + "filter"; - - - public static String getIp(Properties properties){ - return getProperty(properties,IP,"127.0.0.1"); - } - - public static int getPort(Properties properties){ - return Integer.parseInt(getProperty(properties,PORT,"11111")); - } - - public static int getBatchSize(Properties properties){ - return Integer.parseInt(getProperty(properties,SIZE,"50")); - } - - public static String getInstance(Properties properties){ - return getProperty(properties,INSTANCE,"example"); - } - - public static String getUsername(Properties properties){ - return getProperty(properties,USERNAME,""); - } - - public static String getPassword(Properties properties){ - return getProperty(properties,PASSWORD,""); - } - - public static String getFilter(Properties properties){ - return getProperty(properties,FILTER,".*\\..*"); - } - - public static String getProperty(Properties properties, String key, String defaultValue) { - String value = getProperty(properties, key); - if (StringUtils.isEmpty(value)) { - return defaultValue; - } else { - return value; - } - } - - public static String getProperty(Properties properties, String key) { - key = StringUtils.trim(key); - String value = System.getProperty(key); - - if (value == null) { - value = System.getenv(key); - } - - if (value == null) { - value = properties.getProperty(key); - } - - return StringUtils.trim(value); - } -} diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientInstance.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientInstance.java deleted file mode 100644 index a4415d1..0000000 --- a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientInstance.java +++ /dev/null @@ -1,290 +0,0 @@ -package cc.smtweb.system.canal.client; - -import cc.smtweb.framework.core.util.CommUtil; -import cc.smtweb.framework.core.util.JsonUtil; -import com.alibaba.otter.canal.client.CanalConnector; -import com.alibaba.otter.canal.client.CanalConnectors; -import com.alibaba.otter.canal.protocol.CanalEntry; -import com.alibaba.otter.canal.protocol.Message; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.util.Assert; - -import java.io.FileInputStream; -import java.net.InetSocketAddress; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * @Author yaoq - * @Date 2022年09月13日 18:02 - * @Description - */ -public class ClientInstance { - private static final String CLASSPATH_URL_PREFIX = "classpath:"; - private static final Logger logger = LoggerFactory.getLogger(ClientInstance.class); - protected volatile boolean running = false; - protected Thread thread = null; - protected CanalConnector connector; - protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { - - public void uncaughtException(Thread t, Throwable e) { - logger.error("解析出错了哦", e); - - } - }; - - private static ClientInstance instance = null; - private static Properties properties = null; - - static { - instance = new ClientInstance(); - } - - public static ClientInstance getInstance() { - return instance; - } - - private void loadProperties() { - try { - String conf = System.getProperty("client.conf", "classpath:client.properties"); - properties = new Properties(); - if (conf.startsWith(CLASSPATH_URL_PREFIX)) { - conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX); - properties.load(ClientInstance.class.getClassLoader().getResourceAsStream(conf)); - } else { - properties.load(new FileInputStream(conf)); - } - } catch (Exception e) { - logger.error("加载canal客户端配置参数失败！", e); - } - } - - private ClientInstance() { - loadProperties(); - connector = CanalConnectors.newSingleConnector(new - InetSocketAddress(ClientConsts.getIp(properties), - ClientConsts.getPort(properties)), ClientConsts.getInstance(properties), ClientConsts.getUsername(properties), ClientConsts.getPassword(properties)); - connector.connect(); - connector.subscribe(ClientConsts.getFilter(properties)); - connector.rollback(); - - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - logger.info("## stop the canal client"); - ClientInstance.this.stop(); - } catch (Throwable e) { - logger.warn("##something goes wrong when stopping canal:", e); - } finally { - logger.info("## canal client is down."); - } - })); - } - - //开始 - public void start() { - Assert.notNull(connector, "connector is null"); - thread = new Thread(() -> process()); - - thread.setUncaughtExceptionHandler(handler); - running = true; - thread.start(); - } - - //停止 - public void stop() { - if (!running) { - return; - } - running = false; - if (thread != null) { - try { - thread.join(); - } catch (InterruptedException e) { - // ignore - } - } - } - - protected void process() { - while (running) { - try { - connector.disconnect(); - connector.connect(); - connector.unsubscribe(); - connector.subscribe(ClientConsts.getFilter(properties)); - while (running) { - Message message = connector.getWithoutAck(ClientConsts.getBatchSize(properties) * 1024); // 获取指定数量的数据 - long batchId = message.getId(); - int size = message.getEntries().size(); - if (batchId == -1 || size == 0) { - Thread.sleep(3000L);//无数据时，睡眠。 - continue; - } else { - dataHandle(message.getEntries()); - } - if (batchId != -1) { - connector.ack(batchId); // 提交确认 - } - } - } catch (Throwable e) { - logger.error("process error!", e); - try { - Thread.sleep(1000L); - } catch (InterruptedException e1) { - // ignore - } - if (!isReConnection()) { - connector.rollback(); // 处理失败, 回滚数据 - } - } finally { - connector.disconnect(); - } - } - } - - private boolean isReConnection() { - if (!connector.checkValid()) return false; - int count = 1; - int rate = 1; - while (connector.checkValid()) { - rate = (count + 100) / 100; - try { - logger.debug("canal服务连接中断，第【" + count + "】次重连开始"); - connector.disconnect(); - connector.connect(); - logger.debug("canal服务连接中断，第【" + count + "】次重连成功"); - break; - } catch (Throwable e2) { - if (count > 99) { - logger.error("canal服务连接异常！", "重新连接canal服务端已失败【" + count + "】次，请检查canal服务是否启动，请及时启动canal服务端，或联系研发人员！"); - } - logger.debug("canal服务连接中断，第【" + count + "】次重连失败：" + CommUtil.getOrigMsg(e2)); - logger.debug("canal服务连接中断，" + (3 * rate) + "分钟后进行第【" + (count + 1) + "】连接测试！"); - try { - Thread.sleep(180 * rate * 1000L); - } catch (InterruptedException e1) { - // ignore - } - } - count++; - } - return true; - } - - /** - * 数据处理 - * - * @param entrys - */ - private void dataHandle(List entrys) throws Exception { - for (CanalEntry.Entry entry : entrys) { - if (CanalEntry.EntryType.ROWDATA == entry.getEntryType()) { - CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); - CanalEntry.EventType eventType = rowChange.getEventType(); - if (eventType == CanalEntry.EventType.DELETE) { - saveDeleteSql(entry); - } else if (eventType == CanalEntry.EventType.UPDATE) { - saveUpdateSql(entry); - } else if (eventType == CanalEntry.EventType.INSERT) { - saveInsertSql(entry); - } - } - } - } - - /** - * 保存更新语句 - * - * @param entry - */ - private void saveUpdateSql(CanalEntry.Entry entry) { - try { - CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); - List rowDatasList = rowChange.getRowDatasList(); - for (CanalEntry.RowData rowData : rowDatasList) { - List newColumnList = rowData.getAfterColumnsList(); - Map data = new HashMap<>(); - newColumnList.stream().forEach(item -> { - if (StringUtils.isNotEmpty(item.getValue())) { - data.put(lineToHump(item.getName()), item.getValue()); - } - }); - List oldColumnList = rowData.getBeforeColumnsList(); - ClientVO update = ClientVO.ok(entry.getHeader().getTableName(), "UPDATE", data, oldColumnList.get(0).getValue()); - System.out.println("更新返回 ： " + JsonUtil.encodeString(update)); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * 保存删除语句 - * - * @param entry - */ - private void saveDeleteSql(CanalEntry.Entry entry) { - try { - CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); - List rowDatasList = rowChange.getRowDatasList(); - for (CanalEntry.RowData rowData : rowDatasList) { - List oldColumnList = rowData.getBeforeColumnsList(); - ClientVO delete = ClientVO.ok(entry.getHeader().getTableName(), "DELETE", null, oldColumnList.get(0).getValue()); - System.out.println("删除返回 ： " + JsonUtil.encodeString(delete)); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * 保存插入语句 - * - * @param entry - */ - private void saveInsertSql(CanalEntry.Entry entry) { - try { - CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); - List rowDatasList = rowChange.getRowDatasList(); - for (CanalEntry.RowData rowData : rowDatasList) { - List columnList = rowData.getAfterColumnsList(); - Map data = new HashMap<>(); - columnList.stream().forEach(item -> { - if (StringUtils.isNotEmpty(item.getValue())) { - data.put(lineToHump(item.getName()), item.getValue()); - } - }); - ClientVO insert = ClientVO.ok(entry.getHeader().getTableName(), "INSERT", data, null); - System.out.println("插入返回 ： " + JsonUtil.encodeString(insert)); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * 下划线转驼峰 - * - * @param str - */ - private static Pattern linePattern = Pattern.compile("_(\\w)"); - - public static String lineToHump(String str) { - str = str.toLowerCase(); - Matcher matcher = linePattern.matcher(str); - StringBuffer sb = new StringBuffer(); - while (matcher.find()) { - matcher.appendReplacement(sb, matcher.group(1).toUpperCase()); - } - matcher.appendTail(sb); - return sb.toString(); - } -} - - diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientStartedListener.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientStartedListener.java deleted file mode 100644 index 5dad1df..0000000 --- a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientStartedListener.java +++ /dev/null @@ -1,32 +0,0 @@ -package cc.smtweb.system.canal.client; - -import cc.smtweb.framework.core.annotation.SwStartListener; -import cc.smtweb.framework.core.common.SwConsts; -import cc.smtweb.framework.core.mvc.controller.IStartListener; -import lombok.extern.slf4j.Slf4j; - -/** - * @Author yaoq - * @Date 2022年09月06日 10:31 - * @Description - */ -@Slf4j -@SwStartListener -public class ClientStartedListener implements IStartListener { - - @Override - public int order() { - return SwConsts.DEFAULT_ORDER + 3; - } - - - @Override - public void init() { - - } - - @Override - public void run() { - ClientInstance.getInstance().start(); - } -} diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientVO.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientVO.java deleted file mode 100644 index 02cbb46..0000000 --- a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientVO.java +++ /dev/null @@ -1,56 +0,0 @@ -package cc.smtweb.system.canal.client; - -import java.util.Map; - -/** - * @Author yaoq - * @Date 2022年09月13日 18:20 - * @Description - */ -public class ClientVO { - private String tableName; // 表名 - private String type; // 类型（更新、删除、插入） - private Map data; // 数据JSON 自己转对应表格实体类 - private String id; // 更新或删除都是根据ID来 - - public static ClientVO ok(String tableName, String type, Map data, String id){ - ClientVO canalVO=new ClientVO(); - canalVO.setId(id); - canalVO.setTableName(tableName); - canalVO.setType(type); - canalVO.setData(data); - return canalVO; - } - - public String getTableName() { - return tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public Map getData() { - return data; - } - - public void setData(Map data) { - this.data = data; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } -} diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientApplication.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientApplication.java new file mode 100644 index 0000000..8d7e97a --- /dev/null +++ b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientApplication.java @@ -0,0 +1,13 @@ +package cc.smtweb.system.canal.example; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class ClientApplication { + + public static void main(String[] args) { + SpringApplication.run(ClientApplication.class, args); + } + +} diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientConfiguration.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientConfiguration.java new file mode 100644 index 0000000..ad1b7b6 --- /dev/null +++ b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientConfiguration.java @@ -0,0 +1,21 @@ +package cc.smtweb.system.canal.example; + +import cc.smtweb.framework.core.mvc.config.ControllerConfig; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +/** + * @author kevin + */ +@Configuration +@ComponentScan +public class ClientConfiguration { + /** + * 配置自定义service扫描路径 {module}/{service}/{method} + */ + @Bean + public ControllerConfig clientConfiguration() { + return new ControllerConfig("canalClient", "cc.smtweb.system.canal.client", null); + } +} diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientConsts.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientConsts.java new file mode 100644 index 0000000..fdda311 --- /dev/null +++ b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientConsts.java @@ -0,0 +1,76 @@ +package cc.smtweb.system.canal.example; + +import org.apache.commons.lang.StringUtils; + +import java.util.Properties; + +/** + * 启动常用变量 + * + * @author jianghang 2012-11-8 下午03:15:55 + * @version 1.0.0 + */ +public class ClientConsts { + + public static final String ROOT = "canal.client"; + public static final String SIZE = ROOT + "." + "size"; + public static final String INSTANCE = ROOT + "." + "instance"; + public static final String IP = ROOT + "." + "ip"; + public static final String PORT = ROOT + "." + "port"; + public static final String USERNAME = ROOT + "." + "username"; + public static final String PASSWORD = ROOT + "." + "password"; + public static final String FILTER = ROOT + "." + "filter"; + + + public static String getIp(Properties properties){ + return getProperty(properties,IP,"127.0.0.1"); + } + + public static int getPort(Properties properties){ + return Integer.parseInt(getProperty(properties,PORT,"11111")); + } + + public static int getBatchSize(Properties properties){ + return Integer.parseInt(getProperty(properties,SIZE,"50")); + } + + public static String getInstance(Properties properties){ + return getProperty(properties,INSTANCE,"example"); + } + + public static String getUsername(Properties properties){ + return getProperty(properties,USERNAME,""); + } + + public static String getPassword(Properties properties){ + return getProperty(properties,PASSWORD,""); + } + + public static String getFilter(Properties properties){ + return getProperty(properties,FILTER,".*\\..*"); + } + + public static String getProperty(Properties properties, String key, String defaultValue) { + String value = getProperty(properties, key); + if (StringUtils.isEmpty(value)) { + return defaultValue; + } else { + return value; + } + } + + public static String getProperty(Properties properties, String key) { + key = StringUtils.trim(key); + String value = System.getProperty(key); + + if (value == null) { + value = System.getenv(key); + } + + if (value == null) { + value = properties.getProperty(key); + } + + return StringUtils.trim(value); + } +} diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientInstance.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientInstance.java new file mode 100644 index 0000000..4c87a73 --- /dev/null +++ b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientInstance.java @@ -0,0 +1,290 @@ +package cc.smtweb.system.canal.example; + +import cc.smtweb.framework.core.util.CommUtil; +import cc.smtweb.framework.core.util.JsonUtil; +import com.alibaba.otter.canal.client.CanalConnector; +import com.alibaba.otter.canal.client.CanalConnectors; +import com.alibaba.otter.canal.protocol.CanalEntry; +import com.alibaba.otter.canal.protocol.Message; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.Assert; + +import java.io.FileInputStream; +import java.net.InetSocketAddress; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @Author yaoq + * @Date 2022年09月13日 18:02 + * @Description + */ +public class ClientInstance { + private static final String CLASSPATH_URL_PREFIX = "classpath:"; + private static final Logger logger = LoggerFactory.getLogger(ClientInstance.class); + protected volatile boolean running = false; + protected Thread thread = null; + protected CanalConnector connector; + protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { + + public void uncaughtException(Thread t, Throwable e) { + logger.error("解析出错了哦", e); + + } + }; + + private static ClientInstance instance = null; + private static Properties properties = null; + + static { + instance = new ClientInstance(); + } + + public static ClientInstance getInstance() { + return instance; + } + + private void loadProperties() { + try { + String conf = System.getProperty("client.conf", "classpath:client.properties"); + properties = new Properties(); + if (conf.startsWith(CLASSPATH_URL_PREFIX)) { + conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX); + properties.load(ClientInstance.class.getClassLoader().getResourceAsStream(conf)); + } else { + properties.load(new FileInputStream(conf)); + } + } catch (Exception e) { + logger.error("加载canal客户端配置参数失败！", e); + } + } + + private ClientInstance() { + loadProperties(); + connector = CanalConnectors.newSingleConnector(new + InetSocketAddress(ClientConsts.getIp(properties), + ClientConsts.getPort(properties)), ClientConsts.getInstance(properties), ClientConsts.getUsername(properties), ClientConsts.getPassword(properties)); + //connector.connect(); + //connector.subscribe(ClientConsts.getFilter(properties)); + //connector.rollback(); + + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + logger.info("## stop the canal client"); + ClientInstance.this.stop(); + } catch (Throwable e) { + logger.warn("##something goes wrong when stopping canal:", e); + } finally { + logger.info("## canal client is down."); + } + })); + } + + //开始 + public void start() { + Assert.notNull(connector, "connector is null"); + thread = new Thread(() -> process()); + + thread.setUncaughtExceptionHandler(handler); + running = true; + thread.start(); + } + + //停止 + public void stop() { + if (!running) { + return; + } + running = false; + if (thread != null) { + try { + thread.join(); + } catch (InterruptedException e) { + // ignore + } + } + } + + protected void process() { + while (running) { + try { + connector.disconnect(); + connector.connect(); + connector.unsubscribe(); + connector.subscribe(ClientConsts.getFilter(properties)); + while (running) { + Message message = connector.getWithoutAck(ClientConsts.getBatchSize(properties) * 1024); // 获取指定数量的数据 + long batchId = message.getId(); + int size = message.getEntries().size(); + if (batchId == -1 || size == 0) { + Thread.sleep(3000L);//无数据时，睡眠。 + continue; + } else { + dataHandle(message.getEntries()); + } + if (batchId != -1) { + connector.ack(batchId); // 提交确认 + } + } + } catch (Throwable e) { + logger.error("process error!", e); + try { + Thread.sleep(1000L); + } catch (InterruptedException e1) { + // ignore + } + if (!isReConnection()) { + connector.rollback(); // 处理失败, 回滚数据 + } + } finally { + connector.disconnect(); + } + } + } + + private boolean isReConnection() { + if (!connector.checkValid()) return false; + int count = 1; + int rate = 1; + while (connector.checkValid()) { + rate = (count + 100) / 100; + try { + logger.debug("canal服务连接中断，第【" + count + "】次重连开始"); + connector.disconnect(); + connector.connect(); + logger.debug("canal服务连接中断，第【" + count + "】次重连成功"); + break; + } catch (Throwable e2) { + if (count > 99) { + logger.error("canal服务连接异常！", "重新连接canal服务端已失败【" + count + "】次，请检查canal服务是否启动，请及时启动canal服务端，或联系研发人员！"); + } + logger.debug("canal服务连接中断，第【" + count + "】次重连失败：" + CommUtil.getOrigMsg(e2)); + logger.debug("canal服务连接中断，" + (3 * rate) + "分钟后进行第【" + (count + 1) + "】连接测试！"); + try { + Thread.sleep(180 * rate * 1000L); + } catch (InterruptedException e1) { + // ignore + } + } + count++; + } + return true; + } + + /** + * 数据处理 + * + * @param entrys + */ + private void dataHandle(List entrys) throws Exception { + for (CanalEntry.Entry entry : entrys) { + if (CanalEntry.EntryType.ROWDATA == entry.getEntryType()) { + CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); + CanalEntry.EventType eventType = rowChange.getEventType(); + if (eventType == CanalEntry.EventType.DELETE) { + saveDeleteSql(entry); + } else if (eventType == CanalEntry.EventType.UPDATE) { + saveUpdateSql(entry); + } else if (eventType == CanalEntry.EventType.INSERT) { + saveInsertSql(entry); + } + } + } + } + + /** + * 保存更新语句 + * + * @param entry + */ + private void saveUpdateSql(CanalEntry.Entry entry) { + try { + CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); + List rowDatasList = rowChange.getRowDatasList(); + for (CanalEntry.RowData rowData : rowDatasList) { + List newColumnList = rowData.getAfterColumnsList(); + Map data = new HashMap<>(); + newColumnList.stream().forEach(item -> { + if (StringUtils.isNotEmpty(item.getValue())) { + data.put(lineToHump(item.getName()), item.getValue()); + } + }); + List oldColumnList = rowData.getBeforeColumnsList(); + ClientVO update = ClientVO.ok(entry.getHeader().getTableName(), "UPDATE", data, oldColumnList.get(0).getValue()); + System.out.println("更新返回 ： " + JsonUtil.encodeString(update)); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 保存删除语句 + * + * @param entry + */ + private void saveDeleteSql(CanalEntry.Entry entry) { + try { + CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); + List rowDatasList = rowChange.getRowDatasList(); + for (CanalEntry.RowData rowData : rowDatasList) { + List oldColumnList = rowData.getBeforeColumnsList(); + ClientVO delete = ClientVO.ok(entry.getHeader().getTableName(), "DELETE", null, oldColumnList.get(0).getValue()); + System.out.println("删除返回 ： " + JsonUtil.encodeString(delete)); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 保存插入语句 + * + * @param entry + */ + private void saveInsertSql(CanalEntry.Entry entry) { + try { + CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); + List rowDatasList = rowChange.getRowDatasList(); + for (CanalEntry.RowData rowData : rowDatasList) { + List columnList = rowData.getAfterColumnsList(); + Map data = new HashMap<>(); + columnList.stream().forEach(item -> { + if (StringUtils.isNotEmpty(item.getValue())) { + data.put(lineToHump(item.getName()), item.getValue()); + } + }); + ClientVO insert = ClientVO.ok(entry.getHeader().getTableName(), "INSERT", data, null); + System.out.println("插入返回 ： " + JsonUtil.encodeString(insert)); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 下划线转驼峰 + * + * @param str + */ + private static Pattern linePattern = Pattern.compile("_(\\w)"); + + public static String lineToHump(String str) { + str = str.toLowerCase(); + Matcher matcher = linePattern.matcher(str); + StringBuffer sb = new StringBuffer(); + while (matcher.find()) { + matcher.appendReplacement(sb, matcher.group(1).toUpperCase()); + } + matcher.appendTail(sb); + return sb.toString(); + } +} + + diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientLauncher.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientLauncher.java new file mode 100644 index 0000000..c9216a6 --- /dev/null +++ b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientLauncher.java @@ -0,0 +1,14 @@ +package cc.smtweb.system.canal.example; + +/** + * @Author yaoq + * @Date 2022年09月14日 10:27 + * @Description + */ +public class ClientLauncher { + + public static void main(String[] args) { + ClientInstance.getInstance().start(); + } + +} diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientStartedListener.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientStartedListener.java new file mode 100644 index 0000000..f750878 --- /dev/null +++ b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientStartedListener.java @@ -0,0 +1,37 @@ +package cc.smtweb.system.canal.example; + +import cc.smtweb.framework.core.annotation.SwStartListener; +import cc.smtweb.framework.core.common.SwConsts; +import cc.smtweb.framework.core.mvc.controller.IStartListener; +import lombok.extern.slf4j.Slf4j; + +/** + * @Author yaoq + * @Date 2022年09月06日 10:31 + * @Description + */ +@Slf4j +@SwStartListener +public class ClientStartedListener implements IStartListener { + + @Override + public int order() { + return SwConsts.DEFAULT_ORDER + 3; + } + + + @Override + public void init() { + + } + + @Override + public void run() { + ClientInstance.getInstance().start(); + } + + @Override + public void close() { + ClientInstance.getInstance().stop(); + } +} diff --git a/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientVO.java b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientVO.java new file mode 100644 index 0000000..d55abec --- /dev/null +++ b/smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/example/ClientVO.java @@ -0,0 +1,56 @@ +package cc.smtweb.system.canal.example; + +import java.util.Map; + +/** + * @Author yaoq + * @Date 2022年09月13日 18:20 + * @Description + */ +public class ClientVO { + private String tableName; // 表名 + private String type; // 类型（更新、删除、插入） + private Map data; // 数据JSON 自己转对应表格实体类 + private String id; // 更新或删除都是根据ID来 + + public static ClientVO ok(String tableName, String type, Map data, String id){ + ClientVO canalVO=new ClientVO(); + canalVO.setId(id); + canalVO.setTableName(tableName); + canalVO.setType(type); + canalVO.setData(data); + return canalVO; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Map getData() { + return data; + } + + public void setData(Map data) { + this.data = data; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } +} diff --git a/smtweb-framework/canal/client/src/main/resources/META-INF/spring.factories b/smtweb-framework/canal/client/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000..1476d0e --- /dev/null +++ b/smtweb-framework/canal/client/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + cc.smtweb.system.canal.example.ClientConfiguration diff --git a/smtweb-framework/canal/client/src/main/resources/client.properties b/smtweb-framework/canal/client/src/main/resources/client.properties index 6314051..990225f 100644 --- a/smtweb-framework/canal/client/src/main/resources/client.properties +++ b/smtweb-framework/canal/client/src/main/resources/client.properties @@ -1,14 +1,14 @@ -#canal ??????? -# ??????? ??MB +#canal 客户端配置 +# 单次读取数据量大小 单位：MB canal.server.size=50 -# ?????? +# canal 服务端实例名，端口，IP canal.server.instance=example canal.server.ip=127.0.0.1 canal.server.port=11111 -# ????????? ?????? +# canal 服务端访问用户名，密码 canal.server.username= canal.server.password= -# ??????????.*\\..*??????scmz\\..* +# 数据库匹配规则.*\\..*, scmz\\..* canal.server.filter=.*\\..* diff --git a/smtweb-framework/canal/server/canal.server.iml b/smtweb-framework/canal/server/canal.server.iml deleted file mode 100644 index 9f6a7c5..0000000 --- a/smtweb-framework/canal/server/canal.server.iml +++ /dev/null @@ -1,198 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/smtweb-framework/canal/server/pom.xml b/smtweb-framework/canal/server/pom.xml index 89b0415..6835884 100644 --- a/smtweb-framework/canal/server/pom.xml +++ b/smtweb-framework/canal/server/pom.xml @@ -1,4 +1,5 @@ - + 4.0.0 @@ -9,28 +10,16 @@ cc.smtweb - canal.server + canal.deployer 1.1.6 com.alibaba.otter - canal.protocol - 1.1.6 - - - com.alibaba.otter - canal.client - 1.1.6 - - - - com.alibaba.otter canal.server 1.1.6 - com.alibaba.otter @@ -91,27 +80,33 @@ jar-with-dependencies provided + org.springframework.boot spring-boot-autoconfigure + cc.smtweb sw-framework-core 3.1.0-SNAPSHOT - compile cc.smtweb sw-system-bpm 3.1.0-SNAPSHOT - compile + + + maven-jar-plugin @@ -127,7 +122,7 @@ - + org.apache.maven.plugins maven-dependency-plugin 2.10 @@ -187,7 +182,7 @@ ${basedir}/src/main/assembly/dev.xml - canal + canal-deployer ${project.build.directory} diff --git a/smtweb-framework/canal/server/src/main/bin/startup.bat b/smtweb-framework/canal/server/src/main/bin/startup.bat index 0cad8cb..bb43125 100644 --- a/smtweb-framework/canal/server/src/main/bin/startup.bat +++ b/smtweb-framework/canal/server/src/main/bin/startup.bat @@ -21,7 +21,7 @@ set CANAL_OPTS= -DappName=otter-canal -Dlogback.configurationFile="%logback_conf set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS% -set CMD_STR= java %JAVA_OPTS% -classpath "%CLASSPATH%" java %JAVA_OPTS% -classpath "%CLASSPATH%" CanalLauncher +set CMD_STR= java %JAVA_OPTS% -classpath "%CLASSPATH%" java %JAVA_OPTS% -classpath "%CLASSPATH%" cc.smtweb.system.canal.deployer.CanalLauncher echo start cmd : %CMD_STR% -java %JAVA_OPTS% -classpath "%CLASSPATH%" CanalLauncher +java %JAVA_OPTS% -classpath "%CLASSPATH%" cc.smtweb.system.canal.deployer.CanalLauncher diff --git a/smtweb-framework/canal/server/src/main/bin/startup.sh b/smtweb-framework/canal/server/src/main/bin/startup.sh index b8c0ed3..2a2be9b 100644 --- a/smtweb-framework/canal/server/src/main/bin/startup.sh +++ b/smtweb-framework/canal/server/src/main/bin/startup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash current_path=`pwd` case "`uname`" in @@ -21,7 +21,7 @@ if [ -f $base/bin/canal.pid ] ; then exit 1 fi -if [ ! -d $base/logs/canal ] ; then +if [ ! -d $base/logs/canal ] ; then mkdir -p $base/logs/canal fi @@ -43,30 +43,30 @@ if [ -z "$JAVA" ]; then fi fi -case "$#" +case "$#" in -0 ) +0 ) ;; -1 ) +1 ) var=$* if [ "$var" = "local" ]; then canal_conf=$canal_local_conf else - if [ -f $var ] ; then + if [ -f $var ] ; then canal_conf=$var else echo "THE PARAMETER IS NOT CORRECT.PLEASE CHECK AGAIN." exit fi fi;; -2 ) +2 ) var=$1 if [ "$var" = "local" ]; then canal_conf=$canal_local_conf else if [ -f $var ] ; then canal_conf=$var - else + else if [ "$1" = "debug" ]; then DEBUG_PORT=$2 DEBUG_SUSPEND="n" @@ -103,24 +103,24 @@ JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true CANAL_OPTS="-DappName=otter-canal -Dlogback.configurationFile=$logback_configurationFile -Dcanal.conf=$canal_conf" if [ -e $canal_conf -a -e $logback_configurationFile ] -then - +then + for i in $base/lib/*; do CLASSPATH=$i:"$CLASSPATH"; done CLASSPATH="$base/conf:$CLASSPATH"; - + echo "cd to $bin_abs_path for workaround relative path" cd $bin_abs_path - + echo LOG CONFIGURATION : $logback_configurationFile - echo canal conf : $canal_conf + echo canal conf : $canal_conf echo CLASSPATH :$CLASSPATH - $JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.deployer.CanalLauncher 1>>$base/logs/canal/canal_stdout.log 2>&1 & - echo $! > $base/bin/canal.pid - + $JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH cc.smtweb.system.canal.server.CanalLauncher 1>>$base/logs/canal/canal_stdout.log 2>&1 & + echo $! > $base/bin/canal.pid + echo "cd to $current_path for continue" cd $current_path -else +else echo "canal conf("$canal_conf") OR log configration file($logback_configurationFile) is not exist,please create then first!" fi diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalApplication.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalApplication.java new file mode 100644 index 0000000..4b4ac86 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalApplication.java @@ -0,0 +1,13 @@ +package cc.smtweb.system.canal.deployer; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class CanalApplication { + + public static void main(String[] args) { + SpringApplication.run(CanalApplication.class, args); + } + +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalConfiguration.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalConfiguration.java new file mode 100644 index 0000000..5519782 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalConfiguration.java @@ -0,0 +1,21 @@ +package cc.smtweb.system.canal.deployer; + +import cc.smtweb.framework.core.mvc.config.ControllerConfig; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +/** + * @author kevin + */ +@Configuration +@ComponentScan +public class CanalConfiguration { + /** + * 配置自定义service扫描路径 {module}/{service}/{method} + */ + @Bean + public ControllerConfig canalConfiguration() { + return new ControllerConfig("canalServer", "cc.smtweb.system.canal.server", null); + } +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalConstants.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalConstants.java new file mode 100644 index 0000000..cc8a1cb --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalConstants.java @@ -0,0 +1,97 @@ +package cc.smtweb.system.canal.deployer; + +import java.text.MessageFormat; + +/** + * 启动常用变量 + * + * @author jianghang 2012-11-8 下午03:15:55 + * @version 1.0.0 + */ +public class CanalConstants { + + public static final String MDC_DESTINATION = "destination"; + public static final String ROOT = "canal"; + public static final String CANAL_ID = ROOT + "." + "id"; + public static final String CANAL_IP = ROOT + "." + "ip"; + public static final String CANAL_REGISTER_IP = ROOT + "." + "register.ip"; + public static final String CANAL_PORT = ROOT + "." + "port"; + public static final String CANAL_USER = ROOT + "." + "user"; + public static final String CANAL_PASSWD = ROOT + "." + "passwd"; + public static final String CANAL_METRICS_PULL_PORT = ROOT + "." + "metrics.pull.port"; + public static final String CANAL_ADMIN_MANAGER = ROOT + "." + "admin.manager"; + public static final String CANAL_ADMIN_PORT = ROOT + "." + "admin.port"; + public static final String CANAL_ADMIN_USER = ROOT + "." + "admin.user"; + public static final String CANAL_ADMIN_PASSWD = ROOT + "." + "admin.passwd"; + public static final String CANAL_ADMIN_AUTO_REGISTER = ROOT + "." + "admin.register.auto"; + public static final String CANAL_ADMIN_AUTO_CLUSTER = ROOT + "." + "admin.register.cluster"; + public static final String CANAL_ADMIN_REGISTER_NAME = ROOT + "." + "admin.register.name"; + public static final String CANAL_ZKSERVERS = ROOT + "." + "zkServers"; + public static final String CANAL_WITHOUT_NETTY = ROOT + "." + "withoutNetty"; + + public static final String CANAL_DESTINATIONS = ROOT + "." + "destinations"; + public static final String CANAL_AUTO_SCAN = ROOT + "." + "auto.scan"; + public static final String CANAL_AUTO_SCAN_INTERVAL = ROOT + "." + "auto.scan.interval"; + public static final String CANAL_CONF_DIR = ROOT + "." + "conf.dir"; + public static final String CANAL_SERVER_MODE = ROOT + "." + "serverMode"; + + public static final String CANAL_DESTINATION_SPLIT = ","; + public static final String GLOBAL_NAME = "global"; + + public static final String INSTANCE_MODE_TEMPLATE = ROOT + "." + "instance.{0}.mode"; + public static final String INSTANCE_LAZY_TEMPLATE = ROOT + "." + "instance.{0}.lazy"; + public static final String INSTANCE_MANAGER_ADDRESS_TEMPLATE = ROOT + "." + "instance.{0}.manager.address"; + public static final String INSTANCE_SPRING_XML_TEMPLATE = ROOT + "." + "instance.{0}.spring.xml"; + + public static final String CANAL_DESTINATION_PROPERTY = ROOT + ".instance.destination"; + + public static final String CANAL_SOCKETCHANNEL = ROOT + "." + "socketChannel"; + + public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey"; + public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey"; + +// public static final String CANAL_MQ_SERVERS = ROOT + "." + "mq.servers"; +// public static final String CANAL_MQ_RETRIES = ROOT + "." + "mq.retries"; +// public static final String CANAL_MQ_BATCHSIZE = ROOT + "." + "mq.batchSize"; +// public static final String CANAL_MQ_LINGERMS = ROOT + "." + "mq.lingerMs"; +// public static final String CANAL_MQ_MAXREQUESTSIZE = ROOT + "." + "mq.maxRequestSize"; +// public static final String CANAL_MQ_BUFFERMEMORY = ROOT + "." + "mq.bufferMemory"; +// public static final String CANAL_MQ_CANALBATCHSIZE = ROOT + "." + "mq.canalBatchSize"; +// public static final String CANAL_MQ_CANALGETTIMEOUT = ROOT + "." + "mq.canalGetTimeout"; +// public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage"; +// public static final String CANAL_MQ_PARALLELTHREADSIZE = ROOT + "." + "mq.parallelThreadSize"; +// public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType"; +// public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks"; +// public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction"; +// public static final String CANAL_MQ_PRODUCERGROUP = ROOT + "." + "mq.producerGroup"; +// public static final String CANAL_MQ_PROPERTIES = ROOT + "." + "mq.properties"; +// public static final String CANAL_MQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "mq.enableMessageTrace"; +// public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.accessChannel"; +// public static final String CANAL_MQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "mq.customizedTraceTopic"; +// public static final String CANAL_MQ_NAMESPACE = ROOT + "." + "mq.namespace"; +// public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable"; +// public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH = ROOT + "." + "mq.kafka.kerberos.krb5FilePath"; +// public static final String CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH = ROOT + "." + "mq.kafka.kerberos.jaasFilePath"; +// public static final String CANAL_MQ_USERNAME = ROOT + "." + "mq.username"; +// public static final String CANAL_MQ_PASSWORD = ROOT + "." + "mq.password"; +// public static final String CANAL_MQ_VHOST = ROOT + "." + "mq.vhost"; +// public static final String CANAL_MQ_ALIYUN_UID = ROOT + "." + "mq.aliyunuid"; +// public static final String CANAL_MQ_EXCHANGE = ROOT + "." + "mq.exchange"; +// public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash"; + + public static String getInstanceModeKey(String destination) { + return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination); + } + + public static String getInstanceManagerAddressKey(String destination) { + return MessageFormat.format(INSTANCE_MANAGER_ADDRESS_TEMPLATE, destination); + } + + public static String getInstancSpringXmlKey(String destination) { + return MessageFormat.format(INSTANCE_SPRING_XML_TEMPLATE, destination); + } + + public static String getInstancLazyKey(String destination) { + return MessageFormat.format(INSTANCE_LAZY_TEMPLATE, destination); + } +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalController.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalController.java new file mode 100644 index 0000000..fde52aa --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalController.java @@ -0,0 +1,606 @@ +package cc.smtweb.system.canal.deployer; + +import java.util.Map; +import java.util.Properties; + +import cc.smtweb.system.canal.deployer.monitor.InstanceAction; +import cc.smtweb.system.canal.deployer.monitor.InstanceConfigMonitor; +import cc.smtweb.system.canal.deployer.monitor.ManagerInstanceConfigMonitor; +import cc.smtweb.system.canal.deployer.monitor.SpringInstanceConfigMonitor; +import org.I0Itec.zkclient.IZkStateListener; +import org.I0Itec.zkclient.exception.ZkNoNodeException; +import org.I0Itec.zkclient.exception.ZkNodeExistsException; +import org.apache.commons.lang.BooleanUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.zookeeper.Watcher.Event.KeeperState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; + +import com.alibaba.otter.canal.common.utils.AddressUtils; +import com.alibaba.otter.canal.common.zookeeper.ZkClientx; +import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; +import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningData; +import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningListener; +import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitor; +import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitors; +import cc.smtweb.system.canal.deployer.InstanceConfig.InstanceMode; +import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator; +import com.alibaba.otter.canal.instance.manager.PlainCanalInstanceGenerator; +import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient; +import com.alibaba.otter.canal.instance.spring.SpringCanalInstanceGenerator; +import com.alibaba.otter.canal.server.CanalMQStarter; +import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded; +import com.alibaba.otter.canal.server.exception.CanalServerException; +import com.alibaba.otter.canal.server.netty.CanalServerWithNetty; +import com.google.common.base.Function; +import com.google.common.collect.MapMaker; +import com.google.common.collect.MigrateMap; + +/** + * canal调度控制器 + * + * @author jianghang 2012-11-8 下午12:03:11 + * @version 1.0.0 + */ +public class CanalController { + + private static final Logger logger = LoggerFactory.getLogger(CanalController.class); + private String ip; + private String registerIp; + private int port; + private int adminPort; + // 默认使用spring的方式载入 + private Map instanceConfigs; + private InstanceConfig globalInstanceConfig; + private Map managerClients; + // 监听instance config的变化 + private boolean autoScan = true; + private InstanceAction defaultAction; + private Map instanceConfigMonitors; + private CanalServerWithEmbedded embededCanalServer; + private CanalServerWithNetty canalServer; + + private CanalInstanceGenerator instanceGenerator; + private ZkClientx zkclientx; + + private CanalMQStarter canalMQStarter; + private String adminUser; + private String adminPasswd; + + public CanalController(){ + this(System.getProperties()); + } + + public CanalController(final Properties properties){ + managerClients = MigrateMap.makeComputingMap(this::getManagerClient); + + // 初始化全局参数设置 + globalInstanceConfig = initGlobalConfig(properties); + instanceConfigs = new MapMaker().makeMap(); + // 初始化instance config + initInstanceConfig(properties); + + // init socketChannel + String socketChannel = getProperty(properties, CanalConstants.CANAL_SOCKETCHANNEL); + if (StringUtils.isNotEmpty(socketChannel)) { + System.setProperty(CanalConstants.CANAL_SOCKETCHANNEL, socketChannel); + } + + // 兼容1.1.0版本的ak/sk参数名 + String accesskey = getProperty(properties, "canal.instance.rds.accesskey"); + String secretkey = getProperty(properties, "canal.instance.rds.secretkey"); + if (StringUtils.isNotEmpty(accesskey)) { + System.setProperty(CanalConstants.CANAL_ALIYUN_ACCESSKEY, accesskey); + } + if (StringUtils.isNotEmpty(secretkey)) { + System.setProperty(CanalConstants.CANAL_ALIYUN_SECRETKEY, secretkey); + } + + // 准备canal server + ip = getProperty(properties, CanalConstants.CANAL_IP); + registerIp = getProperty(properties, CanalConstants.CANAL_REGISTER_IP); + port = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_PORT, "11111")); + adminPort = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_ADMIN_PORT, "11110")); + embededCanalServer = CanalServerWithEmbedded.instance(); + embededCanalServer.setCanalInstanceGenerator(instanceGenerator);// 设置自定义的instanceGenerator + int metricsPort = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_METRICS_PULL_PORT, "11112")); + embededCanalServer.setMetricsPort(metricsPort); + + this.adminUser = getProperty(properties, CanalConstants.CANAL_ADMIN_USER); + this.adminPasswd = getProperty(properties, CanalConstants.CANAL_ADMIN_PASSWD); + embededCanalServer.setUser(getProperty(properties, CanalConstants.CANAL_USER)); + embededCanalServer.setPasswd(getProperty(properties, CanalConstants.CANAL_PASSWD)); + + String canalWithoutNetty = getProperty(properties, CanalConstants.CANAL_WITHOUT_NETTY); + if (canalWithoutNetty == null || "false".equals(canalWithoutNetty)) { + canalServer = CanalServerWithNetty.instance(); + canalServer.setIp(ip); + canalServer.setPort(port); + } + + // 处理下ip为空，默认使用hostIp暴露到zk中 + if (StringUtils.isEmpty(ip) && StringUtils.isEmpty(registerIp)) { + ip = registerIp = AddressUtils.getHostIp(); + } + + if (StringUtils.isEmpty(ip)) { + ip = AddressUtils.getHostIp(); + } + + if (StringUtils.isEmpty(registerIp)) { + registerIp = ip; // 兼容以前配置 + } + final String zkServers = getProperty(properties, CanalConstants.CANAL_ZKSERVERS); + if (StringUtils.isNotEmpty(zkServers)) { + zkclientx = ZkClientx.getZkClient(zkServers); + // 初始化系统目录 + zkclientx.createPersistent(ZookeeperPathUtils.DESTINATION_ROOT_NODE, true); + zkclientx.createPersistent(ZookeeperPathUtils.CANAL_CLUSTER_ROOT_NODE, true); + } + + final ServerRunningData serverData = new ServerRunningData(registerIp + ":" + port); + ServerRunningMonitors.setServerData(serverData); + ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap((Function) destination -> { + ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData); + runningMonitor.setDestination(destination); + runningMonitor.setListener(new ServerRunningListener() { + + public void processActiveEnter() { + try { + MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); + embededCanalServer.start(destination); + if (canalMQStarter != null) { + canalMQStarter.startDestination(destination); + } + } finally { + MDC.remove(CanalConstants.MDC_DESTINATION); + } + } + + public void processActiveExit() { + try { + MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); + if (canalMQStarter != null) { + canalMQStarter.stopDestination(destination); + } + embededCanalServer.stop(destination); + } finally { + MDC.remove(CanalConstants.MDC_DESTINATION); + } + } + + public void processStart() { + try { + if (zkclientx != null) { + final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, + registerIp + ":" + port); + initCid(path); + zkclientx.subscribeStateChanges(new IZkStateListener() { + + public void handleStateChanged(KeeperState state) throws Exception { + + } + + public void handleNewSession() throws Exception { + initCid(path); + } + + @Override + public void handleSessionEstablishmentError(Throwable error) throws Exception { + logger.error("failed to connect to zookeeper", error); + } + }); + } + } finally { + MDC.remove(CanalConstants.MDC_DESTINATION); + } + } + + public void processStop() { + try { + MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); + if (zkclientx != null) { + final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, + registerIp + ":" + port); + releaseCid(path); + } + } finally { + MDC.remove(CanalConstants.MDC_DESTINATION); + } + } + + }); + if (zkclientx != null) { + runningMonitor.setZkClient(zkclientx); + } + // 触发创建一下cid节点 + runningMonitor.init(); + return runningMonitor; + })); + + // 初始化monitor机制 + autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN)); + if (autoScan) { + defaultAction = new InstanceAction() { + + public void start(String destination) { + InstanceConfig config = instanceConfigs.get(destination); + if (config == null) { + // 重新读取一下instance config + config = parseInstanceConfig(properties, destination); + instanceConfigs.put(destination, config); + } + + if (!embededCanalServer.isStart(destination)) { + // HA机制启动 + ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination); + if (!config.getLazy() && !runningMonitor.isStart()) { + runningMonitor.start(); + } + } + + logger.info("auto notify start {} successful.", destination); + } + + public void stop(String destination) { + // 此处的stop，代表强制退出，非HA机制，所以需要退出HA的monitor和配置信息 + InstanceConfig config = instanceConfigs.remove(destination); + if (config != null) { + embededCanalServer.stop(destination); + ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination); + if (runningMonitor.isStart()) { + runningMonitor.stop(); + } + } + + logger.info("auto notify stop {} successful.", destination); + } + + public void reload(String destination) { + // 目前任何配置变化，直接重启，简单处理 + stop(destination); + start(destination); + + logger.info("auto notify reload {} successful.", destination); + } + + @Override + public void release(String destination) { + // 此处的release，代表强制释放，主要针对HA机制释放运行，让给其他机器抢占 + InstanceConfig config = instanceConfigs.get(destination); + if (config != null) { + ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination); + if (runningMonitor.isStart()) { + boolean release = runningMonitor.release(); + if (!release) { + // 如果是单机模式,则直接清除配置 + instanceConfigs.remove(destination); + // 停掉服务 + runningMonitor.stop(); + if (instanceConfigMonitors.containsKey(InstanceConfig.InstanceMode.MANAGER)) { + ManagerInstanceConfigMonitor monitor = (ManagerInstanceConfigMonitor) instanceConfigMonitors.get(InstanceConfig.InstanceMode.MANAGER); + Map instanceActions = monitor.getActions(); + if (instanceActions.containsKey(destination)) { + // 清除内存中的autoScan cache + monitor.release(destination); + } + } + } + } + } + + logger.info("auto notify release {} successful.", destination); + } + }; + + instanceConfigMonitors = MigrateMap.makeComputingMap(mode -> { + int scanInterval = Integer.valueOf(getProperty(properties, + CanalConstants.CANAL_AUTO_SCAN_INTERVAL, + "5")); + + if (mode.isSpring()) { + SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor(); + monitor.setScanIntervalInSecond(scanInterval); + monitor.setDefaultAction(defaultAction); + // 设置conf目录，默认是user.dir + conf目录组成 + String rootDir = getProperty(properties, CanalConstants.CANAL_CONF_DIR); + if (StringUtils.isEmpty(rootDir)) { + rootDir = "../conf"; + } + + if (StringUtils.equals("otter-canal", System.getProperty("appName"))) { + monitor.setRootConf(rootDir); + } else { + // eclipse debug模式 + monitor.setRootConf("src/main/resources/"); + } + return monitor; + } else if (mode.isManager()) { + ManagerInstanceConfigMonitor monitor = new ManagerInstanceConfigMonitor(); + monitor.setScanIntervalInSecond(scanInterval); + monitor.setDefaultAction(defaultAction); + String managerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER); + monitor.setConfigClient(getManagerClient(managerAddress)); + return monitor; + } else { + throw new UnsupportedOperationException("unknow mode :" + mode + " for monitor"); + } + }); + } + } + + private InstanceConfig initGlobalConfig(Properties properties) { + String adminManagerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER); + InstanceConfig globalConfig = new InstanceConfig(); + String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(CanalConstants.GLOBAL_NAME)); + if (StringUtils.isNotEmpty(adminManagerAddress)) { + // 如果指定了manager地址,则强制适用manager + globalConfig.setMode(InstanceMode.MANAGER); + } else if (StringUtils.isNotEmpty(modeStr)) { + globalConfig.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr))); + } + + String lazyStr = getProperty(properties, CanalConstants.getInstancLazyKey(CanalConstants.GLOBAL_NAME)); + if (StringUtils.isNotEmpty(lazyStr)) { + globalConfig.setLazy(Boolean.valueOf(lazyStr)); + } + + String managerAddress = getProperty(properties, + CanalConstants.getInstanceManagerAddressKey(CanalConstants.GLOBAL_NAME)); + if (StringUtils.isNotEmpty(managerAddress)) { + if (StringUtils.equals(managerAddress, "${canal.admin.manager}")) { + managerAddress = adminManagerAddress; + } + + globalConfig.setManagerAddress(managerAddress); + } + + String springXml = getProperty(properties, CanalConstants.getInstancSpringXmlKey(CanalConstants.GLOBAL_NAME)); + if (StringUtils.isNotEmpty(springXml)) { + globalConfig.setSpringXml(springXml); + } + + instanceGenerator = destination -> { + InstanceConfig config = instanceConfigs.get(destination); + if (config == null) { + throw new CanalServerException("can't find destination:" + destination); + } + + if (config.getMode().isManager()) { + PlainCanalInstanceGenerator instanceGenerator = new PlainCanalInstanceGenerator(properties); + instanceGenerator.setCanalConfigClient(managerClients.get(config.getManagerAddress())); + instanceGenerator.setSpringXml(config.getSpringXml()); + return instanceGenerator.generate(destination); + } else if (config.getMode().isSpring()) { + SpringCanalInstanceGenerator instanceGenerator = new SpringCanalInstanceGenerator(); + instanceGenerator.setSpringXml(config.getSpringXml()); + return instanceGenerator.generate(destination); + } else { + throw new UnsupportedOperationException("unknow mode :" + config.getMode()); + } + + }; + + return globalConfig; + } + + private PlainCanalConfigClient getManagerClient(String managerAddress) { + return new PlainCanalConfigClient(managerAddress, this.adminUser, this.adminPasswd, this.registerIp, adminPort); + } + + private void initInstanceConfig(Properties properties) { + String destinationStr = getProperty(properties, CanalConstants.CANAL_DESTINATIONS); + String[] destinations = StringUtils.split(destinationStr, CanalConstants.CANAL_DESTINATION_SPLIT); + + for (String destination : destinations) { + InstanceConfig config = parseInstanceConfig(properties, destination); + InstanceConfig oldConfig = instanceConfigs.put(destination, config); + + if (oldConfig != null) { + logger.warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config); + } + } + } + + private InstanceConfig parseInstanceConfig(Properties properties, String destination) { + String adminManagerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER); + InstanceConfig config = new InstanceConfig(globalInstanceConfig); + String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(destination)); + if (StringUtils.isNotEmpty(adminManagerAddress)) { + // 如果指定了manager地址,则强制适用manager + config.setMode(InstanceMode.MANAGER); + } else if (StringUtils.isNotEmpty(modeStr)) { + config.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr))); + } + + String lazyStr = getProperty(properties, CanalConstants.getInstancLazyKey(destination)); + if (!StringUtils.isEmpty(lazyStr)) { + config.setLazy(Boolean.valueOf(lazyStr)); + } + + if (config.getMode().isManager()) { + String managerAddress = getProperty(properties, CanalConstants.getInstanceManagerAddressKey(destination)); + if (StringUtils.isNotEmpty(managerAddress)) { + if (StringUtils.equals(managerAddress, "${canal.admin.manager}")) { + managerAddress = adminManagerAddress; + } + config.setManagerAddress(managerAddress); + } + } + + String springXml = getProperty(properties, CanalConstants.getInstancSpringXmlKey(destination)); + if (StringUtils.isNotEmpty(springXml)) { + config.setSpringXml(springXml); + } + + return config; + } + + public static String getProperty(Properties properties, String key, String defaultValue) { + String value = getProperty(properties, key); + if (StringUtils.isEmpty(value)) { + return defaultValue; + } else { + return value; + } + } + + public static String getProperty(Properties properties, String key) { + key = StringUtils.trim(key); + String value = System.getProperty(key); + + if (value == null) { + value = System.getenv(key); + } + + if (value == null) { + value = properties.getProperty(key); + } + + return StringUtils.trim(value); + } + + public void start() throws Throwable { + logger.info("## start the canal server[{}({}):{}]", ip, registerIp, port); + // 创建整个canal的工作节点 + final String path = ZookeeperPathUtils.getCanalClusterNode(registerIp + ":" + port); + initCid(path); + if (zkclientx != null) { + this.zkclientx.subscribeStateChanges(new IZkStateListener() { + + public void handleStateChanged(KeeperState state) throws Exception { + + } + + public void handleNewSession() throws Exception { + initCid(path); + } + + @Override + public void handleSessionEstablishmentError(Throwable error) throws Exception { + logger.error("failed to connect to zookeeper", error); + } + }); + } + // 优先启动embeded服务 + embededCanalServer.start(); + // 尝试启动一下非lazy状态的通道 + for (Map.Entry entry : instanceConfigs.entrySet()) { + final String destination = entry.getKey(); + InstanceConfig config = entry.getValue(); + // 创建destination的工作节点 + if (!embededCanalServer.isStart(destination)) { + // HA机制启动 + ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination); + if (!config.getLazy() && !runningMonitor.isStart()) { + runningMonitor.start(); + } + } + + if (autoScan) { + instanceConfigMonitors.get(config.getMode()).register(destination, defaultAction); + } + } + + if (autoScan) { + instanceConfigMonitors.get(globalInstanceConfig.getMode()).start(); + for (InstanceConfigMonitor monitor : instanceConfigMonitors.values()) { + if (!monitor.isStart()) { + monitor.start(); + } + } + } + + // 启动网络接口 + if (canalServer != null) { + canalServer.start(); + } + } + + public void stop() throws Throwable { + + if (canalServer != null) { + canalServer.stop(); + } + + if (autoScan) { + for (InstanceConfigMonitor monitor : instanceConfigMonitors.values()) { + if (monitor.isStart()) { + monitor.stop(); + } + } + } + + for (ServerRunningMonitor runningMonitor : ServerRunningMonitors.getRunningMonitors().values()) { + if (runningMonitor.isStart()) { + runningMonitor.stop(); + } + } + + // 释放canal的工作节点 + releaseCid(ZookeeperPathUtils.getCanalClusterNode(registerIp + ":" + port)); + logger.info("## stop the canal server[{}({}):{}]", ip, registerIp, port); + + if (zkclientx != null) { + zkclientx.close(); + } + + // 关闭时清理缓存 + if (instanceConfigs != null) { + instanceConfigs.clear(); + } + if (managerClients != null) { + managerClients.clear(); + } + if (instanceConfigMonitors != null) { + instanceConfigMonitors.clear(); + } + + ZkClientx.clearClients(); + } + + private void initCid(String path) { + // logger.info("## init the canalId = {}", cid); + // 初始化系统目录 + if (zkclientx != null) { + try { + zkclientx.createEphemeral(path); + } catch (ZkNoNodeException e) { + // 如果父目录不存在，则创建 + String parentDir = path.substring(0, path.lastIndexOf('/')); + zkclientx.createPersistent(parentDir, true); + zkclientx.createEphemeral(path); + } catch (ZkNodeExistsException e) { + // ignore + // 因为第一次启动时创建了cid,但在stop/start的时可能会关闭和新建,允许出现NodeExists问题s + } + + } + } + + private void releaseCid(String path) { + // logger.info("## release the canalId = {}", cid); + // 初始化系统目录 + if (zkclientx != null) { + zkclientx.delete(path); + } + } + + public CanalMQStarter getCanalMQStarter() { + return canalMQStarter; + } + + public void setCanalMQStarter(CanalMQStarter canalMQStarter) { + this.canalMQStarter = canalMQStarter; + } + + public Map getInstanceConfigMonitors() { + return instanceConfigMonitors; + } + + public Map getInstanceConfigs() { + return instanceConfigs; + } + +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalLauncher.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalLauncher.java new file mode 100644 index 0000000..0cba780 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalLauncher.java @@ -0,0 +1,141 @@ +package cc.smtweb.system.canal.deployer; + +import com.alibaba.otter.canal.common.utils.AddressUtils; +import com.alibaba.otter.canal.common.utils.NamedThreadFactory; +import com.alibaba.otter.canal.instance.manager.plain.PlainCanal; +import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient; +import org.apache.commons.lang.BooleanUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileInputStream; +import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * canal独立版本启动的入口类 + * + * @author jianghang 2012-11-6 下午05:20:49 + * @version 1.0.0 + */ +public class CanalLauncher { + + private static final String CLASSPATH_URL_PREFIX = "classpath:"; + private static final Logger logger = LoggerFactory.getLogger(CanalLauncher.class); + public static final CountDownLatch runningLatch = new CountDownLatch(1); + private static ScheduledExecutorService executor = Executors.newScheduledThreadPool(1, + new NamedThreadFactory("canal-server-scan")); + + public static void main(String[] args) { + startServer(); + } + + public static void startServer() { + try { + logger.info("## canal sever start begin"); + logger.info("## set default uncaught exception handler"); + setGlobalUncaughtExceptionHandler(); + + // 支持rocketmq client 配置日志路径 + System.setProperty("rocketmq.client.logUseSlf4j", "true"); + + logger.info("## load canal configurations"); + String conf = System.getProperty("canal.conf", "classpath:canal.properties"); + Properties properties = new Properties(); + if (conf.startsWith(CLASSPATH_URL_PREFIX)) { + conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX); + properties.load(CanalLauncher.class.getClassLoader().getResourceAsStream(conf)); + } else { + properties.load(new FileInputStream(conf)); + } + + final CanalStarter canalStater = new CanalStarter(properties); + String managerAddress = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER); + if (StringUtils.isNotEmpty(managerAddress)) { + String user = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_USER); + String passwd = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PASSWD); + String adminPort = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PORT, "11110"); + boolean autoRegister = BooleanUtils.toBoolean(CanalController.getProperty(properties, + CanalConstants.CANAL_ADMIN_AUTO_REGISTER)); + String autoCluster = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_AUTO_CLUSTER); + String name = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_REGISTER_NAME); + if (StringUtils.isEmpty(name)) { + name = AddressUtils.getHostName(); + } + + String registerIp = CanalController.getProperty(properties, CanalConstants.CANAL_REGISTER_IP); + if (StringUtils.isEmpty(registerIp)) { + registerIp = AddressUtils.getHostIp(); + } + final PlainCanalConfigClient configClient = new PlainCanalConfigClient(managerAddress, + user, + passwd, + registerIp, + Integer.parseInt(adminPort), + autoRegister, + autoCluster, + name); + PlainCanal canalConfig = configClient.findServer(null); + if (canalConfig == null) { + throw new IllegalArgumentException("managerAddress:" + managerAddress + + " can't not found config for [" + registerIp + ":" + adminPort + + "]"); + } + Properties managerProperties = canalConfig.getProperties(); + // merge local + managerProperties.putAll(properties); + int scanIntervalInSecond = Integer.valueOf(CanalController.getProperty(managerProperties, + CanalConstants.CANAL_AUTO_SCAN_INTERVAL, + "5")); + executor.scheduleWithFixedDelay(new Runnable() { + + private PlainCanal lastCanalConfig; + + public void run() { + try { + if (lastCanalConfig == null) { + lastCanalConfig = configClient.findServer(null); + } else { + PlainCanal newCanalConfig = configClient.findServer(lastCanalConfig.getMd5()); + if (newCanalConfig != null) { + // 远程配置canal.properties修改重新加载整个应用 + canalStater.stop(); + Properties managerProperties = newCanalConfig.getProperties(); + // merge local + managerProperties.putAll(properties); + canalStater.setProperties(managerProperties); + canalStater.start(); + + lastCanalConfig = newCanalConfig; + } + } + + } catch (Throwable e) { + logger.error("scan failed", e); + } + } + + }, 0, scanIntervalInSecond, TimeUnit.SECONDS); + canalStater.setProperties(managerProperties); + } else { + canalStater.setProperties(properties); + } + + canalStater.start(); + logger.info("## canal server start success"); + runningLatch.await(); + executor.shutdownNow(); + } catch (Throwable e) { + logger.error("## Something goes wrong when starting up the canal Server:", e); + } + } + + private static void setGlobalUncaughtExceptionHandler() { + Thread.setDefaultUncaughtExceptionHandler((t, e) -> logger.error("UnCaughtException", e)); + } + +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalStartedListener.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalStartedListener.java new file mode 100644 index 0000000..2721d91 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalStartedListener.java @@ -0,0 +1,39 @@ +package cc.smtweb.system.canal.deployer; + +import cc.smtweb.framework.core.annotation.SwStartListener; +import cc.smtweb.framework.core.common.SwConsts; +import cc.smtweb.framework.core.mvc.controller.IStartListener; +import cc.smtweb.framework.core.systask.SysThreadPool; +import cc.smtweb.framework.core.systask.SysThreadWorker; +import lombok.extern.slf4j.Slf4j; + +/** + * @Author yaoq + * @Date 2022年09月06日 10:31 + * @Description + */ +@Slf4j +@SwStartListener +public class CanalStartedListener implements IStartListener { + + @Override + public int order() { + return SwConsts.DEFAULT_ORDER + 2; + } + + + @Override + public void init() { + + } + + @Override + public void run() { + SysThreadPool.getInstance().addTask(new SysThreadWorker("canal server") { + @Override + public void localWork() throws Exception { + CanalLauncher.startServer(); + } + }); + } +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalStarter.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalStarter.java new file mode 100644 index 0000000..f6dfbb0 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/CanalStarter.java @@ -0,0 +1,169 @@ +package cc.smtweb.system.canal.deployer; + +import java.util.Properties; + +import com.alibaba.otter.canal.connector.core.config.MQProperties; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.admin.netty.CanalAdminWithNetty; +import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer; +import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader; +import cc.smtweb.system.canal.deployer.admin.CanalAdminController; +import com.alibaba.otter.canal.server.CanalMQStarter; + +/** + * Canal server 启动类 + * + * @author rewerma 2020-01-27 + * @version 1.0.2 + */ +public class CanalStarter { + + private static final Logger logger = LoggerFactory.getLogger(CanalStarter.class); + + private static final String CONNECTOR_SPI_DIR = "/plugin"; + private static final String CONNECTOR_STANDBY_SPI_DIR = "/canal/plugin"; + + private CanalController controller = null; + private CanalMQProducer canalMQProducer = null; + private Thread shutdownThread = null; + private CanalMQStarter canalMQStarter = null; + private volatile Properties properties; + private volatile boolean running = false; + + private CanalAdminWithNetty canalAdmin; + + public CanalStarter(Properties properties){ + this.properties = properties; + } + + public boolean isRunning() { + return running; + } + + public Properties getProperties() { + return properties; + } + + public void setProperties(Properties properties) { + this.properties = properties; + } + + public CanalController getController() { + return controller; + } + + /** + * 启动方法 + * + * @throws Throwable + */ + public synchronized void start() throws Throwable { + String serverMode = CanalController.getProperty(properties, CanalConstants.CANAL_SERVER_MODE); + if (!"tcp".equalsIgnoreCase(serverMode)) { + ExtensionLoader loader = ExtensionLoader.getExtensionLoader(CanalMQProducer.class); + canalMQProducer = loader + .getExtension(serverMode.toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR); + if (canalMQProducer != null) { + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(canalMQProducer.getClass().getClassLoader()); + canalMQProducer.init(properties); + Thread.currentThread().setContextClassLoader(cl); + } + } + + if (canalMQProducer != null) { + MQProperties mqProperties = canalMQProducer.getMqProperties(); + // disable netty + System.setProperty(CanalConstants.CANAL_WITHOUT_NETTY, "true"); + if (mqProperties.isFlatMessage()) { + // 设置为raw避免ByteString->Entry的二次解析 + System.setProperty("canal.instance.memory.rawEntry", "false"); + } + } + + logger.info("## start the canal server."); + controller = new CanalController(properties); + controller.start(); + logger.info("## the canal server is running now ......"); + shutdownThread = new Thread(() -> { + try { + logger.info("## stop the canal server"); + controller.stop(); + CanalLauncher.runningLatch.countDown(); + } catch (Throwable e) { + logger.warn("##something goes wrong when stopping canal Server:", e); + } finally { + logger.info("## canal server is down."); + } + }); + Runtime.getRuntime().addShutdownHook(shutdownThread); + + if (canalMQProducer != null) { + canalMQStarter = new CanalMQStarter(canalMQProducer); + String destinations = CanalController.getProperty(properties, CanalConstants.CANAL_DESTINATIONS); + canalMQStarter.start(destinations); + controller.setCanalMQStarter(canalMQStarter); + } + + // start canalAdmin + String port = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PORT); + if (canalAdmin == null && StringUtils.isNotEmpty(port)) { + String user = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_USER); + String passwd = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PASSWD); + CanalAdminController canalAdmin = new CanalAdminController(this); + canalAdmin.setUser(user); + canalAdmin.setPasswd(passwd); + + String ip = CanalController.getProperty(properties, CanalConstants.CANAL_IP); + + logger.debug("canal admin port:{}, canal admin user:{}, canal admin password: {}, canal ip:{}", + port, + user, + passwd, + ip); + + CanalAdminWithNetty canalAdminWithNetty = CanalAdminWithNetty.instance(); + canalAdminWithNetty.setCanalAdmin(canalAdmin); + canalAdminWithNetty.setPort(Integer.parseInt(port)); + canalAdminWithNetty.setIp(ip); + canalAdminWithNetty.start(); + this.canalAdmin = canalAdminWithNetty; + } + + running = true; + } + + public synchronized void stop() throws Throwable { + stop(false); + } + + /** + * 销毁方法，远程配置变更时调用 + * + * @throws Throwable + */ + public synchronized void stop(boolean stopByAdmin) throws Throwable { + if (!stopByAdmin && canalAdmin != null) { + canalAdmin.stop(); + canalAdmin = null; + } + + if (controller != null) { + controller.stop(); + controller = null; + } + if (shutdownThread != null) { + Runtime.getRuntime().removeShutdownHook(shutdownThread); + shutdownThread = null; + } + if (canalMQProducer != null && canalMQStarter != null) { + canalMQStarter.destroy(); + canalMQStarter = null; + canalMQProducer = null; + } + running = false; + } +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/InstanceConfig.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/InstanceConfig.java new file mode 100644 index 0000000..3fa9c8c --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/InstanceConfig.java @@ -0,0 +1,93 @@ +package cc.smtweb.system.canal.deployer; + +import org.apache.commons.lang.builder.ToStringBuilder; + +import com.alibaba.otter.canal.common.utils.CanalToStringStyle; + +/** + * 启动的相关配置 + * + * @author jianghang 2012-11-8 下午02:50:54 + * @version 1.0.0 + */ +public class InstanceConfig { + + private InstanceConfig globalConfig; + private InstanceMode mode; + private Boolean lazy; + private String managerAddress; + private String springXml; + + public InstanceConfig(){ + + } + + public InstanceConfig(InstanceConfig globalConfig){ + this.globalConfig = globalConfig; + } + + public static enum InstanceMode { + SPRING, MANAGER; + + public boolean isSpring() { + return this == InstanceMode.SPRING; + } + + public boolean isManager() { + return this == InstanceMode.MANAGER; + } + } + + public Boolean getLazy() { + if (lazy == null && globalConfig != null) { + return globalConfig.getLazy(); + } else { + return lazy; + } + } + + public void setLazy(Boolean lazy) { + this.lazy = lazy; + } + + public InstanceMode getMode() { + if (mode == null && globalConfig != null) { + return globalConfig.getMode(); + } else { + return mode; + } + } + + public void setMode(InstanceMode mode) { + this.mode = mode; + } + + public String getManagerAddress() { + if (managerAddress == null && globalConfig != null) { + return globalConfig.getManagerAddress(); + } else { + return managerAddress; + } + } + + public void setManagerAddress(String managerAddress) { + this.managerAddress = managerAddress; + } + + public String getSpringXml() { + if (springXml == null && globalConfig != null) { + return globalConfig.getSpringXml(); + } else { + return springXml; + } + } + + public void setSpringXml(String springXml) { + this.springXml = springXml; + } + + public String toString() { + return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); + } + +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/admin/CanalAdminController.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/admin/CanalAdminController.java new file mode 100644 index 0000000..cc58950 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/admin/CanalAdminController.java @@ -0,0 +1,256 @@ +package cc.smtweb.system.canal.deployer.admin; + +import java.io.File; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import cc.smtweb.system.canal.deployer.CanalStarter; +import cc.smtweb.system.canal.deployer.monitor.InstanceConfigMonitor; +import cc.smtweb.system.canal.deployer.monitor.ManagerInstanceConfigMonitor; +import cc.smtweb.system.canal.deployer.monitor.SpringInstanceConfigMonitor; +import cc.smtweb.system.canal.deployer.monitor.InstanceAction; +import org.apache.commons.io.filefilter.TrueFileFilter; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.admin.CanalAdmin; +import com.alibaba.otter.canal.common.utils.FileUtils; +import cc.smtweb.system.canal.deployer.InstanceConfig; +import com.alibaba.otter.canal.instance.core.CanalInstance; +import com.alibaba.otter.canal.protocol.SecurityUtil; +import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded; +import com.google.common.base.Joiner; + +/** + * 提供canal admin的管理操作 + * + * @author agapple 2019年8月24日 下午11:39:01 + * @since 1.1.4 + */ +public class CanalAdminController implements CanalAdmin { + + private static final Logger logger = LoggerFactory.getLogger(CanalAdminController.class); + private String user; + private String passwd; + private CanalStarter canalStater; + + public CanalAdminController(CanalStarter canalStater){ + this.canalStater = canalStater; + } + + @Override + public boolean check() { + return canalStater.isRunning(); + } + + @Override + public synchronized boolean start() { + try { + if (!canalStater.isRunning()) { + canalStater.start(); + return true; + } + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } + return false; + } + + @Override + public synchronized boolean stop() { + try { + if (canalStater.isRunning()) { + canalStater.stop(true); + return true; + } + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } + return false; + } + + @Override + public synchronized boolean restart() { + stop(); + return start(); + } + + @Override + public boolean auth(String user, String passwd, byte[] seed) { + // 如果user/passwd密码为空,则任何用户账户都能登录 + if ((StringUtils.isEmpty(this.user) || StringUtils.equals(this.user, user))) { + if (StringUtils.isEmpty(this.passwd)) { + return true; + } else if (StringUtils.isEmpty(passwd)) { + // 如果server密码有配置,客户端密码为空,则拒绝 + return false; + } + + try { + byte[] passForClient = SecurityUtil.hexStr2Bytes(passwd); + return SecurityUtil.scrambleServerAuth(passForClient, SecurityUtil.hexStr2Bytes(this.passwd), seed); + } catch (NoSuchAlgorithmException e) { + return false; + } + } + + return false; + } + + @Override + public String getRunningInstances() { + try { + Map instances = CanalServerWithEmbedded.instance().getCanalInstances(); + List runningInstances = new ArrayList<>(); + instances.forEach((destination, instance) -> { + if (instance.isStart()) { + runningInstances.add(destination); + } + }); + + return Joiner.on(",").join(runningInstances); + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } + return ""; + } + + @Override + public boolean checkInstance(String destination) { + Map instances = CanalServerWithEmbedded.instance().getCanalInstances(); + if (instances == null || !instances.containsKey(destination)) { + return false; + } else { + CanalInstance instance = instances.get(destination); + return instance.isStart(); + } + } + + @Override + public boolean startInstance(String destination) { + try { + InstanceAction instanceAction = getInstanceAction(destination); + if (instanceAction != null) { + instanceAction.start(destination); + return true; + } + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } + return false; + } + + @Override + public boolean stopInstance(String destination) { + try { + InstanceAction instanceAction = getInstanceAction(destination); + if (instanceAction != null) { + instanceAction.stop(destination); + return true; + } + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } + return false; + } + + @Override + public boolean releaseInstance(String destination) { + try { + InstanceAction instanceAction = getInstanceAction(destination); + if (instanceAction != null) { + instanceAction.release(destination); + return true; + } + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } + return false; + } + + @Override + public boolean restartInstance(String destination) { + try { + InstanceAction instanceAction = getInstanceAction(destination); + if (instanceAction != null) { + instanceAction.reload(destination); + return true; + } + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } + return false; + } + + @Override + public String listCanalLog() { + Collection files = org.apache.commons.io.FileUtils.listFiles(new File("../logs/canal/"), + TrueFileFilter.TRUE, + TrueFileFilter.TRUE); + List names = files.stream().map(File::getName).collect(Collectors.toList()); + return Joiner.on(",").join(names); + } + + @Override + public String canalLog(int lines) { + return FileUtils.readFileFromOffset("../logs/canal/canal.log", lines, "UTF-8"); + } + + @Override + public String listInstanceLog(String destination) { + Collection files = org.apache.commons.io.FileUtils.listFiles(new File("../logs/" + destination + "/"), + TrueFileFilter.TRUE, + TrueFileFilter.TRUE); + List names = files.stream().map(File::getName).collect(Collectors.toList()); + return Joiner.on(",").join(names); + } + + @Override + public String instanceLog(String destination, String fileName, int lines) { + if (StringUtils.isEmpty(fileName)) { + fileName = destination + ".log"; + } + return FileUtils.readFileFromOffset("../logs/" + destination + "/" + fileName, lines, "UTF-8"); + } + + private InstanceAction getInstanceAction(String destination) { + Map monitors = canalStater.getController() + .getInstanceConfigMonitors(); + + InstanceAction instanceAction = null; + if (monitors.containsKey(InstanceConfig.InstanceMode.SPRING)) { + SpringInstanceConfigMonitor monitor = (SpringInstanceConfigMonitor) monitors.get(InstanceConfig.InstanceMode.SPRING); + Map instanceActions = monitor.getActions(); + instanceAction = instanceActions.get(destination); + } + + if (instanceAction != null) { + return instanceAction; + } + + if (monitors.containsKey(InstanceConfig.InstanceMode.MANAGER)) { + ManagerInstanceConfigMonitor monitor = (ManagerInstanceConfigMonitor) monitors.get(InstanceConfig.InstanceMode.MANAGER); + Map instanceActions = monitor.getActions(); + instanceAction = instanceActions.get(destination); + } + return instanceAction; + } + + public void setUser(String user) { + this.user = user; + } + + public void setPasswd(String passwd) { + this.passwd = passwd; + } + + public void setCanalStater(CanalStarter canalStater) { + this.canalStater = canalStater; + } + +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/monitor/InstanceAction.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/monitor/InstanceAction.java new file mode 100644 index 0000000..7354a3d --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/monitor/InstanceAction.java @@ -0,0 +1,30 @@ +package cc.smtweb.system.canal.deployer.monitor; + +/** + * config配置变化 + * + * @author jianghang 2013-2-18 下午01:19:29 + * @version 1.0.1 + */ +public interface InstanceAction { + + /** + * 启动destination + */ + void start(String destination); + + /** + * 主动释放destination运行 + */ + void release(String destination); + + /** + * 停止destination + */ + void stop(String destination); + + /** + * 重载destination，可能需要stop,start操作，或者只是更新下内存配置 + */ + void reload(String destination); +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/monitor/InstanceConfigMonitor.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/monitor/InstanceConfigMonitor.java new file mode 100644 index 0000000..cdc3852 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/monitor/InstanceConfigMonitor.java @@ -0,0 +1,16 @@ +package cc.smtweb.system.canal.deployer.monitor; + +import com.alibaba.otter.canal.common.CanalLifeCycle; + +/** + * 监听instance file的文件变化，触发instance start/stop等操作 + * + * @author jianghang 2013-2-6 下午06:19:56 + * @version 1.0.1 + */ +public interface InstanceConfigMonitor extends CanalLifeCycle { + + void register(String destination, InstanceAction action); + + void unregister(String destination); +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/monitor/ManagerInstanceConfigMonitor.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/monitor/ManagerInstanceConfigMonitor.java new file mode 100644 index 0000000..ca413fa --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/monitor/ManagerInstanceConfigMonitor.java @@ -0,0 +1,184 @@ +package cc.smtweb.system.canal.deployer.monitor; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; +import com.alibaba.otter.canal.common.CanalLifeCycle; +import com.alibaba.otter.canal.common.utils.NamedThreadFactory; +import com.alibaba.otter.canal.instance.manager.plain.PlainCanal; +import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient; +import com.google.common.collect.Lists; +import com.google.common.collect.MapMaker; +import com.google.common.collect.MigrateMap; + +/** + * 基于manager配置的实现 + * + * @author agapple 2019年8月26日 下午10:00:20 + * @since 1.1.4 + */ +public class ManagerInstanceConfigMonitor extends AbstractCanalLifeCycle implements InstanceConfigMonitor, CanalLifeCycle { + + private static final Logger logger = LoggerFactory.getLogger(ManagerInstanceConfigMonitor.class); + private long scanIntervalInSecond = 5; + private InstanceAction defaultAction = null; + private Map actions = new MapMaker().makeMap(); + private Map configs = MigrateMap.makeComputingMap(destination -> new PlainCanal()); + private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1, + new NamedThreadFactory("canal-instance-scan")); + + private volatile boolean isFirst = true; + private PlainCanalConfigClient configClient; + + public void start() { + super.start(); + executor.scheduleWithFixedDelay(() -> { + try { + scan(); + if (isFirst) { + isFirst = false; + } + } catch (Throwable e) { + logger.error("scan failed", e); + } + }, 0, scanIntervalInSecond, TimeUnit.SECONDS); + } + + public void stop() { + super.stop(); + executor.shutdownNow(); + actions.clear(); + } + + public void register(String destination, InstanceAction action) { + if (action != null) { + actions.put(destination, action); + } else { + actions.put(destination, defaultAction); + } + } + + public void unregister(String destination) { + actions.remove(destination); + } + + private void scan() { + String instances = configClient.findInstances(null); + if (instances == null) { + return; + } + + final List is = Lists.newArrayList(StringUtils.split(instances, ',')); + List start = new ArrayList<>(); + List stop = new ArrayList<>(); + List restart = new ArrayList<>(); + for (String instance : is) { + if (!configs.containsKey(instance)) { + PlainCanal newPlainCanal = configClient.findInstance(instance, null); + if (newPlainCanal != null) { + configs.put(instance, newPlainCanal); + start.add(instance); + } + } else { + PlainCanal plainCanal = configs.get(instance); + PlainCanal newPlainCanal = configClient.findInstance(instance, plainCanal.getMd5()); + if (newPlainCanal != null) { + // 配置有变化 + restart.add(instance); + configs.put(instance, newPlainCanal); + } + } + } + + configs.forEach((instance, plainCanal) -> { + if (!is.contains(instance)) { + stop.add(instance); + } + }); + + stop.forEach(instance -> { + notifyStop(instance); + }); + + restart.forEach(instance -> { + notifyReload(instance); + }); + + start.forEach(instance -> { + notifyStart(instance); + }); + + } + + private void notifyStart(String destination) { + try { + defaultAction.start(destination); + actions.put(destination, defaultAction); + // 启动成功后记录配置文件信息 + } catch (Throwable e) { + logger.error(String.format("scan add found[%s] but start failed", destination), e); + } + } + + private void notifyStop(String destination) { + InstanceAction action = actions.remove(destination); + if (action != null) { + try { + action.stop(destination); + configs.remove(destination); + } catch (Throwable e) { + logger.error(String.format("scan delete found[%s] but stop failed", destination), e); + actions.put(destination, action);// 再重新加回去，下一次scan时再执行删除 + } + } + } + + private void notifyReload(String destination) { + InstanceAction action = actions.get(destination); + if (action != null) { + try { + action.reload(destination); + } catch (Throwable e) { + logger.error(String.format("scan reload found[%s] but reload failed", destination), e); + } + } + } + + public void release(String destination) { + InstanceAction action = actions.remove(destination); + if (action != null) { + try { + configs.remove(destination); + } catch (Throwable e) { + logger.error(String.format("scan delete found[%s] but stop failed", destination), e); + actions.put(destination, action);// 再重新加回去，下一次scan时再执行删除 + } + } + } + + public void setDefaultAction(InstanceAction defaultAction) { + this.defaultAction = defaultAction; + } + + public void setScanIntervalInSecond(long scanIntervalInSecond) { + this.scanIntervalInSecond = scanIntervalInSecond; + } + + public void setConfigClient(PlainCanalConfigClient configClient) { + this.configClient = configClient; + } + + public Map getActions() { + return actions; + } + +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/monitor/SpringInstanceConfigMonitor.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/monitor/SpringInstanceConfigMonitor.java new file mode 100644 index 0000000..9d63a5c --- /dev/null +++ b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/deployer/monitor/SpringInstanceConfigMonitor.java @@ -0,0 +1,297 @@ +package cc.smtweb.system.canal.deployer.monitor; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; + +import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; +import com.alibaba.otter.canal.common.CanalLifeCycle; +import com.alibaba.otter.canal.common.utils.NamedThreadFactory; +import com.google.common.collect.MapMaker; +import com.google.common.collect.MigrateMap; + +/** + * 监听基于spring配置的instance变化 + * + * @author jianghang 2013-2-6 下午06:23:55 + * @version 1.0.1 + */ +public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implements InstanceConfigMonitor, CanalLifeCycle { + + private static final Logger logger = LoggerFactory.getLogger(SpringInstanceConfigMonitor.class); + private String rootConf; + // 扫描周期，单位秒 + private long scanIntervalInSecond = 5; + private InstanceAction defaultAction = null; + private Map actions = new MapMaker().makeMap(); + private Map lastFiles = MigrateMap.makeComputingMap(InstanceConfigFiles::new); + private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1, + new NamedThreadFactory("canal-instance-scan")); + + private volatile boolean isFirst = true; + + public Map getActions() { + return actions; + } + + public void start() { + super.start(); + Assert.notNull(rootConf, "root conf dir is null!"); + + executor.scheduleWithFixedDelay(() -> { + try { + scan(); + if (isFirst) { + isFirst = false; + } + } catch (Throwable e) { + logger.error("scan failed", e); + } + }, 0, scanIntervalInSecond, TimeUnit.SECONDS); + } + + public void stop() { + super.stop(); + executor.shutdownNow(); + actions.clear(); + lastFiles.clear(); + } + + public void register(String destination, InstanceAction action) { + if (action != null) { + actions.put(destination, action); + } else { + actions.put(destination, defaultAction); + } + } + + public void unregister(String destination) { + actions.remove(destination); + } + + public void setRootConf(String rootConf) { + this.rootConf = rootConf; + } + + private void scan() { + File rootdir = new File(rootConf); + if (!rootdir.exists()) { + return; + } + + File[] instanceDirs = rootdir.listFiles(pathname -> { + String filename = pathname.getName(); + return pathname.isDirectory() && !"spring".equalsIgnoreCase(filename); + }); + + // 扫描目录的新增 + Set currentInstanceNames = new HashSet<>(); + + // 判断目录内文件的变化 + for (File instanceDir : instanceDirs) { + String destination = instanceDir.getName(); + currentInstanceNames.add(destination); + File[] instanceConfigs = instanceDir.listFiles((dir, name) -> { + // return !StringUtils.endsWithIgnoreCase(name, ".dat"); + // 限制一下，只针对instance.properties文件,避免因为.svn或者其他生成的临时文件导致出现reload + return StringUtils.equalsIgnoreCase(name, "instance.properties"); + }); + + if (!actions.containsKey(destination) && instanceConfigs.length > 0) { + // 存在合法的instance.properties，并且第一次添加时，进行启动操作 + notifyStart(instanceDir, destination, instanceConfigs); + } else if (actions.containsKey(destination)) { + // 历史已经启动过 + if (instanceConfigs.length == 0) { // 如果不存在合法的instance.properties + notifyStop(destination); + } else { + InstanceConfigFiles lastFile = lastFiles.get(destination); + // 历史启动过 所以配置文件信息必然存在 + if (!isFirst && CollectionUtils.isEmpty(lastFile.getInstanceFiles())) { + logger.error("[{}] is started, but not found instance file info.", destination); + } + + boolean hasChanged = judgeFileChanged(instanceConfigs, lastFile.getInstanceFiles()); + // 通知变化 + if (hasChanged) { + notifyReload(destination); + } + + if (hasChanged || CollectionUtils.isEmpty(lastFile.getInstanceFiles())) { + // 更新内容 + List newFileInfo = new ArrayList<>(); + for (File instanceConfig : instanceConfigs) { + newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified())); + } + + lastFile.setInstanceFiles(newFileInfo); + } + } + } + + } + + // 判断目录是否删除 + Set deleteInstanceNames = new HashSet<>(); + for (String destination : actions.keySet()) { + if (!currentInstanceNames.contains(destination)) { + deleteInstanceNames.add(destination); + } + } + for (String deleteInstanceName : deleteInstanceNames) { + notifyStop(deleteInstanceName); + } + } + + private void notifyStart(File instanceDir, String destination, File[] instanceConfigs) { + try { + defaultAction.start(destination); + actions.put(destination, defaultAction); + + // 启动成功后记录配置文件信息 + InstanceConfigFiles lastFile = lastFiles.get(destination); + List newFileInfo = new ArrayList<>(); + for (File instanceConfig : instanceConfigs) { + newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified())); + } + lastFile.setInstanceFiles(newFileInfo); + } catch (Throwable e) { + logger.error(String.format("scan add found[%s] but start failed", destination), e); + } + } + + private void notifyStop(String destination) { + InstanceAction action = actions.remove(destination); + if (action != null) { + try { + action.stop(destination); + lastFiles.remove(destination); + } catch (Throwable e) { + logger.error(String.format("scan delete found[%s] but stop failed", destination), e); + actions.put(destination, action);// 再重新加回去，下一次scan时再执行删除 + } + } + } + + private void notifyReload(String destination) { + InstanceAction action = actions.get(destination); + if (action != null) { + try { + action.reload(destination); + } catch (Throwable e) { + logger.error(String.format("scan reload found[%s] but reload failed", destination), e); + } + } + } + + private boolean judgeFileChanged(File[] instanceConfigs, List fileInfos) { + boolean hasChanged = false; + for (File instanceConfig : instanceConfigs) { + for (FileInfo fileInfo : fileInfos) { + if (instanceConfig.getName().equals(fileInfo.getName())) { + hasChanged |= (instanceConfig.lastModified() != fileInfo.getLastModified()); + if (hasChanged) { + return hasChanged; + } + } + } + } + + return hasChanged; + } + + public void setDefaultAction(InstanceAction defaultAction) { + this.defaultAction = defaultAction; + } + + public void setScanIntervalInSecond(long scanIntervalInSecond) { + this.scanIntervalInSecond = scanIntervalInSecond; + } + + public static class InstanceConfigFiles { + + private String destination; // instance + // name + private List springFile = new ArrayList<>(); // spring的instance + // xml + private FileInfo rootFile; // canal.properties + private List instanceFiles = new ArrayList<>(); // instance对应的配置 + + public InstanceConfigFiles(String destination){ + this.destination = destination; + } + + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + + public List getSpringFile() { + return springFile; + } + + public void setSpringFile(List springFile) { + this.springFile = springFile; + } + + public FileInfo getRootFile() { + return rootFile; + } + + public void setRootFile(FileInfo rootFile) { + this.rootFile = rootFile; + } + + public List getInstanceFiles() { + return instanceFiles; + } + + public void setInstanceFiles(List instanceFiles) { + this.instanceFiles = instanceFiles; + } + + } + + public static class FileInfo { + + private String name; + private long lastModified = 0; + + public FileInfo(String name, long lastModified){ + this.name = name; + this.lastModified = lastModified; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public long getLastModified() { + return lastModified; + } + + public void setLastModified(long lastModified) { + this.lastModified = lastModified; + } + + } +} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalApplication.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalApplication.java deleted file mode 100644 index 7a644ac..0000000 --- a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalApplication.java +++ /dev/null @@ -1,13 +0,0 @@ -package cc.smtweb.system.canal.server; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; - -@SpringBootApplication -public class CanalApplication { - - public static void main(String[] args) { - SpringApplication.run(CanalApplication.class, args); - } - -} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalConstants.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalConstants.java deleted file mode 100644 index d9a6a48..0000000 --- a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalConstants.java +++ /dev/null @@ -1,97 +0,0 @@ -package cc.smtweb.system.canal.server; - -import java.text.MessageFormat; - -/** - * 启动常用变量 - * - * @author jianghang 2012-11-8 下午03:15:55 - * @version 1.0.0 - */ -public class CanalConstants { - - public static final String MDC_DESTINATION = "destination"; - public static final String ROOT = "canal"; - public static final String CANAL_ID = ROOT + "." + "id"; - public static final String CANAL_IP = ROOT + "." + "ip"; - public static final String CANAL_REGISTER_IP = ROOT + "." + "register.ip"; - public static final String CANAL_PORT = ROOT + "." + "port"; - public static final String CANAL_USER = ROOT + "." + "user"; - public static final String CANAL_PASSWD = ROOT + "." + "passwd"; - public static final String CANAL_METRICS_PULL_PORT = ROOT + "." + "metrics.pull.port"; - public static final String CANAL_ADMIN_MANAGER = ROOT + "." + "admin.manager"; - public static final String CANAL_ADMIN_PORT = ROOT + "." + "admin.port"; - public static final String CANAL_ADMIN_USER = ROOT + "." + "admin.user"; - public static final String CANAL_ADMIN_PASSWD = ROOT + "." + "admin.passwd"; - public static final String CANAL_ADMIN_AUTO_REGISTER = ROOT + "." + "admin.register.auto"; - public static final String CANAL_ADMIN_AUTO_CLUSTER = ROOT + "." + "admin.register.cluster"; - public static final String CANAL_ADMIN_REGISTER_NAME = ROOT + "." + "admin.register.name"; - public static final String CANAL_ZKSERVERS = ROOT + "." + "zkServers"; - public static final String CANAL_WITHOUT_NETTY = ROOT + "." + "withoutNetty"; - - public static final String CANAL_DESTINATIONS = ROOT + "." + "destinations"; - public static final String CANAL_AUTO_SCAN = ROOT + "." + "auto.scan"; - public static final String CANAL_AUTO_SCAN_INTERVAL = ROOT + "." + "auto.scan.interval"; - public static final String CANAL_CONF_DIR = ROOT + "." + "conf.dir"; - public static final String CANAL_SERVER_MODE = ROOT + "." + "serverMode"; - - public static final String CANAL_DESTINATION_SPLIT = ","; - public static final String GLOBAL_NAME = "global"; - - public static final String INSTANCE_MODE_TEMPLATE = ROOT + "." + "instance.{0}.mode"; - public static final String INSTANCE_LAZY_TEMPLATE = ROOT + "." + "instance.{0}.lazy"; - public static final String INSTANCE_MANAGER_ADDRESS_TEMPLATE = ROOT + "." + "instance.{0}.manager.address"; - public static final String INSTANCE_SPRING_XML_TEMPLATE = ROOT + "." + "instance.{0}.spring.xml"; - - public static final String CANAL_DESTINATION_PROPERTY = ROOT + ".instance.destination"; - - public static final String CANAL_SOCKETCHANNEL = ROOT + "." + "socketChannel"; - - public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey"; - public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey"; - -// public static final String CANAL_MQ_SERVERS = ROOT + "." + "mq.servers"; -// public static final String CANAL_MQ_RETRIES = ROOT + "." + "mq.retries"; -// public static final String CANAL_MQ_BATCHSIZE = ROOT + "." + "mq.batchSize"; -// public static final String CANAL_MQ_LINGERMS = ROOT + "." + "mq.lingerMs"; -// public static final String CANAL_MQ_MAXREQUESTSIZE = ROOT + "." + "mq.maxRequestSize"; -// public static final String CANAL_MQ_BUFFERMEMORY = ROOT + "." + "mq.bufferMemory"; -// public static final String CANAL_MQ_CANALBATCHSIZE = ROOT + "." + "mq.canalBatchSize"; -// public static final String CANAL_MQ_CANALGETTIMEOUT = ROOT + "." + "mq.canalGetTimeout"; -// public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage"; -// public static final String CANAL_MQ_PARALLELTHREADSIZE = ROOT + "." + "mq.parallelThreadSize"; -// public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType"; -// public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks"; -// public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction"; -// public static final String CANAL_MQ_PRODUCERGROUP = ROOT + "." + "mq.producerGroup"; -// public static final String CANAL_MQ_PROPERTIES = ROOT + "." + "mq.properties"; -// public static final String CANAL_MQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "mq.enableMessageTrace"; -// public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.accessChannel"; -// public static final String CANAL_MQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "mq.customizedTraceTopic"; -// public static final String CANAL_MQ_NAMESPACE = ROOT + "." + "mq.namespace"; -// public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable"; -// public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH = ROOT + "." + "mq.kafka.kerberos.krb5FilePath"; -// public static final String CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH = ROOT + "." + "mq.kafka.kerberos.jaasFilePath"; -// public static final String CANAL_MQ_USERNAME = ROOT + "." + "mq.username"; -// public static final String CANAL_MQ_PASSWORD = ROOT + "." + "mq.password"; -// public static final String CANAL_MQ_VHOST = ROOT + "." + "mq.vhost"; -// public static final String CANAL_MQ_ALIYUN_UID = ROOT + "." + "mq.aliyunuid"; -// public static final String CANAL_MQ_EXCHANGE = ROOT + "." + "mq.exchange"; -// public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash"; - - public static String getInstanceModeKey(String destination) { - return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination); - } - - public static String getInstanceManagerAddressKey(String destination) { - return MessageFormat.format(INSTANCE_MANAGER_ADDRESS_TEMPLATE, destination); - } - - public static String getInstancSpringXmlKey(String destination) { - return MessageFormat.format(INSTANCE_SPRING_XML_TEMPLATE, destination); - } - - public static String getInstancLazyKey(String destination) { - return MessageFormat.format(INSTANCE_LAZY_TEMPLATE, destination); - } -} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalController.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalController.java deleted file mode 100644 index 75f2339..0000000 --- a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalController.java +++ /dev/null @@ -1,606 +0,0 @@ -package cc.smtweb.system.canal.server; - -import java.util.Map; -import java.util.Properties; - -import cc.smtweb.system.canal.server.monitor.InstanceAction; -import cc.smtweb.system.canal.server.monitor.InstanceConfigMonitor; -import cc.smtweb.system.canal.server.monitor.ManagerInstanceConfigMonitor; -import cc.smtweb.system.canal.server.monitor.SpringInstanceConfigMonitor; -import org.I0Itec.zkclient.IZkStateListener; -import org.I0Itec.zkclient.exception.ZkNoNodeException; -import org.I0Itec.zkclient.exception.ZkNodeExistsException; -import org.apache.commons.lang.BooleanUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.zookeeper.Watcher.Event.KeeperState; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.MDC; - -import com.alibaba.otter.canal.common.utils.AddressUtils; -import com.alibaba.otter.canal.common.zookeeper.ZkClientx; -import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; -import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningData; -import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningListener; -import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitor; -import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitors; -import cc.smtweb.system.canal.server.InstanceConfig.InstanceMode; -import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator; -import com.alibaba.otter.canal.instance.manager.PlainCanalInstanceGenerator; -import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient; -import com.alibaba.otter.canal.instance.spring.SpringCanalInstanceGenerator; -import com.alibaba.otter.canal.server.CanalMQStarter; -import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded; -import com.alibaba.otter.canal.server.exception.CanalServerException; -import com.alibaba.otter.canal.server.netty.CanalServerWithNetty; -import com.google.common.base.Function; -import com.google.common.collect.MapMaker; -import com.google.common.collect.MigrateMap; - -/** - * canal调度控制器 - * - * @author jianghang 2012-11-8 下午12:03:11 - * @version 1.0.0 - */ -public class CanalController { - - private static final Logger logger = LoggerFactory.getLogger(CanalController.class); - private String ip; - private String registerIp; - private int port; - private int adminPort; - // 默认使用spring的方式载入 - private Map instanceConfigs; - private InstanceConfig globalInstanceConfig; - private Map managerClients; - // 监听instance config的变化 - private boolean autoScan = true; - private InstanceAction defaultAction; - private Map instanceConfigMonitors; - private CanalServerWithEmbedded embededCanalServer; - private CanalServerWithNetty canalServer; - - private CanalInstanceGenerator instanceGenerator; - private ZkClientx zkclientx; - - private CanalMQStarter canalMQStarter; - private String adminUser; - private String adminPasswd; - - public CanalController(){ - this(System.getProperties()); - } - - public CanalController(final Properties properties){ - managerClients = MigrateMap.makeComputingMap(this::getManagerClient); - - // 初始化全局参数设置 - globalInstanceConfig = initGlobalConfig(properties); - instanceConfigs = new MapMaker().makeMap(); - // 初始化instance config - initInstanceConfig(properties); - - // init socketChannel - String socketChannel = getProperty(properties, CanalConstants.CANAL_SOCKETCHANNEL); - if (StringUtils.isNotEmpty(socketChannel)) { - System.setProperty(CanalConstants.CANAL_SOCKETCHANNEL, socketChannel); - } - - // 兼容1.1.0版本的ak/sk参数名 - String accesskey = getProperty(properties, "canal.instance.rds.accesskey"); - String secretkey = getProperty(properties, "canal.instance.rds.secretkey"); - if (StringUtils.isNotEmpty(accesskey)) { - System.setProperty(CanalConstants.CANAL_ALIYUN_ACCESSKEY, accesskey); - } - if (StringUtils.isNotEmpty(secretkey)) { - System.setProperty(CanalConstants.CANAL_ALIYUN_SECRETKEY, secretkey); - } - - // 准备canal server - ip = getProperty(properties, CanalConstants.CANAL_IP); - registerIp = getProperty(properties, CanalConstants.CANAL_REGISTER_IP); - port = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_PORT, "11111")); - adminPort = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_ADMIN_PORT, "11110")); - embededCanalServer = CanalServerWithEmbedded.instance(); - embededCanalServer.setCanalInstanceGenerator(instanceGenerator);// 设置自定义的instanceGenerator - int metricsPort = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_METRICS_PULL_PORT, "11112")); - embededCanalServer.setMetricsPort(metricsPort); - - this.adminUser = getProperty(properties, CanalConstants.CANAL_ADMIN_USER); - this.adminPasswd = getProperty(properties, CanalConstants.CANAL_ADMIN_PASSWD); - embededCanalServer.setUser(getProperty(properties, CanalConstants.CANAL_USER)); - embededCanalServer.setPasswd(getProperty(properties, CanalConstants.CANAL_PASSWD)); - - String canalWithoutNetty = getProperty(properties, CanalConstants.CANAL_WITHOUT_NETTY); - if (canalWithoutNetty == null || "false".equals(canalWithoutNetty)) { - canalServer = CanalServerWithNetty.instance(); - canalServer.setIp(ip); - canalServer.setPort(port); - } - - // 处理下ip为空，默认使用hostIp暴露到zk中 - if (StringUtils.isEmpty(ip) && StringUtils.isEmpty(registerIp)) { - ip = registerIp = AddressUtils.getHostIp(); - } - - if (StringUtils.isEmpty(ip)) { - ip = AddressUtils.getHostIp(); - } - - if (StringUtils.isEmpty(registerIp)) { - registerIp = ip; // 兼容以前配置 - } - final String zkServers = getProperty(properties, CanalConstants.CANAL_ZKSERVERS); - if (StringUtils.isNotEmpty(zkServers)) { - zkclientx = ZkClientx.getZkClient(zkServers); - // 初始化系统目录 - zkclientx.createPersistent(ZookeeperPathUtils.DESTINATION_ROOT_NODE, true); - zkclientx.createPersistent(ZookeeperPathUtils.CANAL_CLUSTER_ROOT_NODE, true); - } - - final ServerRunningData serverData = new ServerRunningData(registerIp + ":" + port); - ServerRunningMonitors.setServerData(serverData); - ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap((Function) destination -> { - ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData); - runningMonitor.setDestination(destination); - runningMonitor.setListener(new ServerRunningListener() { - - public void processActiveEnter() { - try { - MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); - embededCanalServer.start(destination); - if (canalMQStarter != null) { - canalMQStarter.startDestination(destination); - } - } finally { - MDC.remove(CanalConstants.MDC_DESTINATION); - } - } - - public void processActiveExit() { - try { - MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); - if (canalMQStarter != null) { - canalMQStarter.stopDestination(destination); - } - embededCanalServer.stop(destination); - } finally { - MDC.remove(CanalConstants.MDC_DESTINATION); - } - } - - public void processStart() { - try { - if (zkclientx != null) { - final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, - registerIp + ":" + port); - initCid(path); - zkclientx.subscribeStateChanges(new IZkStateListener() { - - public void handleStateChanged(KeeperState state) throws Exception { - - } - - public void handleNewSession() throws Exception { - initCid(path); - } - - @Override - public void handleSessionEstablishmentError(Throwable error) throws Exception { - logger.error("failed to connect to zookeeper", error); - } - }); - } - } finally { - MDC.remove(CanalConstants.MDC_DESTINATION); - } - } - - public void processStop() { - try { - MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); - if (zkclientx != null) { - final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, - registerIp + ":" + port); - releaseCid(path); - } - } finally { - MDC.remove(CanalConstants.MDC_DESTINATION); - } - } - - }); - if (zkclientx != null) { - runningMonitor.setZkClient(zkclientx); - } - // 触发创建一下cid节点 - runningMonitor.init(); - return runningMonitor; - })); - - // 初始化monitor机制 - autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN)); - if (autoScan) { - defaultAction = new InstanceAction() { - - public void start(String destination) { - InstanceConfig config = instanceConfigs.get(destination); - if (config == null) { - // 重新读取一下instance config - config = parseInstanceConfig(properties, destination); - instanceConfigs.put(destination, config); - } - - if (!embededCanalServer.isStart(destination)) { - // HA机制启动 - ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination); - if (!config.getLazy() && !runningMonitor.isStart()) { - runningMonitor.start(); - } - } - - logger.info("auto notify start {} successful.", destination); - } - - public void stop(String destination) { - // 此处的stop，代表强制退出，非HA机制，所以需要退出HA的monitor和配置信息 - InstanceConfig config = instanceConfigs.remove(destination); - if (config != null) { - embededCanalServer.stop(destination); - ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination); - if (runningMonitor.isStart()) { - runningMonitor.stop(); - } - } - - logger.info("auto notify stop {} successful.", destination); - } - - public void reload(String destination) { - // 目前任何配置变化，直接重启，简单处理 - stop(destination); - start(destination); - - logger.info("auto notify reload {} successful.", destination); - } - - @Override - public void release(String destination) { - // 此处的release，代表强制释放，主要针对HA机制释放运行，让给其他机器抢占 - InstanceConfig config = instanceConfigs.get(destination); - if (config != null) { - ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination); - if (runningMonitor.isStart()) { - boolean release = runningMonitor.release(); - if (!release) { - // 如果是单机模式,则直接清除配置 - instanceConfigs.remove(destination); - // 停掉服务 - runningMonitor.stop(); - if (instanceConfigMonitors.containsKey(InstanceConfig.InstanceMode.MANAGER)) { - ManagerInstanceConfigMonitor monitor = (ManagerInstanceConfigMonitor) instanceConfigMonitors.get(InstanceConfig.InstanceMode.MANAGER); - Map instanceActions = monitor.getActions(); - if (instanceActions.containsKey(destination)) { - // 清除内存中的autoScan cache - monitor.release(destination); - } - } - } - } - } - - logger.info("auto notify release {} successful.", destination); - } - }; - - instanceConfigMonitors = MigrateMap.makeComputingMap(mode -> { - int scanInterval = Integer.valueOf(getProperty(properties, - CanalConstants.CANAL_AUTO_SCAN_INTERVAL, - "5")); - - if (mode.isSpring()) { - SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor(); - monitor.setScanIntervalInSecond(scanInterval); - monitor.setDefaultAction(defaultAction); - // 设置conf目录，默认是user.dir + conf目录组成 - String rootDir = getProperty(properties, CanalConstants.CANAL_CONF_DIR); - if (StringUtils.isEmpty(rootDir)) { - rootDir = "../conf"; - } - - if (StringUtils.equals("otter-canal", System.getProperty("appName"))) { - monitor.setRootConf(rootDir); - } else { - // eclipse debug模式 - monitor.setRootConf("src/main/resources/"); - } - return monitor; - } else if (mode.isManager()) { - ManagerInstanceConfigMonitor monitor = new ManagerInstanceConfigMonitor(); - monitor.setScanIntervalInSecond(scanInterval); - monitor.setDefaultAction(defaultAction); - String managerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER); - monitor.setConfigClient(getManagerClient(managerAddress)); - return monitor; - } else { - throw new UnsupportedOperationException("unknow mode :" + mode + " for monitor"); - } - }); - } - } - - private InstanceConfig initGlobalConfig(Properties properties) { - String adminManagerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER); - InstanceConfig globalConfig = new InstanceConfig(); - String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(CanalConstants.GLOBAL_NAME)); - if (StringUtils.isNotEmpty(adminManagerAddress)) { - // 如果指定了manager地址,则强制适用manager - globalConfig.setMode(InstanceMode.MANAGER); - } else if (StringUtils.isNotEmpty(modeStr)) { - globalConfig.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr))); - } - - String lazyStr = getProperty(properties, CanalConstants.getInstancLazyKey(CanalConstants.GLOBAL_NAME)); - if (StringUtils.isNotEmpty(lazyStr)) { - globalConfig.setLazy(Boolean.valueOf(lazyStr)); - } - - String managerAddress = getProperty(properties, - CanalConstants.getInstanceManagerAddressKey(CanalConstants.GLOBAL_NAME)); - if (StringUtils.isNotEmpty(managerAddress)) { - if (StringUtils.equals(managerAddress, "${canal.admin.manager}")) { - managerAddress = adminManagerAddress; - } - - globalConfig.setManagerAddress(managerAddress); - } - - String springXml = getProperty(properties, CanalConstants.getInstancSpringXmlKey(CanalConstants.GLOBAL_NAME)); - if (StringUtils.isNotEmpty(springXml)) { - globalConfig.setSpringXml(springXml); - } - - instanceGenerator = destination -> { - InstanceConfig config = instanceConfigs.get(destination); - if (config == null) { - throw new CanalServerException("can't find destination:" + destination); - } - - if (config.getMode().isManager()) { - PlainCanalInstanceGenerator instanceGenerator = new PlainCanalInstanceGenerator(properties); - instanceGenerator.setCanalConfigClient(managerClients.get(config.getManagerAddress())); - instanceGenerator.setSpringXml(config.getSpringXml()); - return instanceGenerator.generate(destination); - } else if (config.getMode().isSpring()) { - SpringCanalInstanceGenerator instanceGenerator = new SpringCanalInstanceGenerator(); - instanceGenerator.setSpringXml(config.getSpringXml()); - return instanceGenerator.generate(destination); - } else { - throw new UnsupportedOperationException("unknow mode :" + config.getMode()); - } - - }; - - return globalConfig; - } - - private PlainCanalConfigClient getManagerClient(String managerAddress) { - return new PlainCanalConfigClient(managerAddress, this.adminUser, this.adminPasswd, this.registerIp, adminPort); - } - - private void initInstanceConfig(Properties properties) { - String destinationStr = getProperty(properties, CanalConstants.CANAL_DESTINATIONS); - String[] destinations = StringUtils.split(destinationStr, CanalConstants.CANAL_DESTINATION_SPLIT); - - for (String destination : destinations) { - InstanceConfig config = parseInstanceConfig(properties, destination); - InstanceConfig oldConfig = instanceConfigs.put(destination, config); - - if (oldConfig != null) { - logger.warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config); - } - } - } - - private InstanceConfig parseInstanceConfig(Properties properties, String destination) { - String adminManagerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER); - InstanceConfig config = new InstanceConfig(globalInstanceConfig); - String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(destination)); - if (StringUtils.isNotEmpty(adminManagerAddress)) { - // 如果指定了manager地址,则强制适用manager - config.setMode(InstanceMode.MANAGER); - } else if (StringUtils.isNotEmpty(modeStr)) { - config.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr))); - } - - String lazyStr = getProperty(properties, CanalConstants.getInstancLazyKey(destination)); - if (!StringUtils.isEmpty(lazyStr)) { - config.setLazy(Boolean.valueOf(lazyStr)); - } - - if (config.getMode().isManager()) { - String managerAddress = getProperty(properties, CanalConstants.getInstanceManagerAddressKey(destination)); - if (StringUtils.isNotEmpty(managerAddress)) { - if (StringUtils.equals(managerAddress, "${canal.admin.manager}")) { - managerAddress = adminManagerAddress; - } - config.setManagerAddress(managerAddress); - } - } - - String springXml = getProperty(properties, CanalConstants.getInstancSpringXmlKey(destination)); - if (StringUtils.isNotEmpty(springXml)) { - config.setSpringXml(springXml); - } - - return config; - } - - public static String getProperty(Properties properties, String key, String defaultValue) { - String value = getProperty(properties, key); - if (StringUtils.isEmpty(value)) { - return defaultValue; - } else { - return value; - } - } - - public static String getProperty(Properties properties, String key) { - key = StringUtils.trim(key); - String value = System.getProperty(key); - - if (value == null) { - value = System.getenv(key); - } - - if (value == null) { - value = properties.getProperty(key); - } - - return StringUtils.trim(value); - } - - public void start() throws Throwable { - logger.info("## start the canal server[{}({}):{}]", ip, registerIp, port); - // 创建整个canal的工作节点 - final String path = ZookeeperPathUtils.getCanalClusterNode(registerIp + ":" + port); - initCid(path); - if (zkclientx != null) { - this.zkclientx.subscribeStateChanges(new IZkStateListener() { - - public void handleStateChanged(KeeperState state) throws Exception { - - } - - public void handleNewSession() throws Exception { - initCid(path); - } - - @Override - public void handleSessionEstablishmentError(Throwable error) throws Exception { - logger.error("failed to connect to zookeeper", error); - } - }); - } - // 优先启动embeded服务 - embededCanalServer.start(); - // 尝试启动一下非lazy状态的通道 - for (Map.Entry entry : instanceConfigs.entrySet()) { - final String destination = entry.getKey(); - InstanceConfig config = entry.getValue(); - // 创建destination的工作节点 - if (!embededCanalServer.isStart(destination)) { - // HA机制启动 - ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination); - if (!config.getLazy() && !runningMonitor.isStart()) { - runningMonitor.start(); - } - } - - if (autoScan) { - instanceConfigMonitors.get(config.getMode()).register(destination, defaultAction); - } - } - - if (autoScan) { - instanceConfigMonitors.get(globalInstanceConfig.getMode()).start(); - for (InstanceConfigMonitor monitor : instanceConfigMonitors.values()) { - if (!monitor.isStart()) { - monitor.start(); - } - } - } - - // 启动网络接口 - if (canalServer != null) { - canalServer.start(); - } - } - - public void stop() throws Throwable { - - if (canalServer != null) { - canalServer.stop(); - } - - if (autoScan) { - for (InstanceConfigMonitor monitor : instanceConfigMonitors.values()) { - if (monitor.isStart()) { - monitor.stop(); - } - } - } - - for (ServerRunningMonitor runningMonitor : ServerRunningMonitors.getRunningMonitors().values()) { - if (runningMonitor.isStart()) { - runningMonitor.stop(); - } - } - - // 释放canal的工作节点 - releaseCid(ZookeeperPathUtils.getCanalClusterNode(registerIp + ":" + port)); - logger.info("## stop the canal server[{}({}):{}]", ip, registerIp, port); - - if (zkclientx != null) { - zkclientx.close(); - } - - // 关闭时清理缓存 - if (instanceConfigs != null) { - instanceConfigs.clear(); - } - if (managerClients != null) { - managerClients.clear(); - } - if (instanceConfigMonitors != null) { - instanceConfigMonitors.clear(); - } - - ZkClientx.clearClients(); - } - - private void initCid(String path) { - // logger.info("## init the canalId = {}", cid); - // 初始化系统目录 - if (zkclientx != null) { - try { - zkclientx.createEphemeral(path); - } catch (ZkNoNodeException e) { - // 如果父目录不存在，则创建 - String parentDir = path.substring(0, path.lastIndexOf('/')); - zkclientx.createPersistent(parentDir, true); - zkclientx.createEphemeral(path); - } catch (ZkNodeExistsException e) { - // ignore - // 因为第一次启动时创建了cid,但在stop/start的时可能会关闭和新建,允许出现NodeExists问题s - } - - } - } - - private void releaseCid(String path) { - // logger.info("## release the canalId = {}", cid); - // 初始化系统目录 - if (zkclientx != null) { - zkclientx.delete(path); - } - } - - public CanalMQStarter getCanalMQStarter() { - return canalMQStarter; - } - - public void setCanalMQStarter(CanalMQStarter canalMQStarter) { - this.canalMQStarter = canalMQStarter; - } - - public Map getInstanceConfigMonitors() { - return instanceConfigMonitors; - } - - public Map getInstanceConfigs() { - return instanceConfigs; - } - -} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalLauncher.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalLauncher.java deleted file mode 100644 index e84b6f0..0000000 --- a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalLauncher.java +++ /dev/null @@ -1,141 +0,0 @@ -package cc.smtweb.system.canal.server; - -import com.alibaba.otter.canal.common.utils.AddressUtils; -import com.alibaba.otter.canal.common.utils.NamedThreadFactory; -import com.alibaba.otter.canal.instance.manager.plain.PlainCanal; -import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient; -import org.apache.commons.lang.BooleanUtils; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileInputStream; -import java.util.Properties; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -/** - * canal独立版本启动的入口类 - * - * @author jianghang 2012-11-6 下午05:20:49 - * @version 1.0.0 - */ -public class CanalLauncher { - - private static final String CLASSPATH_URL_PREFIX = "classpath:"; - private static final Logger logger = LoggerFactory.getLogger(CanalLauncher.class); - public static final CountDownLatch runningLatch = new CountDownLatch(1); - private static ScheduledExecutorService executor = Executors.newScheduledThreadPool(1, - new NamedThreadFactory("canal-server-scan")); - - public static void main(String[] args) { - startServer(); - } - - public static void startServer() { - try { - logger.info("## canal sever start begin"); - logger.info("## set default uncaught exception handler"); - setGlobalUncaughtExceptionHandler(); - - // 支持rocketmq client 配置日志路径 - System.setProperty("rocketmq.client.logUseSlf4j", "true"); - - logger.info("## load canal configurations"); - String conf = System.getProperty("canal.conf", "classpath:canal.properties"); - Properties properties = new Properties(); - if (conf.startsWith(CLASSPATH_URL_PREFIX)) { - conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX); - properties.load(CanalLauncher.class.getClassLoader().getResourceAsStream(conf)); - } else { - properties.load(new FileInputStream(conf)); - } - - final CanalStarter canalStater = new CanalStarter(properties); - String managerAddress = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER); - if (StringUtils.isNotEmpty(managerAddress)) { - String user = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_USER); - String passwd = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PASSWD); - String adminPort = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PORT, "11110"); - boolean autoRegister = BooleanUtils.toBoolean(CanalController.getProperty(properties, - CanalConstants.CANAL_ADMIN_AUTO_REGISTER)); - String autoCluster = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_AUTO_CLUSTER); - String name = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_REGISTER_NAME); - if (StringUtils.isEmpty(name)) { - name = AddressUtils.getHostName(); - } - - String registerIp = CanalController.getProperty(properties, CanalConstants.CANAL_REGISTER_IP); - if (StringUtils.isEmpty(registerIp)) { - registerIp = AddressUtils.getHostIp(); - } - final PlainCanalConfigClient configClient = new PlainCanalConfigClient(managerAddress, - user, - passwd, - registerIp, - Integer.parseInt(adminPort), - autoRegister, - autoCluster, - name); - PlainCanal canalConfig = configClient.findServer(null); - if (canalConfig == null) { - throw new IllegalArgumentException("managerAddress:" + managerAddress - + " can't not found config for [" + registerIp + ":" + adminPort - + "]"); - } - Properties managerProperties = canalConfig.getProperties(); - // merge local - managerProperties.putAll(properties); - int scanIntervalInSecond = Integer.valueOf(CanalController.getProperty(managerProperties, - CanalConstants.CANAL_AUTO_SCAN_INTERVAL, - "5")); - executor.scheduleWithFixedDelay(new Runnable() { - - private PlainCanal lastCanalConfig; - - public void run() { - try { - if (lastCanalConfig == null) { - lastCanalConfig = configClient.findServer(null); - } else { - PlainCanal newCanalConfig = configClient.findServer(lastCanalConfig.getMd5()); - if (newCanalConfig != null) { - // 远程配置canal.properties修改重新加载整个应用 - canalStater.stop(); - Properties managerProperties = newCanalConfig.getProperties(); - // merge local - managerProperties.putAll(properties); - canalStater.setProperties(managerProperties); - canalStater.start(); - - lastCanalConfig = newCanalConfig; - } - } - - } catch (Throwable e) { - logger.error("scan failed", e); - } - } - - }, 0, scanIntervalInSecond, TimeUnit.SECONDS); - canalStater.setProperties(managerProperties); - } else { - canalStater.setProperties(properties); - } - - canalStater.start(); - logger.info("## canal server start success"); - runningLatch.await(); - executor.shutdownNow(); - } catch (Throwable e) { - logger.error("## Something goes wrong when starting up the canal Server:", e); - } - } - - private static void setGlobalUncaughtExceptionHandler() { - Thread.setDefaultUncaughtExceptionHandler((t, e) -> logger.error("UnCaughtException", e)); - } - -} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStartedListener.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStartedListener.java deleted file mode 100644 index 4530945..0000000 --- a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStartedListener.java +++ /dev/null @@ -1,39 +0,0 @@ -package cc.smtweb.system.canal.server; - -import cc.smtweb.framework.core.annotation.SwStartListener; -import cc.smtweb.framework.core.common.SwConsts; -import cc.smtweb.framework.core.mvc.controller.IStartListener; -import cc.smtweb.framework.core.systask.SysThreadPool; -import cc.smtweb.framework.core.systask.SysThreadWorker; -import lombok.extern.slf4j.Slf4j; - -/** - * @Author yaoq - * @Date 2022年09月06日 10:31 - * @Description - */ -@Slf4j -@SwStartListener -public class CanalStartedListener implements IStartListener { - - @Override - public int order() { - return SwConsts.DEFAULT_ORDER + 2; - } - - - @Override - public void init() { - - } - - @Override - public void run() { - SysThreadPool.getInstance().addTask(new SysThreadWorker("canal server") { - @Override - public void localWork() throws Exception { - CanalLauncher.startServer(); - } - }); - } -} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStarter.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStarter.java deleted file mode 100644 index 58b6621..0000000 --- a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStarter.java +++ /dev/null @@ -1,169 +0,0 @@ -package cc.smtweb.system.canal.server; - -import java.util.Properties; - -import com.alibaba.otter.canal.connector.core.config.MQProperties; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.alibaba.otter.canal.admin.netty.CanalAdminWithNetty; -import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer; -import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader; -import cc.smtweb.system.canal.server.admin.CanalAdminController; -import com.alibaba.otter.canal.server.CanalMQStarter; - -/** - * Canal server 启动类 - * - * @author rewerma 2020-01-27 - * @version 1.0.2 - */ -public class CanalStarter { - - private static final Logger logger = LoggerFactory.getLogger(CanalStarter.class); - - private static final String CONNECTOR_SPI_DIR = "/plugin"; - private static final String CONNECTOR_STANDBY_SPI_DIR = "/canal/plugin"; - - private CanalController controller = null; - private CanalMQProducer canalMQProducer = null; - private Thread shutdownThread = null; - private CanalMQStarter canalMQStarter = null; - private volatile Properties properties; - private volatile boolean running = false; - - private CanalAdminWithNetty canalAdmin; - - public CanalStarter(Properties properties){ - this.properties = properties; - } - - public boolean isRunning() { - return running; - } - - public Properties getProperties() { - return properties; - } - - public void setProperties(Properties properties) { - this.properties = properties; - } - - public CanalController getController() { - return controller; - } - - /** - * 启动方法 - * - * @throws Throwable - */ - public synchronized void start() throws Throwable { - String serverMode = CanalController.getProperty(properties, CanalConstants.CANAL_SERVER_MODE); - if (!"tcp".equalsIgnoreCase(serverMode)) { - ExtensionLoader loader = ExtensionLoader.getExtensionLoader(CanalMQProducer.class); - canalMQProducer = loader - .getExtension(serverMode.toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR); - if (canalMQProducer != null) { - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - Thread.currentThread().setContextClassLoader(canalMQProducer.getClass().getClassLoader()); - canalMQProducer.init(properties); - Thread.currentThread().setContextClassLoader(cl); - } - } - - if (canalMQProducer != null) { - MQProperties mqProperties = canalMQProducer.getMqProperties(); - // disable netty - System.setProperty(CanalConstants.CANAL_WITHOUT_NETTY, "true"); - if (mqProperties.isFlatMessage()) { - // 设置为raw避免ByteString->Entry的二次解析 - System.setProperty("canal.instance.memory.rawEntry", "false"); - } - } - - logger.info("## start the canal server."); - controller = new CanalController(properties); - controller.start(); - logger.info("## the canal server is running now ......"); - shutdownThread = new Thread(() -> { - try { - logger.info("## stop the canal server"); - controller.stop(); - CanalLauncher.runningLatch.countDown(); - } catch (Throwable e) { - logger.warn("##something goes wrong when stopping canal Server:", e); - } finally { - logger.info("## canal server is down."); - } - }); - Runtime.getRuntime().addShutdownHook(shutdownThread); - - if (canalMQProducer != null) { - canalMQStarter = new CanalMQStarter(canalMQProducer); - String destinations = CanalController.getProperty(properties, CanalConstants.CANAL_DESTINATIONS); - canalMQStarter.start(destinations); - controller.setCanalMQStarter(canalMQStarter); - } - - // start canalAdmin - String port = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PORT); - if (canalAdmin == null && StringUtils.isNotEmpty(port)) { - String user = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_USER); - String passwd = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PASSWD); - CanalAdminController canalAdmin = new CanalAdminController(this); - canalAdmin.setUser(user); - canalAdmin.setPasswd(passwd); - - String ip = CanalController.getProperty(properties, CanalConstants.CANAL_IP); - - logger.debug("canal admin port:{}, canal admin user:{}, canal admin password: {}, canal ip:{}", - port, - user, - passwd, - ip); - - CanalAdminWithNetty canalAdminWithNetty = CanalAdminWithNetty.instance(); - canalAdminWithNetty.setCanalAdmin(canalAdmin); - canalAdminWithNetty.setPort(Integer.parseInt(port)); - canalAdminWithNetty.setIp(ip); - canalAdminWithNetty.start(); - this.canalAdmin = canalAdminWithNetty; - } - - running = true; - } - - public synchronized void stop() throws Throwable { - stop(false); - } - - /** - * 销毁方法，远程配置变更时调用 - * - * @throws Throwable - */ - public synchronized void stop(boolean stopByAdmin) throws Throwable { - if (!stopByAdmin && canalAdmin != null) { - canalAdmin.stop(); - canalAdmin = null; - } - - if (controller != null) { - controller.stop(); - controller = null; - } - if (shutdownThread != null) { - Runtime.getRuntime().removeShutdownHook(shutdownThread); - shutdownThread = null; - } - if (canalMQProducer != null && canalMQStarter != null) { - canalMQStarter.destroy(); - canalMQStarter = null; - canalMQProducer = null; - } - running = false; - } -} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/InstanceConfig.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/InstanceConfig.java deleted file mode 100644 index 6775ca9..0000000 --- a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/InstanceConfig.java +++ /dev/null @@ -1,93 +0,0 @@ -package cc.smtweb.system.canal.server; - -import org.apache.commons.lang.builder.ToStringBuilder; - -import com.alibaba.otter.canal.common.utils.CanalToStringStyle; - -/** - * 启动的相关配置 - * - * @author jianghang 2012-11-8 下午02:50:54 - * @version 1.0.0 - */ -public class InstanceConfig { - - private InstanceConfig globalConfig; - private InstanceMode mode; - private Boolean lazy; - private String managerAddress; - private String springXml; - - public InstanceConfig(){ - - } - - public InstanceConfig(InstanceConfig globalConfig){ - this.globalConfig = globalConfig; - } - - public static enum InstanceMode { - SPRING, MANAGER; - - public boolean isSpring() { - return this == InstanceMode.SPRING; - } - - public boolean isManager() { - return this == InstanceMode.MANAGER; - } - } - - public Boolean getLazy() { - if (lazy == null && globalConfig != null) { - return globalConfig.getLazy(); - } else { - return lazy; - } - } - - public void setLazy(Boolean lazy) { - this.lazy = lazy; - } - - public InstanceMode getMode() { - if (mode == null && globalConfig != null) { - return globalConfig.getMode(); - } else { - return mode; - } - } - - public void setMode(InstanceMode mode) { - this.mode = mode; - } - - public String getManagerAddress() { - if (managerAddress == null && globalConfig != null) { - return globalConfig.getManagerAddress(); - } else { - return managerAddress; - } - } - - public void setManagerAddress(String managerAddress) { - this.managerAddress = managerAddress; - } - - public String getSpringXml() { - if (springXml == null && globalConfig != null) { - return globalConfig.getSpringXml(); - } else { - return springXml; - } - } - - public void setSpringXml(String springXml) { - this.springXml = springXml; - } - - public String toString() { - return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); - } - -} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/admin/CanalAdminController.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/admin/CanalAdminController.java deleted file mode 100644 index d504621..0000000 --- a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/admin/CanalAdminController.java +++ /dev/null @@ -1,256 +0,0 @@ -package cc.smtweb.system.canal.server.admin; - -import java.io.File; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import cc.smtweb.system.canal.server.CanalStarter; -import cc.smtweb.system.canal.server.monitor.InstanceConfigMonitor; -import cc.smtweb.system.canal.server.monitor.ManagerInstanceConfigMonitor; -import cc.smtweb.system.canal.server.monitor.SpringInstanceConfigMonitor; -import cc.smtweb.system.canal.server.monitor.InstanceAction; -import org.apache.commons.io.filefilter.TrueFileFilter; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.alibaba.otter.canal.admin.CanalAdmin; -import com.alibaba.otter.canal.common.utils.FileUtils; -import cc.smtweb.system.canal.server.InstanceConfig; -import com.alibaba.otter.canal.instance.core.CanalInstance; -import com.alibaba.otter.canal.protocol.SecurityUtil; -import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded; -import com.google.common.base.Joiner; - -/** - * 提供canal admin的管理操作 - * - * @author agapple 2019年8月24日 下午11:39:01 - * @since 1.1.4 - */ -public class CanalAdminController implements CanalAdmin { - - private static final Logger logger = LoggerFactory.getLogger(CanalAdminController.class); - private String user; - private String passwd; - private CanalStarter canalStater; - - public CanalAdminController(CanalStarter canalStater){ - this.canalStater = canalStater; - } - - @Override - public boolean check() { - return canalStater.isRunning(); - } - - @Override - public synchronized boolean start() { - try { - if (!canalStater.isRunning()) { - canalStater.start(); - return true; - } - } catch (Throwable e) { - logger.error(e.getMessage(), e); - } - return false; - } - - @Override - public synchronized boolean stop() { - try { - if (canalStater.isRunning()) { - canalStater.stop(true); - return true; - } - } catch (Throwable e) { - logger.error(e.getMessage(), e); - } - return false; - } - - @Override - public synchronized boolean restart() { - stop(); - return start(); - } - - @Override - public boolean auth(String user, String passwd, byte[] seed) { - // 如果user/passwd密码为空,则任何用户账户都能登录 - if ((StringUtils.isEmpty(this.user) || StringUtils.equals(this.user, user))) { - if (StringUtils.isEmpty(this.passwd)) { - return true; - } else if (StringUtils.isEmpty(passwd)) { - // 如果server密码有配置,客户端密码为空,则拒绝 - return false; - } - - try { - byte[] passForClient = SecurityUtil.hexStr2Bytes(passwd); - return SecurityUtil.scrambleServerAuth(passForClient, SecurityUtil.hexStr2Bytes(this.passwd), seed); - } catch (NoSuchAlgorithmException e) { - return false; - } - } - - return false; - } - - @Override - public String getRunningInstances() { - try { - Map instances = CanalServerWithEmbedded.instance().getCanalInstances(); - List runningInstances = new ArrayList<>(); - instances.forEach((destination, instance) -> { - if (instance.isStart()) { - runningInstances.add(destination); - } - }); - - return Joiner.on(",").join(runningInstances); - } catch (Throwable e) { - logger.error(e.getMessage(), e); - } - return ""; - } - - @Override - public boolean checkInstance(String destination) { - Map instances = CanalServerWithEmbedded.instance().getCanalInstances(); - if (instances == null || !instances.containsKey(destination)) { - return false; - } else { - CanalInstance instance = instances.get(destination); - return instance.isStart(); - } - } - - @Override - public boolean startInstance(String destination) { - try { - InstanceAction instanceAction = getInstanceAction(destination); - if (instanceAction != null) { - instanceAction.start(destination); - return true; - } - } catch (Throwable e) { - logger.error(e.getMessage(), e); - } - return false; - } - - @Override - public boolean stopInstance(String destination) { - try { - InstanceAction instanceAction = getInstanceAction(destination); - if (instanceAction != null) { - instanceAction.stop(destination); - return true; - } - } catch (Throwable e) { - logger.error(e.getMessage(), e); - } - return false; - } - - @Override - public boolean releaseInstance(String destination) { - try { - InstanceAction instanceAction = getInstanceAction(destination); - if (instanceAction != null) { - instanceAction.release(destination); - return true; - } - } catch (Throwable e) { - logger.error(e.getMessage(), e); - } - return false; - } - - @Override - public boolean restartInstance(String destination) { - try { - InstanceAction instanceAction = getInstanceAction(destination); - if (instanceAction != null) { - instanceAction.reload(destination); - return true; - } - } catch (Throwable e) { - logger.error(e.getMessage(), e); - } - return false; - } - - @Override - public String listCanalLog() { - Collection files = org.apache.commons.io.FileUtils.listFiles(new File("../logs/canal/"), - TrueFileFilter.TRUE, - TrueFileFilter.TRUE); - List names = files.stream().map(File::getName).collect(Collectors.toList()); - return Joiner.on(",").join(names); - } - - @Override - public String canalLog(int lines) { - return FileUtils.readFileFromOffset("../logs/canal/canal.log", lines, "UTF-8"); - } - - @Override - public String listInstanceLog(String destination) { - Collection files = org.apache.commons.io.FileUtils.listFiles(new File("../logs/" + destination + "/"), - TrueFileFilter.TRUE, - TrueFileFilter.TRUE); - List names = files.stream().map(File::getName).collect(Collectors.toList()); - return Joiner.on(",").join(names); - } - - @Override - public String instanceLog(String destination, String fileName, int lines) { - if (StringUtils.isEmpty(fileName)) { - fileName = destination + ".log"; - } - return FileUtils.readFileFromOffset("../logs/" + destination + "/" + fileName, lines, "UTF-8"); - } - - private InstanceAction getInstanceAction(String destination) { - Map monitors = canalStater.getController() - .getInstanceConfigMonitors(); - - InstanceAction instanceAction = null; - if (monitors.containsKey(InstanceConfig.InstanceMode.SPRING)) { - SpringInstanceConfigMonitor monitor = (SpringInstanceConfigMonitor) monitors.get(InstanceConfig.InstanceMode.SPRING); - Map instanceActions = monitor.getActions(); - instanceAction = instanceActions.get(destination); - } - - if (instanceAction != null) { - return instanceAction; - } - - if (monitors.containsKey(InstanceConfig.InstanceMode.MANAGER)) { - ManagerInstanceConfigMonitor monitor = (ManagerInstanceConfigMonitor) monitors.get(InstanceConfig.InstanceMode.MANAGER); - Map instanceActions = monitor.getActions(); - instanceAction = instanceActions.get(destination); - } - return instanceAction; - } - - public void setUser(String user) { - this.user = user; - } - - public void setPasswd(String passwd) { - this.passwd = passwd; - } - - public void setCanalStater(CanalStarter canalStater) { - this.canalStater = canalStater; - } - -} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceAction.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceAction.java deleted file mode 100644 index f1ed700..0000000 --- a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceAction.java +++ /dev/null @@ -1,30 +0,0 @@ -package cc.smtweb.system.canal.server.monitor; - -/** - * config配置变化 - * - * @author jianghang 2013-2-18 下午01:19:29 - * @version 1.0.1 - */ -public interface InstanceAction { - - /** - * 启动destination - */ - void start(String destination); - - /** - * 主动释放destination运行 - */ - void release(String destination); - - /** - * 停止destination - */ - void stop(String destination); - - /** - * 重载destination，可能需要stop,start操作，或者只是更新下内存配置 - */ - void reload(String destination); -} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceConfigMonitor.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceConfigMonitor.java deleted file mode 100644 index 702a2b9..0000000 --- a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceConfigMonitor.java +++ /dev/null @@ -1,16 +0,0 @@ -package cc.smtweb.system.canal.server.monitor; - -import com.alibaba.otter.canal.common.CanalLifeCycle; - -/** - * 监听instance file的文件变化，触发instance start/stop等操作 - * - * @author jianghang 2013-2-6 下午06:19:56 - * @version 1.0.1 - */ -public interface InstanceConfigMonitor extends CanalLifeCycle { - - void register(String destination, InstanceAction action); - - void unregister(String destination); -} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/ManagerInstanceConfigMonitor.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/ManagerInstanceConfigMonitor.java deleted file mode 100644 index d0aa043..0000000 --- a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/ManagerInstanceConfigMonitor.java +++ /dev/null @@ -1,184 +0,0 @@ -package cc.smtweb.system.canal.server.monitor; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; -import com.alibaba.otter.canal.common.CanalLifeCycle; -import com.alibaba.otter.canal.common.utils.NamedThreadFactory; -import com.alibaba.otter.canal.instance.manager.plain.PlainCanal; -import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient; -import com.google.common.collect.Lists; -import com.google.common.collect.MapMaker; -import com.google.common.collect.MigrateMap; - -/** - * 基于manager配置的实现 - * - * @author agapple 2019年8月26日 下午10:00:20 - * @since 1.1.4 - */ -public class ManagerInstanceConfigMonitor extends AbstractCanalLifeCycle implements InstanceConfigMonitor, CanalLifeCycle { - - private static final Logger logger = LoggerFactory.getLogger(ManagerInstanceConfigMonitor.class); - private long scanIntervalInSecond = 5; - private InstanceAction defaultAction = null; - private Map actions = new MapMaker().makeMap(); - private Map configs = MigrateMap.makeComputingMap(destination -> new PlainCanal()); - private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1, - new NamedThreadFactory("canal-instance-scan")); - - private volatile boolean isFirst = true; - private PlainCanalConfigClient configClient; - - public void start() { - super.start(); - executor.scheduleWithFixedDelay(() -> { - try { - scan(); - if (isFirst) { - isFirst = false; - } - } catch (Throwable e) { - logger.error("scan failed", e); - } - }, 0, scanIntervalInSecond, TimeUnit.SECONDS); - } - - public void stop() { - super.stop(); - executor.shutdownNow(); - actions.clear(); - } - - public void register(String destination, InstanceAction action) { - if (action != null) { - actions.put(destination, action); - } else { - actions.put(destination, defaultAction); - } - } - - public void unregister(String destination) { - actions.remove(destination); - } - - private void scan() { - String instances = configClient.findInstances(null); - if (instances == null) { - return; - } - - final List is = Lists.newArrayList(StringUtils.split(instances, ',')); - List start = new ArrayList<>(); - List stop = new ArrayList<>(); - List restart = new ArrayList<>(); - for (String instance : is) { - if (!configs.containsKey(instance)) { - PlainCanal newPlainCanal = configClient.findInstance(instance, null); - if (newPlainCanal != null) { - configs.put(instance, newPlainCanal); - start.add(instance); - } - } else { - PlainCanal plainCanal = configs.get(instance); - PlainCanal newPlainCanal = configClient.findInstance(instance, plainCanal.getMd5()); - if (newPlainCanal != null) { - // 配置有变化 - restart.add(instance); - configs.put(instance, newPlainCanal); - } - } - } - - configs.forEach((instance, plainCanal) -> { - if (!is.contains(instance)) { - stop.add(instance); - } - }); - - stop.forEach(instance -> { - notifyStop(instance); - }); - - restart.forEach(instance -> { - notifyReload(instance); - }); - - start.forEach(instance -> { - notifyStart(instance); - }); - - } - - private void notifyStart(String destination) { - try { - defaultAction.start(destination); - actions.put(destination, defaultAction); - // 启动成功后记录配置文件信息 - } catch (Throwable e) { - logger.error(String.format("scan add found[%s] but start failed", destination), e); - } - } - - private void notifyStop(String destination) { - InstanceAction action = actions.remove(destination); - if (action != null) { - try { - action.stop(destination); - configs.remove(destination); - } catch (Throwable e) { - logger.error(String.format("scan delete found[%s] but stop failed", destination), e); - actions.put(destination, action);// 再重新加回去，下一次scan时再执行删除 - } - } - } - - private void notifyReload(String destination) { - InstanceAction action = actions.get(destination); - if (action != null) { - try { - action.reload(destination); - } catch (Throwable e) { - logger.error(String.format("scan reload found[%s] but reload failed", destination), e); - } - } - } - - public void release(String destination) { - InstanceAction action = actions.remove(destination); - if (action != null) { - try { - configs.remove(destination); - } catch (Throwable e) { - logger.error(String.format("scan delete found[%s] but stop failed", destination), e); - actions.put(destination, action);// 再重新加回去，下一次scan时再执行删除 - } - } - } - - public void setDefaultAction(InstanceAction defaultAction) { - this.defaultAction = defaultAction; - } - - public void setScanIntervalInSecond(long scanIntervalInSecond) { - this.scanIntervalInSecond = scanIntervalInSecond; - } - - public void setConfigClient(PlainCanalConfigClient configClient) { - this.configClient = configClient; - } - - public Map getActions() { - return actions; - } - -} diff --git a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/SpringInstanceConfigMonitor.java b/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/SpringInstanceConfigMonitor.java deleted file mode 100644 index 83a2b13..0000000 --- a/smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/SpringInstanceConfigMonitor.java +++ /dev/null @@ -1,297 +0,0 @@ -package cc.smtweb.system.canal.server.monitor; - -import java.io.File; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.util.Assert; -import org.springframework.util.CollectionUtils; - -import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; -import com.alibaba.otter.canal.common.CanalLifeCycle; -import com.alibaba.otter.canal.common.utils.NamedThreadFactory; -import com.google.common.collect.MapMaker; -import com.google.common.collect.MigrateMap; - -/** - * 监听基于spring配置的instance变化 - * - * @author jianghang 2013-2-6 下午06:23:55 - * @version 1.0.1 - */ -public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implements InstanceConfigMonitor, CanalLifeCycle { - - private static final Logger logger = LoggerFactory.getLogger(SpringInstanceConfigMonitor.class); - private String rootConf; - // 扫描周期，单位秒 - private long scanIntervalInSecond = 5; - private InstanceAction defaultAction = null; - private Map actions = new MapMaker().makeMap(); - private Map lastFiles = MigrateMap.makeComputingMap(InstanceConfigFiles::new); - private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1, - new NamedThreadFactory("canal-instance-scan")); - - private volatile boolean isFirst = true; - - public Map getActions() { - return actions; - } - - public void start() { - super.start(); - Assert.notNull(rootConf, "root conf dir is null!"); - - executor.scheduleWithFixedDelay(() -> { - try { - scan(); - if (isFirst) { - isFirst = false; - } - } catch (Throwable e) { - logger.error("scan failed", e); - } - }, 0, scanIntervalInSecond, TimeUnit.SECONDS); - } - - public void stop() { - super.stop(); - executor.shutdownNow(); - actions.clear(); - lastFiles.clear(); - } - - public void register(String destination, InstanceAction action) { - if (action != null) { - actions.put(destination, action); - } else { - actions.put(destination, defaultAction); - } - } - - public void unregister(String destination) { - actions.remove(destination); - } - - public void setRootConf(String rootConf) { - this.rootConf = rootConf; - } - - private void scan() { - File rootdir = new File(rootConf); - if (!rootdir.exists()) { - return; - } - - File[] instanceDirs = rootdir.listFiles(pathname -> { - String filename = pathname.getName(); - return pathname.isDirectory() && !"spring".equalsIgnoreCase(filename); - }); - - // 扫描目录的新增 - Set currentInstanceNames = new HashSet<>(); - - // 判断目录内文件的变化 - for (File instanceDir : instanceDirs) { - String destination = instanceDir.getName(); - currentInstanceNames.add(destination); - File[] instanceConfigs = instanceDir.listFiles((dir, name) -> { - // return !StringUtils.endsWithIgnoreCase(name, ".dat"); - // 限制一下，只针对instance.properties文件,避免因为.svn或者其他生成的临时文件导致出现reload - return StringUtils.equalsIgnoreCase(name, "instance.properties"); - }); - - if (!actions.containsKey(destination) && instanceConfigs.length > 0) { - // 存在合法的instance.properties，并且第一次添加时，进行启动操作 - notifyStart(instanceDir, destination, instanceConfigs); - } else if (actions.containsKey(destination)) { - // 历史已经启动过 - if (instanceConfigs.length == 0) { // 如果不存在合法的instance.properties - notifyStop(destination); - } else { - InstanceConfigFiles lastFile = lastFiles.get(destination); - // 历史启动过 所以配置文件信息必然存在 - if (!isFirst && CollectionUtils.isEmpty(lastFile.getInstanceFiles())) { - logger.error("[{}] is started, but not found instance file info.", destination); - } - - boolean hasChanged = judgeFileChanged(instanceConfigs, lastFile.getInstanceFiles()); - // 通知变化 - if (hasChanged) { - notifyReload(destination); - } - - if (hasChanged || CollectionUtils.isEmpty(lastFile.getInstanceFiles())) { - // 更新内容 - List newFileInfo = new ArrayList<>(); - for (File instanceConfig : instanceConfigs) { - newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified())); - } - - lastFile.setInstanceFiles(newFileInfo); - } - } - } - - } - - // 判断目录是否删除 - Set deleteInstanceNames = new HashSet<>(); - for (String destination : actions.keySet()) { - if (!currentInstanceNames.contains(destination)) { - deleteInstanceNames.add(destination); - } - } - for (String deleteInstanceName : deleteInstanceNames) { - notifyStop(deleteInstanceName); - } - } - - private void notifyStart(File instanceDir, String destination, File[] instanceConfigs) { - try { - defaultAction.start(destination); - actions.put(destination, defaultAction); - - // 启动成功后记录配置文件信息 - InstanceConfigFiles lastFile = lastFiles.get(destination); - List newFileInfo = new ArrayList<>(); - for (File instanceConfig : instanceConfigs) { - newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified())); - } - lastFile.setInstanceFiles(newFileInfo); - } catch (Throwable e) { - logger.error(String.format("scan add found[%s] but start failed", destination), e); - } - } - - private void notifyStop(String destination) { - InstanceAction action = actions.remove(destination); - if (action != null) { - try { - action.stop(destination); - lastFiles.remove(destination); - } catch (Throwable e) { - logger.error(String.format("scan delete found[%s] but stop failed", destination), e); - actions.put(destination, action);// 再重新加回去，下一次scan时再执行删除 - } - } - } - - private void notifyReload(String destination) { - InstanceAction action = actions.get(destination); - if (action != null) { - try { - action.reload(destination); - } catch (Throwable e) { - logger.error(String.format("scan reload found[%s] but reload failed", destination), e); - } - } - } - - private boolean judgeFileChanged(File[] instanceConfigs, List fileInfos) { - boolean hasChanged = false; - for (File instanceConfig : instanceConfigs) { - for (FileInfo fileInfo : fileInfos) { - if (instanceConfig.getName().equals(fileInfo.getName())) { - hasChanged |= (instanceConfig.lastModified() != fileInfo.getLastModified()); - if (hasChanged) { - return hasChanged; - } - } - } - } - - return hasChanged; - } - - public void setDefaultAction(InstanceAction defaultAction) { - this.defaultAction = defaultAction; - } - - public void setScanIntervalInSecond(long scanIntervalInSecond) { - this.scanIntervalInSecond = scanIntervalInSecond; - } - - public static class InstanceConfigFiles { - - private String destination; // instance - // name - private List springFile = new ArrayList<>(); // spring的instance - // xml - private FileInfo rootFile; // canal.properties - private List instanceFiles = new ArrayList<>(); // instance对应的配置 - - public InstanceConfigFiles(String destination){ - this.destination = destination; - } - - public String getDestination() { - return destination; - } - - public void setDestination(String destination) { - this.destination = destination; - } - - public List getSpringFile() { - return springFile; - } - - public void setSpringFile(List springFile) { - this.springFile = springFile; - } - - public FileInfo getRootFile() { - return rootFile; - } - - public void setRootFile(FileInfo rootFile) { - this.rootFile = rootFile; - } - - public List getInstanceFiles() { - return instanceFiles; - } - - public void setInstanceFiles(List instanceFiles) { - this.instanceFiles = instanceFiles; - } - - } - - public static class FileInfo { - - private String name; - private long lastModified = 0; - - public FileInfo(String name, long lastModified){ - this.name = name; - this.lastModified = lastModified; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public long getLastModified() { - return lastModified; - } - - public void setLastModified(long lastModified) { - this.lastModified = lastModified; - } - - } -} diff --git a/smtweb-framework/canal/server/src/main/resources/META-INF/spring.factories b/smtweb-framework/canal/server/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000..a599c24 --- /dev/null +++ b/smtweb-framework/canal/server/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + cc.smtweb.system.canal.deployer.CanalConfiguration diff --git a/smtweb-framework/canal/server/src/main/resources/canal.properties b/smtweb-framework/canal/server/src/main/resources/canal.properties index 82992bc..5994072 100644 --- a/smtweb-framework/canal/server/src/main/resources/canal.properties +++ b/smtweb-framework/canal/server/src/main/resources/canal.properties @@ -1,17 +1,19 @@ ################################################# ######### common argument ############# ################################################# -# tcp bind ip +# 服务端IP 可以为空 canal.ip = -# register ip to zookeeper -canal.register.ip = +# 服务端口 canal.port = 11111 canal.metrics.pull.port = 11112 -# canal instance user/passwd +# 服务用户名，密码，可以为空 # canal.user = canal # canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458 -# canal admin config + +#######后面的配置不用管####### +canal.register.ip = +#canal admin config #canal.admin.manager = 127.0.0.1:8089 canal.admin.port = 11110 canal.admin.user = admin @@ -161,7 +163,7 @@ rocketmq.namespace = rocketmq.namesrv.addr = 127.0.0.1:9876 rocketmq.retry.times.when.send.failed = 0 rocketmq.vip.channel.enabled = false -rocketmq.tag = +rocketmq.tag = ################################################## ######### RabbitMQ ############# @@ -179,4 +181,4 @@ rabbitmq.deliveryMode = ################################################## pulsarmq.serverUrl = pulsarmq.roleToken = -pulsarmq.topicTenantPrefix = \ No newline at end of file +pulsarmq.topicTenantPrefix = diff --git a/smtweb-framework/canal/server/src/main/resources/example/instance.properties b/smtweb-framework/canal/server/src/main/resources/example/instance.properties index 7a1f752..1930760 100644 --- a/smtweb-framework/canal/server/src/main/resources/example/instance.properties +++ b/smtweb-framework/canal/server/src/main/resources/example/instance.properties @@ -1,11 +1,11 @@ ################################################# ## mysql serverId , v1.0.26+ will autoGen +# 从库ID 必须和主库不一致，多重库也不一致 canal.instance.mysql.slaveId=2 - # enable gtid use true/false canal.instance.gtidon=false - -# position info +# 主库信息 +# 主库地址及端口 canal.instance.master.address=172.28.123.25:3306 #canal.instance.master.journal.name= #canal.instance.master.position= @@ -22,7 +22,6 @@ canal.instance.tsdb.enable=true #canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb #canal.instance.tsdb.dbUsername=canal #canal.instance.tsdb.dbPassword=canal - #canal.instance.standby.address = #canal.instance.standby.journal.name = #canal.instance.standby.position = diff --git a/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysServiceFactory.java b/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysServiceFactory.java index c90f6f8..5779b0b 100644 --- a/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysServiceFactory.java +++ b/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysServiceFactory.java @@ -29,7 +29,7 @@ public class SysServiceFactory { private List listService = new ArrayList<>(); public SysServiceFactory() { - + } //注册任务 @@ -53,6 +53,6 @@ public class SysServiceFactory { //停止任务 public void stop() { - schedule.shutdown(); + if (schedule != null) schedule.shutdown(); } } diff --git a/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadPool.java b/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadPool.java index 0e8df2c..80291c0 100644 --- a/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadPool.java +++ b/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadPool.java @@ -38,7 +38,7 @@ public class SysThreadPool { SecurityManager s = System.getSecurityManager(); group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup(); - namePrefix = "DfpPool-" + + namePrefix = "SysPool-" + poolNumber.getAndIncrement() + "-Thread-"; } @@ -63,14 +63,14 @@ public class SysThreadPool { } try { threadPool.execute(worker); - log.debug("add task to DfpPool success -> [PoolSize：" + threadPool.getPoolSize() + "，ActiveCount：" + threadPool.getActiveCount() + "，CompletedTaskCount：" + threadPool.getCompletedTaskCount() + "]"); + log.debug("add task to SysPool success -> [PoolSize：" + threadPool.getPoolSize() + "，ActiveCount：" + threadPool.getActiveCount() + "，CompletedTaskCount：" + threadPool.getCompletedTaskCount() + "]"); } catch (Exception e) { - log.debug("add task to DfpPool pools error：", e); + log.debug("add task to SysPool pools error：", e); } } //停止任务 - protected void stop() { + public void stop() { if (threadPool != null) threadPool.shutdown(); } diff --git a/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadWorker.java b/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadWorker.java index 8d84183..903a80b 100644 --- a/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadWorker.java +++ b/smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadWorker.java @@ -23,7 +23,7 @@ public abstract class SysThreadWorker implements Runnable { try { localWork(); } catch (Exception e) { - log.error("DfpPool handle error：", e); + log.error("SysPool handle error：", e); } }