Browse Source

Merge remote-tracking branch 'origin/4.0' into 4.0

4.0
lip 2 years ago
parent
commit
d093db1e65
94 changed files with 10391 additions and 50 deletions
  1. +4
    -0
      smtweb-framework/bpm/pom.xml
  2. +9
    -0
      smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/BpmStartedListener.java
  3. +10
    -0
      smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/login/AuthService.java
  4. +1
    -3
      smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/login/LoginHelper.java
  5. +20
    -0
      smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/login/LoginSuccessParam.java
  6. +3
    -0
      smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/login/LoginVO.java
  7. +19
    -0
      smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/login/MappingUserParam.java
  8. +21
    -0
      smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/login/MappingUserRet.java
  9. +57
    -0
      smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/sys/base/job/BaseJob.java
  10. +30
    -0
      smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/sys/base/job/JobCache.java
  11. +81
    -0
      smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/sys/base/job/JobEntity.java
  12. +102
    -0
      smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/sys/base/job/JobLog.java
  13. +113
    -0
      smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/sys/base/job/JobUtils.java
  14. +14
    -0
      smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/sys/base/job/TestJob.java
  15. +2
    -2
      smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/sys/oneTimeService/OneTimeTaskCleanService.java
  16. +168
    -0
      smtweb-framework/canal/client/canal.example.iml
  17. +210
    -0
      smtweb-framework/canal/client/pom.xml
  18. +76
    -0
      smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientConsts.java
  19. +290
    -0
      smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientInstance.java
  20. +32
    -0
      smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientStartedListener.java
  21. +56
    -0
      smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientVO.java
  22. +14
    -0
      smtweb-framework/canal/client/src/main/resources/client.properties
  23. +14
    -0
      smtweb-framework/canal/client/target/classes/client.properties
  24. +198
    -0
      smtweb-framework/canal/server/canal.server.iml
  25. +227
    -0
      smtweb-framework/canal/server/pom.xml
  26. +54
    -0
      smtweb-framework/canal/server/src/main/assembly/dev.xml
  27. +58
    -0
      smtweb-framework/canal/server/src/main/assembly/release.xml
  28. +15
    -0
      smtweb-framework/canal/server/src/main/bin/restart.sh
  29. +27
    -0
      smtweb-framework/canal/server/src/main/bin/startup.bat
  30. +126
    -0
      smtweb-framework/canal/server/src/main/bin/startup.sh
  31. +65
    -0
      smtweb-framework/canal/server/src/main/bin/stop.sh
  32. +13
    -0
      smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalApplication.java
  33. +97
    -0
      smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalConstants.java
  34. +606
    -0
      smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalController.java
  35. +141
    -0
      smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalLauncher.java
  36. +39
    -0
      smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStartedListener.java
  37. +169
    -0
      smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStarter.java
  38. +93
    -0
      smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/InstanceConfig.java
  39. +256
    -0
      smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/admin/CanalAdminController.java
  40. +30
    -0
      smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceAction.java
  41. +16
    -0
      smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceConfigMonitor.java
  42. +184
    -0
      smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/ManagerInstanceConfigMonitor.java
  43. +297
    -0
      smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/SpringInstanceConfigMonitor.java
  44. +182
    -0
      smtweb-framework/canal/server/src/main/resources/canal.properties
  45. +12
    -0
      smtweb-framework/canal/server/src/main/resources/canal_local.properties
  46. +59
    -0
      smtweb-framework/canal/server/src/main/resources/example/instance.properties
  47. +117
    -0
      smtweb-framework/canal/server/src/main/resources/logback.xml.bak
  48. +1375
    -0
      smtweb-framework/canal/server/src/main/resources/metrics/Canal_instances_tmpl.json
  49. +39
    -0
      smtweb-framework/canal/server/src/main/resources/spring/base-instance.xml
  50. +211
    -0
      smtweb-framework/canal/server/src/main/resources/spring/default-instance.xml
  51. +197
    -0
      smtweb-framework/canal/server/src/main/resources/spring/file-instance.xml
  52. +292
    -0
      smtweb-framework/canal/server/src/main/resources/spring/group-instance.xml
  53. +185
    -0
      smtweb-framework/canal/server/src/main/resources/spring/memory-instance.xml
  54. +61
    -0
      smtweb-framework/canal/server/src/main/resources/spring/tsdb/h2-tsdb.xml
  55. +63
    -0
      smtweb-framework/canal/server/src/main/resources/spring/tsdb/mysql-tsdb.xml
  56. +14
    -0
      smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap-config.xml
  57. +44
    -0
      smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml
  58. +48
    -0
      smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml
  59. +39
    -0
      smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql/create_table.sql
  60. +182
    -0
      smtweb-framework/canal/server/target/classes/canal.properties
  61. +12
    -0
      smtweb-framework/canal/server/target/classes/canal_local.properties
  62. +59
    -0
      smtweb-framework/canal/server/target/classes/example/instance.properties
  63. +117
    -0
      smtweb-framework/canal/server/target/classes/logback.xml.bak
  64. +1375
    -0
      smtweb-framework/canal/server/target/classes/metrics/Canal_instances_tmpl.json
  65. +39
    -0
      smtweb-framework/canal/server/target/classes/spring/base-instance.xml
  66. +211
    -0
      smtweb-framework/canal/server/target/classes/spring/default-instance.xml
  67. +197
    -0
      smtweb-framework/canal/server/target/classes/spring/file-instance.xml
  68. +292
    -0
      smtweb-framework/canal/server/target/classes/spring/group-instance.xml
  69. +185
    -0
      smtweb-framework/canal/server/target/classes/spring/memory-instance.xml
  70. +61
    -0
      smtweb-framework/canal/server/target/classes/spring/tsdb/h2-tsdb.xml
  71. +63
    -0
      smtweb-framework/canal/server/target/classes/spring/tsdb/mysql-tsdb.xml
  72. +14
    -0
      smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap-config.xml
  73. +44
    -0
      smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_history.xml
  74. +48
    -0
      smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_snapshot.xml
  75. +39
    -0
      smtweb-framework/canal/server/target/classes/spring/tsdb/sql/create_table.sql
  76. +22
    -0
      smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst
  77. +13
    -0
      smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst
  78. +34
    -0
      smtweb-framework/core/src/main/java/cc/smtweb/framework/core/CoreApplicationClosedListener.java
  79. +2
    -0
      smtweb-framework/core/src/main/java/cc/smtweb/framework/core/CoreApplicationStartedListener.java
  80. +0
    -5
      smtweb-framework/core/src/main/java/cc/smtweb/framework/core/CoreAutoConfiguration.java
  81. +9
    -1
      smtweb-framework/core/src/main/java/cc/smtweb/framework/core/common/SwConsts.java
  82. +16
    -0
      smtweb-framework/core/src/main/java/cc/smtweb/framework/core/container/base/IListenerIntf.java
  83. +40
    -0
      smtweb-framework/core/src/main/java/cc/smtweb/framework/core/container/base/SwParameter.java
  84. +166
    -0
      smtweb-framework/core/src/main/java/cc/smtweb/framework/core/container/base/SwReturn.java
  85. +113
    -0
      smtweb-framework/core/src/main/java/cc/smtweb/framework/core/container/factory/ListenerFactory.java
  86. +13
    -2
      smtweb-framework/core/src/main/java/cc/smtweb/framework/core/db/jdbc/JdbcEngine.java
  87. +7
    -7
      smtweb-framework/core/src/main/java/cc/smtweb/framework/core/mvc/config/WebMvcConfig.java
  88. +3
    -0
      smtweb-framework/core/src/main/java/cc/smtweb/framework/core/mvc/controller/IStartListener.java
  89. +1
    -1
      smtweb-framework/core/src/main/java/cc/smtweb/framework/core/session/UserSession.java
  90. +9
    -12
      smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/BaseSysService.java
  91. +4
    -3
      smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SingleRequestHelper.java
  92. +2
    -2
      smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysServiceFactory.java
  93. +1
    -1
      smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadPool.java
  94. +3
    -11
      smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadWorker.java

+ 4
- 0
smtweb-framework/bpm/pom.xml View File

@@ -35,6 +35,10 @@
<artifactId>spring-boot-starter-freemarker</artifactId>
</dependency>
<dependency>
<groupId>org.quartz-scheduler</groupId>
<artifactId>quartz</artifactId>
</dependency>
<dependency>
<groupId>net.coobird</groupId>
<artifactId>thumbnailator</artifactId>
<version>[0.4, 0.5)</version>


+ 9
- 0
smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/BpmStartedListener.java View File

@@ -9,6 +9,7 @@ import cc.smtweb.framework.core.mvc.controller.IStartListener;
import cc.smtweb.framework.core.mvc.service.TreeHelper;
import cc.smtweb.framework.core.systask.SysServiceFactory;
import cc.smtweb.system.bpm.web.design.db.ModelCatalogTreeHelper;
import cc.smtweb.system.bpm.web.sys.base.job.JobUtils;
import cc.smtweb.system.bpm.web.sys.oneTimeService.OneTimeServiceFactory;
import cc.smtweb.system.bpm.web.sys.oneTimeService.OneTimeTaskCleanService;

@@ -26,6 +27,7 @@ public class BpmStartedListener implements IStartListener {
SwConsts.SysParam.RUN_PROJECTS = "bpm";
SysServiceFactory.getInstance().reg(new OneTimeTaskCleanService());
TreeHelper.regTreeHelper(ModelCatalog.ENTITY_NAME, ModelCatalogTreeHelper.class);
}

@Override
@@ -35,5 +37,12 @@ public class BpmStartedListener implements IStartListener {
//初始化缓存
CacheManager.getIntance().init();
OneTimeServiceFactory.getInstance().start();
JobUtils.getInstance().start();
}

@Override
public void close() {
OneTimeServiceFactory.getInstance().stop();
JobUtils.getInstance().stop();
}
}

+ 10
- 0
smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/login/AuthService.java View File

@@ -5,7 +5,10 @@ import cc.smtweb.framework.core.annotation.SwParam;
import cc.smtweb.framework.core.annotation.SwService;
import cc.smtweb.framework.core.cache.CacheManager;
import cc.smtweb.framework.core.common.R;
import cc.smtweb.framework.core.common.SwConsts;
import cc.smtweb.framework.core.common.SwMap;
import cc.smtweb.framework.core.container.base.SwParameter;
import cc.smtweb.framework.core.container.factory.ListenerFactory;
import cc.smtweb.framework.core.db.DbEngine;
import cc.smtweb.framework.core.exception.BizException;
import cc.smtweb.framework.core.session.SessionManager;
@@ -91,6 +94,13 @@ public class AuthService {
data.put("token", loginAckVO.getToken());
data.put("isOk", true);
data.put("msg", "");
// 发布登录成功事件
SwParameter parameter = new SwParameter();
LoginSuccessParam loginSuccessParam = new LoginSuccessParam();
loginSuccessParam.setUs(userSession);
loginSuccessParam.setExtra(loginPO.getExtra() != null?loginPO.getExtra():new SwMap());
parameter.setSingleParam(loginSuccessParam);
ListenerFactory.fireListenerAsyc(SwConsts.EventConst.LOGIN_SUCCESS,parameter);
return R.success(data);
}



+ 1
- 3
smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/login/LoginHelper.java View File

@@ -23,7 +23,6 @@ import java.util.Set;
* 登录辅助类
*/
public class LoginHelper {

private static final String PWD_SALT = "goodpj";

public static User login(LoginVO loginPO) {
@@ -31,12 +30,11 @@ public class LoginHelper {
//先提取验证码
String verifyCode = RedisManager.getInstance().get(key, String.class);
RedisManager.getInstance().del(key);
MappingUserParam mappingUserEvent = new MappingUserParam();
if (StringUtils.isBlank(loginPO.getUsername())) {
throw new BizException("账号不能为空");
}


if (StringUtils.isBlank(loginPO.getPassword())) {
throw new BizException("密码不能为空");
}


+ 20
- 0
smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/login/LoginSuccessParam.java View File

@@ -0,0 +1,20 @@
package cc.smtweb.system.bpm.web.login;

import cc.smtweb.framework.core.common.SwMap;
import cc.smtweb.framework.core.session.UserSession;
import lombok.Data;

import java.io.Serializable;

/**
* @Author: tanghp
* @Date: 2022-09-14 9:53
* @Desc: 登录成功事件的参数对象
*/
@Data
public class LoginSuccessParam implements Serializable {
// 登录的userSession
private UserSession us;
// 其他信息
private SwMap extra;
}

+ 3
- 0
smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/login/LoginVO.java View File

@@ -1,5 +1,6 @@
package cc.smtweb.system.bpm.web.login;

import cc.smtweb.framework.core.common.SwMap;
import lombok.Data;

import java.io.Serializable;
@@ -13,4 +14,6 @@ public class LoginVO implements Serializable {
private String verifyCode;

private String uuid;

private SwMap extra;
}

+ 19
- 0
smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/login/MappingUserParam.java View File

@@ -0,0 +1,19 @@
package cc.smtweb.system.bpm.web.login;

import cc.smtweb.framework.core.common.SwMap;
import lombok.Data;

import java.io.Serializable;

/**
* @Author: tanghp
* @Date: 2022-09-09 16:52
* @Desc: 用户映射事件参数对象
*/
@Data
public class MappingUserParam implements Serializable {
// 来源
private String source;
// 其他信息
private SwMap extra;
}

+ 21
- 0
smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/login/MappingUserRet.java View File

@@ -0,0 +1,21 @@
package cc.smtweb.system.bpm.web.login;

import cc.smtweb.framework.core.common.SwMap;
import lombok.Data;

import java.io.Serializable;

/**
* @Author: tanghp
* @Date: 2022-09-09 17:32
* @Desc: 用户映射返回
*/
@Data
public class MappingUserRet implements Serializable {
// 用户ID
private long userId;
// 映射失败信息
private String failMsg;
// 其他响应信息
private SwMap extra;
}

+ 57
- 0
smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/sys/base/job/BaseJob.java View File

@@ -0,0 +1,57 @@
package cc.smtweb.system.bpm.web.sys.base.job;

import cc.smtweb.framework.core.common.SwConsts;
import cc.smtweb.framework.core.db.DbEngine;
import cc.smtweb.framework.core.db.jdbc.ISimpleWorker;
import cc.smtweb.framework.core.systask.SingleRequestHelper;
import cc.smtweb.framework.core.util.DateUtil;
import lombok.extern.slf4j.Slf4j;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;

/**
* Created by Akmm at 2022-09-14 09:51
* 定时任务的基类
*/
@Slf4j
public abstract class BaseJob implements Job {
@Override
public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
JobEntity job = (JobEntity) jobExecutionContext.getJobDetail().getJobDataMap().get("job");

final String id = this.getClass().getName();
SingleRequestHelper.singleRequest(SwConsts.REDIS_KEY_BASE_JOB, id, job.getName() + "任务执行中,本次忽略.........", () -> {
JobLog jlog = startLog(job);
try {
String info = work();
endLog(jlog, true, info);
} catch (Exception e) {
endLog(jlog, false, e.getMessage());
log.error(job.getName() + ":::执行失败", e);
}

});
}

//任务执行信息,可返回执行结果
protected abstract String work();

//存任务日志-开始
private JobLog startLog(JobEntity job) {
JobLog jlog = new JobLog();
jlog.setId(DbEngine.getInstance().nextId());
jlog.setSjId(job.getId());
jlog.setBeginTime(DateUtil.nowDateTimeLong());
DbEngine.getInstance().doTransSingle(() -> DbEngine.getInstance().findDao(JobLog.class).insertEntity(jlog));
return jlog;
}

//存任务日志-结束
private void endLog(JobLog jlog, boolean sucess, String info) {
jlog.setEndTime(DateUtil.nowDateTimeLong());
jlog.setSuccess(sucess);
jlog.setInfo(info);
DbEngine.getInstance().doTransSingle(() -> DbEngine.getInstance().findDao(JobLog.class).updateEntity(jlog));
}
}

+ 30
- 0
smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/sys/base/job/JobCache.java View File

@@ -0,0 +1,30 @@
package cc.smtweb.system.bpm.web.sys.base.job;

import cc.smtweb.framework.core.annotation.SwCache;
import cc.smtweb.framework.core.cache.AbstractEntityCache;
import cc.smtweb.framework.core.cache.CacheManager;

/**
* Created by 1 at 2022-09-13 19:35:56
* 实体【[定时任务](SYS_JOB)】的缓存类
*/
@SwCache(ident = "SYS_JOB", title = "定时任务")
public class JobCache extends AbstractEntityCache<JobEntity> {
//缓存key:按编码
public final static String mk_code = "code";

public static JobCache getInstance() {
return CacheManager.getIntance().getCache(JobCache.class);
}

public JobCache() {
//缓存key:按编码
regMap(mk_code, "sj_code");
}

//缓存key:按编码
public final JobEntity getByCode(String key) {
return getByKey(mk_code, key);
}

}

+ 81
- 0
smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/sys/base/job/JobEntity.java View File

@@ -0,0 +1,81 @@
package cc.smtweb.system.bpm.web.sys.base.job;

import cc.smtweb.framework.core.annotation.SwTable;
import cc.smtweb.framework.core.db.impl.DefaultEntity;

/**
* Created by 1 at 2022-09-13 19:35:56
* 实体【[定时任务](SYS_JOB)】的Entity类
*/
@SwTable("SYS_JOB")
public class JobEntity extends DefaultEntity {
public static final String ENTITY_NAME = "SYS_JOB";

public JobEntity() {
super(ENTITY_NAME);
}

/** 主键 */
public long getId() {
return getLong("sj_id");
}

/** 主键 */
public void setId(long sj_id) {
put("sj_id", sj_id);
}
/** 编码 */
public String getCode() {
return getStr("sj_code");
}

/** 编码 */
public void setCode(String sj_code) {
put("sj_code", sj_code);
}
/** 名称 */
public String getName() {
return getStr("sj_name");
}

/** 名称 */
public void setName(String sj_name) {
put("sj_name", sj_name);
}
/** 执行类的路径 */
public String getExeClass() {
return getStr("sj_exe_class");
}

/** 执行类的路径 */
public void setExeClass(String sj_exe_class) {
put("sj_exe_class", sj_exe_class);
}
/** cron表达式 */
public String getCron() {
return getStr("sj_cron");
}

/** cron表达式 */
public void setCron(String sj_cron) {
put("sj_cron", sj_cron);
}
/** 是否启用 */
public boolean isEnable() {
return getBool("sj_enable");
}

/** 是否启用 */
public void setEnable(boolean sj_enable) {
setBool("sj_enable", sj_enable);
}
/** 备注 */
public String getRemark() {
return getStr("sj_remark");
}

/** 备注 */
public void setRemark(String sj_remark) {
put("sj_remark", sj_remark);
}
}

+ 102
- 0
smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/sys/base/job/JobLog.java View File

@@ -0,0 +1,102 @@
package cc.smtweb.system.bpm.web.sys.base.job;

import cc.smtweb.framework.core.annotation.SwTable;
import cc.smtweb.framework.core.common.SwMap;
import cc.smtweb.framework.core.db.impl.DefaultEntity;

/**
* Created by 1 at 2022-09-14 10:09:03
* 实体【[定时任务执行日志](SYS_JOB_LOG)】的Entity类
*/
@SwTable("SYS_JOB_LOG")
public class JobLog extends DefaultEntity {
public static final String ENTITY_NAME = "SYS_JOB_LOG";

public JobLog() {
super(ENTITY_NAME);
}

/**
* 主键
*/
public long getId() {
return getLong("sjl_id");
}

/**
* 主键
*/
public void setId(long sjl_id) {
put("sjl_id", sjl_id);
}

/**
* 任务id
*/
public long getSjId() {
return getLong("sjl_sj_id");
}

/**
* 任务id
*/
public void setSjId(long sjl_sj_id) {
put("sjl_sj_id", sjl_sj_id);
}

/**
* 开始时间
*/
public long getBeginTime() {
return getLong("sjl_begin_time");
}

/**
* 开始时间
*/
public void setBeginTime(long sjl_begin_time) {
put("sjl_begin_time", sjl_begin_time);
}

/**
* 完成时间
*/
public long getEndTime() {
return getLong("sjl_end_time");
}

/**
* 完成时间
*/
public void setEndTime(long sjl_end_time) {
put("sjl_end_time", sjl_end_time);
}

/**
* 是否成功
*/
public boolean isSuccess() {
return getBool("sjl_success");
}

/**
* 是否成功
*/
public void setSuccess(boolean sjl_success) {
setBool("sjl_success", sjl_success);
}

/**
* 执行情况
*/
public String getInfo() {
return getStr("sjl_info");
}

/**
* 执行情况
*/
public void setInfo(String sjl_info) {
put("sjl_info", sjl_info);
}
}

+ 113
- 0
smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/sys/base/job/JobUtils.java View File

@@ -0,0 +1,113 @@
package cc.smtweb.system.bpm.web.sys.base.job;

import lombok.extern.slf4j.Slf4j;
import org.quartz.*;
import org.quartz.impl.StdSchedulerFactory;

import java.util.*;

/**
* Created by Akmm at 13-2-27 下午9:32
* 定时任务工厂
*/
@Slf4j
public class JobUtils {
private static Scheduler scheduler = null;
private static JobUtils instance = null;
private Map<String, JobEntity> mapJob = new HashMap<>();


public static Scheduler getScheduler() {
return scheduler;
}

public static JobUtils getInstance() {
if (instance == null) {
synchronized (JobUtils.class) {
if (instance == null) {
instance = new JobUtils();
}
}
}
return instance;
}

private JobUtils() {
try {
SchedulerFactory schedulerFactory = new StdSchedulerFactory();
scheduler = schedulerFactory.getScheduler();
} catch (Exception e) {
log.error("计划任务初始化失败", e);
e.printStackTrace();
}
}

//初始化所有任务
private void initJob() throws Exception {
Collection<JobEntity> list = JobCache.getInstance().getAll();
for (JobEntity job : list) {
if (job.isEnable()) {
addJob(job);
}
}

}

public void start() {
try {
clear();
if (!scheduler.isStarted()) scheduler.start();
initJob();
} catch (Exception e) {
log.error("计划任务开始失败", e);
e.printStackTrace();
}
}

public void stop() {
try {
if (!scheduler.isShutdown()) {
clear();
scheduler.shutdown();
}
} catch (Exception e) {
log.error("计划任务开始失败", e);
e.printStackTrace();
}
}

public void clear() throws SchedulerException {
for (JobEntity job : mapJob.values()) {
try {
deleteJob(job);
} catch (Exception e) {
log.debug("任务" + job.getId() + "删除失败!");
}
}
}

//添加任务
public synchronized void addJob(JobEntity job) throws Exception {
String jobId = String.valueOf(job.getId());
JobDetail jobDetail = JobBuilder.newJob((Class<? extends org.quartz.Job>) Class.forName(job.getExeClass()))
.withIdentity(jobId).build();
jobDetail.getJobDataMap().put("job", job);
// 按新的cronExpression表达式构建一个新的trigger
CronTrigger trigger = TriggerBuilder.newTrigger()
.withIdentity(jobId)
.startNow()
.withSchedule(CronScheduleBuilder.cronSchedule(job.getCron()))
.build();

scheduler.scheduleJob(jobDetail, trigger);
}

//删除任务
public synchronized void deleteJob(JobEntity job) throws Exception {
String jobId = String.valueOf(job.getId());

scheduler.pauseTrigger(TriggerKey.triggerKey(jobId));
scheduler.unscheduleJob(TriggerKey.triggerKey(jobId));
scheduler.deleteJob(JobKey.jobKey(jobId));
}
}

+ 14
- 0
smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/sys/base/job/TestJob.java View File

@@ -0,0 +1,14 @@
package cc.smtweb.system.bpm.web.sys.base.job;

import cc.smtweb.framework.core.util.DateUtil;

/**
* Created by Akmm at 2022-09-14 10:32
*/
public class TestJob extends BaseJob {
@Override
protected String work() {
System.out.println("执行了" + DateUtil.nowDateTimeString());
return "哈哈,成功了";
}
}

+ 2
- 2
smtweb-framework/bpm/src/main/java/cc/smtweb/system/bpm/web/sys/oneTimeService/OneTimeTaskCleanService.java View File

@@ -21,8 +21,8 @@ public class OneTimeTaskCleanService extends BaseSysService {
}

@Override
public void work() throws Exception {
public void work() {
String t = DateUtil.getNowYm() + "01000000";
DbEngine.getInstance().update("delete from " + EntityHelper.getSchemaTableName(OnetimeTask.ENTITY_NAME) + " where statu=? and create_time<?", OneTimeStatu.SUCCESS.value, t);
DbEngine.getInstance().update("delete from " + EntityHelper.getSchemaTableName(OnetimeTask.ENTITY_NAME) + " where sot_statu=? and sot_create_time<?", OneTimeStatu.SUCCESS.value, t);
}
}

+ 168
- 0
smtweb-framework/canal/client/canal.example.iml View File

@@ -0,0 +1,168 @@
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="FacetManager">
<facet type="web" name="Web">
<configuration>
<webroots />
</configuration>
</facet>
<facet type="jpa" name="JPA">
<configuration>
<setting name="validation-enabled" value="true" />
<datasource-mapping />
<naming-strategy-map />
</configuration>
</facet>
<facet type="Spring" name="Spring">
<configuration />
</facet>
</component>
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Maven: com.alibaba.otter:canal.client:1.1.6" level="project" />
<orderEntry type="library" name="Maven: com.google.protobuf:protobuf-java:3.6.1" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-all:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-buffer:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-dns:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-haproxy:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-http:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-http2:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-memcache:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-mqtt:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-redis:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-smtp:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-socks:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-stomp:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-xml:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-common:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-handler:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-transport-native-unix-common:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-handler-proxy:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-resolver:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-resolver-dns:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-transport:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-transport-rxtx:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-transport-sctp:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-transport-udt:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-transport-classes-epoll:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-transport-classes-kqueue:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-resolver-dns-classes-macos:4.1.78.Final" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.netty:netty-transport-native-epoll:linux-x86_64:4.1.78.Final" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.netty:netty-transport-native-epoll:linux-aarch_64:4.1.78.Final" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.netty:netty-transport-native-kqueue:osx-x86_64:4.1.78.Final" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.netty:netty-transport-native-kqueue:osx-aarch_64:4.1.78.Final" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.netty:netty-resolver-dns-native-macos:osx-x86_64:4.1.78.Final" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.netty:netty-resolver-dns-native-macos:osx-aarch_64:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.5" level="project" />
<orderEntry type="library" name="Maven: org.jboss.netty:netty:3.2.2.Final" level="project" />
<orderEntry type="library" name="Maven: com.101tec:zkclient:0.10" level="project" />
<orderEntry type="library" name="Maven: commons-io:commons-io:2.4" level="project" />
<orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" />
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.15" level="project" />
<orderEntry type="library" name="Maven: com.google.guava:guava:22.0" level="project" />
<orderEntry type="library" name="Maven: com.google.code.findbugs:jsr305:1.3.9" level="project" />
<orderEntry type="library" name="Maven: com.google.errorprone:error_prone_annotations:2.0.18" level="project" />
<orderEntry type="library" name="Maven: com.google.j2objc:j2objc-annotations:1.1" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.mojo:animal-sniffer-annotations:1.14" level="project" />
<orderEntry type="library" name="Maven: ch.qos.logback:logback-core:1.2.11" level="project" />
<orderEntry type="library" name="Maven: ch.qos.logback:logback-classic:1.2.11" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:jcl-over-slf4j:1.7.36" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.36" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-core:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-jcl:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-aop:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-beans:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-context:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-expression:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-jdbc:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-tx:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-orm:5.3.21" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.otter:canal.protocol:1.1.6" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.otter:canal.common:1.1.6" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.fastjson2:fastjson2:2.0.4" level="project" />
<orderEntry type="library" name="Maven: com.alibaba:druid:1.2.11" level="project" />
<orderEntry type="module" module-name="sw-system-bpm" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-web:2.6.9" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter:2.6.9" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot:2.6.9" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-autoconfigure:2.6.9" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-logging:2.6.9" level="project" />
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-to-slf4j:2.17.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-api:2.17.2" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:jul-to-slf4j:1.7.36" level="project" />
<orderEntry type="library" name="Maven: jakarta.annotation:jakarta.annotation-api:1.3.5" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-json:2.6.9" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.13.3" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.3" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.module:jackson-module-parameter-names:2.13.3" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-tomcat:2.6.9" level="project" />
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-core:9.0.64" level="project" />
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-el:9.0.64" level="project" />
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-websocket:9.0.64" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-web:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-webmvc:5.3.21" level="project" />
<orderEntry type="module" module-name="sw-framework-core" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4.15" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.13.3" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-jdbc:2.6.9" level="project" />
<orderEntry type="library" name="Maven: com.zaxxer:HikariCP:4.0.3" level="project" />
<orderEntry type="library" name="Maven: io.lettuce:lettuce-core:6.1.8.RELEASE" level="project" />
<orderEntry type="library" name="Maven: io.projectreactor:reactor-core:3.4.19" level="project" />
<orderEntry type="library" name="Maven: org.reactivestreams:reactive-streams:1.0.4" level="project" />
<orderEntry type="library" name="Maven: com.esotericsoftware:kryo:4.0.0" level="project" />
<orderEntry type="library" name="Maven: com.esotericsoftware:reflectasm:1.11.3" level="project" />
<orderEntry type="library" name="Maven: org.ow2.asm:asm:5.0.4" level="project" />
<orderEntry type="library" name="Maven: com.esotericsoftware:minlog:1.3.0" level="project" />
<orderEntry type="library" name="Maven: org.objenesis:objenesis:2.2" level="project" />
<orderEntry type="library" name="Maven: org.projectlombok:lombok:1.18.24" level="project" />
<orderEntry type="library" name="Maven: org.dom4j:dom4j:2.1.3" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-jexl3:3.2.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.tika:tika-core:2.1.0" level="project" />
<orderEntry type="library" name="Maven: com.github.ben-manes.caffeine:caffeine:2.9.3" level="project" />
<orderEntry type="library" name="Maven: org.checkerframework:checker-qual:3.19.0" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-freemarker:2.6.9" level="project" />
<orderEntry type="library" name="Maven: org.freemarker:freemarker:2.3.31" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-context-support:5.3.21" level="project" />
<orderEntry type="library" name="Maven: net.coobird:thumbnailator:0.4.17" level="project" />
<orderEntry type="library" name="Maven: org.jclarion:image4j:0.7" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.3" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.13.3" level="project" />
<orderEntry type="library" name="Maven: org.yaml:snakeyaml:1.29" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.13.3" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.13.3" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.woodstox:stax2-api:4.2.1" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.woodstox:woodstox-core:6.2.7" level="project" />
<orderEntry type="library" name="Maven: org.apache.velocity:velocity-engine-core:2.3" level="project" />
<orderEntry type="library" name="Maven: com.aliyun:aliyun-java-sdk-core:4.0.6" level="project" />
<orderEntry type="library" name="Maven: com.google.code.gson:gson:2.8.9" level="project" />
<orderEntry type="library" name="Maven: javax.xml.bind:jaxb-api:2.3.1" level="project" />
<orderEntry type="library" name="Maven: javax.activation:javax.activation-api:1.2.0" level="project" />
<orderEntry type="library" name="Maven: com.sun.xml.bind:jaxb-core:2.1.14" level="project" />
<orderEntry type="library" name="Maven: com.sun.xml.bind:jaxb-impl:2.1" level="project" />
<orderEntry type="library" name="Maven: javax.activation:activation:1.1.1" level="project" />
<orderEntry type="library" name="Maven: commons-net:commons-net:3.6" level="project" />
<orderEntry type="library" name="Maven: com.jcraft:jsch:0.1.53" level="project" />
<orderEntry type="library" name="Maven: commons-fileupload:commons-fileupload:1.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.5.13" level="project" />
<orderEntry type="library" name="Maven: mysql:mysql-connector-java:8.0.29" level="project" />
<orderEntry type="library" name="Maven: org.apache.ddlutils:ddlutils:1.0" level="project" />
<orderEntry type="library" name="Maven: commons-logging:commons-logging:1.0.4" level="project" />
<orderEntry type="library" name="Maven: oro:oro:2.0.8" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-pool2:2.5.0" level="project" />
<orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils:1.8.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.7" level="project" />
<orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.13.2" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:2.2" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest:2.2" level="project" />
</component>
</module>

+ 210
- 0
smtweb-framework/canal/client/pom.xml View File

@@ -0,0 +1,210 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.6.9</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>

<groupId>cc.smtweb</groupId>
<artifactId>canal.client</artifactId>
<version>1.1.6</version>

<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.client</artifactId>
<version>1.1.6</version>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.protocol</artifactId>
<version>1.1.6</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
<version>1.2.11</version>
</dependency>

<dependency>
<groupId>cc.smtweb</groupId>
<artifactId>sw-system-bpm</artifactId>
<version>3.1.0-SNAPSHOT</version>
</dependency>

<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<dependency>
<groupId>org.apache.ddlutils</groupId>
<artifactId>ddlutils</artifactId>
<version>1.0</version>
<exclusions>
<exclusion>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils-core</artifactId>
</exclusion>
<exclusion>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</exclusion>
<exclusion>
<groupId>commons-dbcp</groupId>
<artifactId>commons-dbcp</artifactId>
</exclusion>
<exclusion>
<groupId>commons-pool</groupId>
<artifactId>commons-pool</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging-api</artifactId>
</exclusion>
<exclusion>
<groupId>dom4j</groupId>
<artifactId>dom4j</artifactId>
</exclusion>
<exclusion>
<groupId>stax</groupId>
<artifactId>stax-api</artifactId>
</exclusion>
<exclusion>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
</exclusion>
<exclusion>
<groupId>commons-digester</groupId>
<artifactId>commons-digester</artifactId>
</exclusion>
<exclusion>
<groupId>commons-betwixt</groupId>
<artifactId>commons-betwixt</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
<version>2.5.0</version>
</dependency>
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
<version>1.8.2</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.7</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2</version>
</dependency>

<!-- test dependency -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

<build>
<plugins>
<!-- deploy模块的packaging通常是jar,如果项目中没有java 源代码或资源文件,加上这一段配置使项目能通过构建 -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.5</version>
<configuration>
<archive>
<addMavenDescriptor>true</addMavenDescriptor>
</archive>
</configuration>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<!-- 这是最新版本,推荐使用这个版本 -->
<version>2.2.1</version>
<executions>
<execution>
<id>assemble</id>
<goals>
<goal>single</goal>
</goals>
<phase>package</phase>
</execution>
</executions>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<attach>false</attach>
</configuration>
</plugin>
</plugins>
</build>

<profiles>
<profile>
<id>dev</id>
<activation>
<activeByDefault>true</activeByDefault>
<property>
<name>env</name>
<value>!release</value>
</property>
</activation>

<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<!-- maven assembly插件需要一个描述文件 来告诉插件包的结构以及打包所需的文件来自哪里 -->
<descriptors>
<descriptor>${basedir}/src/main/assembly/dev.xml</descriptor>
</descriptors>
<finalName>canal-example</finalName>
<outputDirectory>${project.build.directory}</outputDirectory>
</configuration>
</plugin>
</plugins>
</build>

</profile>

<profile>
<id>release</id>
<activation>
<property>
<name>env</name>
<value>release</value>
</property>
</activation>

<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<!-- 发布模式使用的maven assembly插件描述文件 -->
<descriptors>
<descriptor>${basedir}/src/main/assembly/release.xml</descriptor>
</descriptors>
<!-- 如果一个应用的包含多个deploy模块,如果使用同样的包名, 如果把它们复制的一个目录中可能会失败,所以包名加了 artifactId以示区分 -->
<finalName>${project.artifactId}-1.1.6</finalName>
<!-- scm 要求 release 模式打出的包放到顶级目录下的target子目录中 -->
<outputDirectory>${project.parent.build.directory}</outputDirectory>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

+ 76
- 0
smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientConsts.java View File

@@ -0,0 +1,76 @@
package cc.smtweb.system.canal.client;

import org.apache.commons.lang.StringUtils;

import java.util.Properties;

/**
* 启动常用变量
*
* @author jianghang 2012-11-8 下午03:15:55
* @version 1.0.0
*/
public class ClientConsts {

public static final String ROOT = "canal.client";
public static final String SIZE = ROOT + "." + "size";
public static final String INSTANCE = ROOT + "." + "instance";
public static final String IP = ROOT + "." + "ip";
public static final String PORT = ROOT + "." + "port";
public static final String USERNAME = ROOT + "." + "username";
public static final String PASSWORD = ROOT + "." + "password";
public static final String FILTER = ROOT + "." + "filter";


public static String getIp(Properties properties){
return getProperty(properties,IP,"127.0.0.1");
}

public static int getPort(Properties properties){
return Integer.parseInt(getProperty(properties,PORT,"11111"));
}

public static int getBatchSize(Properties properties){
return Integer.parseInt(getProperty(properties,SIZE,"50"));
}

public static String getInstance(Properties properties){
return getProperty(properties,INSTANCE,"example");
}

public static String getUsername(Properties properties){
return getProperty(properties,USERNAME,"");
}

public static String getPassword(Properties properties){
return getProperty(properties,PASSWORD,"");
}

public static String getFilter(Properties properties){
return getProperty(properties,FILTER,".*\\..*");
}

public static String getProperty(Properties properties, String key, String defaultValue) {
String value = getProperty(properties, key);
if (StringUtils.isEmpty(value)) {
return defaultValue;
} else {
return value;
}
}

public static String getProperty(Properties properties, String key) {
key = StringUtils.trim(key);
String value = System.getProperty(key);

if (value == null) {
value = System.getenv(key);
}

if (value == null) {
value = properties.getProperty(key);
}

return StringUtils.trim(value);
}
}

+ 290
- 0
smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientInstance.java View File

@@ -0,0 +1,290 @@
package cc.smtweb.system.canal.client;

import cc.smtweb.framework.core.util.CommUtil;
import cc.smtweb.framework.core.util.JsonUtil;
import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.Message;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;

import java.io.FileInputStream;
import java.net.InetSocketAddress;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
* @Author yaoq
* @Date 2022年09月13日 18:02
* @Description
*/
public class ClientInstance {
private static final String CLASSPATH_URL_PREFIX = "classpath:";
private static final Logger logger = LoggerFactory.getLogger(ClientInstance.class);
protected volatile boolean running = false;
protected Thread thread = null;
protected CanalConnector connector;
protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {

public void uncaughtException(Thread t, Throwable e) {
logger.error("解析出错了哦", e);

}
};

private static ClientInstance instance = null;
private static Properties properties = null;

static {
instance = new ClientInstance();
}

public static ClientInstance getInstance() {
return instance;
}

private void loadProperties() {
try {
String conf = System.getProperty("client.conf", "classpath:client.properties");
properties = new Properties();
if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
properties.load(ClientInstance.class.getClassLoader().getResourceAsStream(conf));
} else {
properties.load(new FileInputStream(conf));
}
} catch (Exception e) {
logger.error("加载canal客户端配置参数失败!", e);
}
}

private ClientInstance() {
loadProperties();
connector = CanalConnectors.newSingleConnector(new
InetSocketAddress(ClientConsts.getIp(properties),
ClientConsts.getPort(properties)), ClientConsts.getInstance(properties), ClientConsts.getUsername(properties), ClientConsts.getPassword(properties));
connector.connect();
connector.subscribe(ClientConsts.getFilter(properties));
connector.rollback();

Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
logger.info("## stop the canal client");
ClientInstance.this.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping canal:", e);
} finally {
logger.info("## canal client is down.");
}
}));
}

//开始
public void start() {
Assert.notNull(connector, "connector is null");
thread = new Thread(() -> process());

thread.setUncaughtExceptionHandler(handler);
running = true;
thread.start();
}

//停止
public void stop() {
if (!running) {
return;
}
running = false;
if (thread != null) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
}
}

protected void process() {
while (running) {
try {
connector.disconnect();
connector.connect();
connector.unsubscribe();
connector.subscribe(ClientConsts.getFilter(properties));
while (running) {
Message message = connector.getWithoutAck(ClientConsts.getBatchSize(properties) * 1024); // 获取指定数量的数据
long batchId = message.getId();
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
Thread.sleep(3000L);//无数据时,睡眠。
continue;
} else {
dataHandle(message.getEntries());
}
if (batchId != -1) {
connector.ack(batchId); // 提交确认
}
}
} catch (Throwable e) {
logger.error("process error!", e);
try {
Thread.sleep(1000L);
} catch (InterruptedException e1) {
// ignore
}
if (!isReConnection()) {
connector.rollback(); // 处理失败, 回滚数据
}
} finally {
connector.disconnect();
}
}
}

private boolean isReConnection() {
if (!connector.checkValid()) return false;
int count = 1;
int rate = 1;
while (connector.checkValid()) {
rate = (count + 100) / 100;
try {
logger.debug("canal服务连接中断,第【" + count + "】次重连开始");
connector.disconnect();
connector.connect();
logger.debug("canal服务连接中断,第【" + count + "】次重连成功");
break;
} catch (Throwable e2) {
if (count > 99) {
logger.error("canal服务连接异常!", "重新连接canal服务端已失败【" + count + "】次,请检查canal服务是否启动,请及时启动canal服务端,或联系研发人员!");
}
logger.debug("canal服务连接中断,第【" + count + "】次重连失败:" + CommUtil.getOrigMsg(e2));
logger.debug("canal服务连接中断," + (3 * rate) + "分钟后进行第【" + (count + 1) + "】连接测试!");
try {
Thread.sleep(180 * rate * 1000L);
} catch (InterruptedException e1) {
// ignore
}
}
count++;
}
return true;
}

/**
* 数据处理
*
* @param entrys
*/
private void dataHandle(List<CanalEntry.Entry> entrys) throws Exception {
for (CanalEntry.Entry entry : entrys) {
if (CanalEntry.EntryType.ROWDATA == entry.getEntryType()) {
CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
CanalEntry.EventType eventType = rowChange.getEventType();
if (eventType == CanalEntry.EventType.DELETE) {
saveDeleteSql(entry);
} else if (eventType == CanalEntry.EventType.UPDATE) {
saveUpdateSql(entry);
} else if (eventType == CanalEntry.EventType.INSERT) {
saveInsertSql(entry);
}
}
}
}

/**
* 保存更新语句
*
* @param entry
*/
private void saveUpdateSql(CanalEntry.Entry entry) {
try {
CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
List<CanalEntry.RowData> rowDatasList = rowChange.getRowDatasList();
for (CanalEntry.RowData rowData : rowDatasList) {
List<CanalEntry.Column> newColumnList = rowData.getAfterColumnsList();
Map<String, String> data = new HashMap<>();
newColumnList.stream().forEach(item -> {
if (StringUtils.isNotEmpty(item.getValue())) {
data.put(lineToHump(item.getName()), item.getValue());
}
});
List<CanalEntry.Column> oldColumnList = rowData.getBeforeColumnsList();
ClientVO update = ClientVO.ok(entry.getHeader().getTableName(), "UPDATE", data, oldColumnList.get(0).getValue());
System.out.println("更新返回 : " + JsonUtil.encodeString(update));
}
} catch (Exception e) {
e.printStackTrace();
}
}

/**
* 保存删除语句
*
* @param entry
*/
private void saveDeleteSql(CanalEntry.Entry entry) {
try {
CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
List<CanalEntry.RowData> rowDatasList = rowChange.getRowDatasList();
for (CanalEntry.RowData rowData : rowDatasList) {
List<CanalEntry.Column> oldColumnList = rowData.getBeforeColumnsList();
ClientVO delete = ClientVO.ok(entry.getHeader().getTableName(), "DELETE", null, oldColumnList.get(0).getValue());
System.out.println("删除返回 : " + JsonUtil.encodeString(delete));
}
} catch (Exception e) {
e.printStackTrace();
}
}

/**
* 保存插入语句
*
* @param entry
*/
private void saveInsertSql(CanalEntry.Entry entry) {
try {
CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
List<CanalEntry.RowData> rowDatasList = rowChange.getRowDatasList();
for (CanalEntry.RowData rowData : rowDatasList) {
List<CanalEntry.Column> columnList = rowData.getAfterColumnsList();
Map<String, String> data = new HashMap<>();
columnList.stream().forEach(item -> {
if (StringUtils.isNotEmpty(item.getValue())) {
data.put(lineToHump(item.getName()), item.getValue());
}
});
ClientVO insert = ClientVO.ok(entry.getHeader().getTableName(), "INSERT", data, null);
System.out.println("插入返回 : " + JsonUtil.encodeString(insert));
}
} catch (Exception e) {
e.printStackTrace();
}
}

/**
* 下划线转驼峰
*
* @param str
*/
private static Pattern linePattern = Pattern.compile("_(\\w)");

public static String lineToHump(String str) {
str = str.toLowerCase();
Matcher matcher = linePattern.matcher(str);
StringBuffer sb = new StringBuffer();
while (matcher.find()) {
matcher.appendReplacement(sb, matcher.group(1).toUpperCase());
}
matcher.appendTail(sb);
return sb.toString();
}
}



+ 32
- 0
smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientStartedListener.java View File

@@ -0,0 +1,32 @@
package cc.smtweb.system.canal.client;

import cc.smtweb.framework.core.annotation.SwStartListener;
import cc.smtweb.framework.core.common.SwConsts;
import cc.smtweb.framework.core.mvc.controller.IStartListener;
import lombok.extern.slf4j.Slf4j;

/**
* @Author yaoq
* @Date 2022年09月06日 10:31
* @Description
*/
@Slf4j
@SwStartListener
public class ClientStartedListener implements IStartListener {

@Override
public int order() {
return SwConsts.DEFAULT_ORDER + 3;
}


@Override
public void init() {

}

@Override
public void run() {
ClientInstance.getInstance().start();
}
}

+ 56
- 0
smtweb-framework/canal/client/src/main/java/cc/smtweb/system/canal/client/ClientVO.java View File

@@ -0,0 +1,56 @@
package cc.smtweb.system.canal.client;

import java.util.Map;

/**
* @Author yaoq
* @Date 2022年09月13日 18:20
* @Description
*/
public class ClientVO {
private String tableName; // 表名
private String type; // 类型(更新、删除、插入)
private Map<String,String> data; // 数据JSON 自己转对应表格实体类
private String id; // 更新或删除都是根据ID来

public static ClientVO ok(String tableName, String type, Map<String,String> data, String id){
ClientVO canalVO=new ClientVO();
canalVO.setId(id);
canalVO.setTableName(tableName);
canalVO.setType(type);
canalVO.setData(data);
return canalVO;
}

public String getTableName() {
return tableName;
}

public void setTableName(String tableName) {
this.tableName = tableName;
}

public String getType() {
return type;
}

public void setType(String type) {
this.type = type;
}

public Map<String, String> getData() {
return data;
}

public void setData(Map<String, String> data) {
this.data = data;
}

public String getId() {
return id;
}

public void setId(String id) {
this.id = id;
}
}

+ 14
- 0
smtweb-framework/canal/client/src/main/resources/client.properties View File

@@ -0,0 +1,14 @@
#canal ???????
# ??????? ??MB
canal.server.size=50
# ??????
canal.server.instance=example
canal.server.ip=127.0.0.1
canal.server.port=11111
# ????????? ??????
canal.server.username=
canal.server.password=
# ??????????.*\\..*??????scmz\\..*
canal.server.filter=.*\\..*



+ 14
- 0
smtweb-framework/canal/client/target/classes/client.properties View File

@@ -0,0 +1,14 @@
#canal ???????
# ??????? ??MB
canal.server.size=50
# ??????
canal.server.instance=example
canal.server.ip=127.0.0.1
canal.server.port=11111
# ????????? ??????
canal.server.username=
canal.server.password=
# ??????????.*\\..*??????scmz\\..*
canal.server.filter=.*\\..*



+ 198
- 0
smtweb-framework/canal/server/canal.server.iml View File

@@ -0,0 +1,198 @@
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="FacetManager">
<facet type="web" name="Web">
<configuration>
<webroots />
</configuration>
</facet>
<facet type="jpa" name="JPA">
<configuration>
<setting name="validation-enabled" value="true" />
<datasource-mapping />
<naming-strategy-map />
</configuration>
</facet>
<facet type="Spring" name="Spring">
<configuration />
</facet>
</component>
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Maven: com.alibaba.otter:canal.protocol:1.1.6" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.otter:canal.common:1.1.6" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.fastjson2:fastjson2:2.0.4" level="project" />
<orderEntry type="library" name="Maven: com.google.code.findbugs:jsr305:3.0.2" level="project" />
<orderEntry type="library" name="Maven: com.google.protobuf:protobuf-java:3.6.1" level="project" />
<orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.otter:canal.client:1.1.6" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-all:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-buffer:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-dns:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-haproxy:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-http:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-http2:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-memcache:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-mqtt:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-redis:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-smtp:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-socks:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-stomp:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-xml:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-common:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-handler:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-transport-native-unix-common:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-handler-proxy:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-resolver:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-resolver-dns:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-transport:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-transport-rxtx:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-transport-sctp:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-transport-udt:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-transport-classes-epoll:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-transport-classes-kqueue:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-resolver-dns-classes-macos:4.1.78.Final" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.netty:netty-transport-native-epoll:linux-x86_64:4.1.78.Final" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.netty:netty-transport-native-epoll:linux-aarch_64:4.1.78.Final" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.netty:netty-transport-native-kqueue:osx-x86_64:4.1.78.Final" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.netty:netty-transport-native-kqueue:osx-aarch_64:4.1.78.Final" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.netty:netty-resolver-dns-native-macos:osx-x86_64:4.1.78.Final" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.netty:netty-resolver-dns-native-macos:osx-aarch_64:4.1.78.Final" level="project" />
<orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.5" level="project" />
<orderEntry type="library" name="Maven: org.jboss.netty:netty:3.2.2.Final" level="project" />
<orderEntry type="library" name="Maven: com.101tec:zkclient:0.10" level="project" />
<orderEntry type="library" name="Maven: commons-io:commons-io:2.4" level="project" />
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.15" level="project" />
<orderEntry type="library" name="Maven: com.google.guava:guava:22.0" level="project" />
<orderEntry type="library" name="Maven: com.google.errorprone:error_prone_annotations:2.0.18" level="project" />
<orderEntry type="library" name="Maven: com.google.j2objc:j2objc-annotations:1.1" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.mojo:animal-sniffer-annotations:1.14" level="project" />
<orderEntry type="library" name="Maven: ch.qos.logback:logback-core:1.2.11" level="project" />
<orderEntry type="library" name="Maven: ch.qos.logback:logback-classic:1.2.11" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:jcl-over-slf4j:1.7.36" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.36" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-core:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-jcl:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-aop:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-beans:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-context:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-expression:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-jdbc:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-tx:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-orm:5.3.21" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.otter:canal.server:1.1.6" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.otter:canal.instance.core:1.1.6" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.otter:canal.store:1.1.6" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.otter:canal.meta:1.1.6" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.otter:canal.parse:1.1.6" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.otter:canal.sink:1.1.6" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.otter:canal.parse.dbsync:1.1.6" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.otter:canal.filter:1.1.6" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.otter:canal.parse.driver:1.1.6" level="project" />
<orderEntry type="library" name="Maven: com.alibaba:druid:1.2.10" level="project" />
<orderEntry type="library" name="Maven: org.mybatis:mybatis-spring:2.0.4" level="project" />
<orderEntry type="library" name="Maven: org.mybatis:mybatis:3.5.4" level="project" />
<orderEntry type="library" name="Maven: com.h2database:h2:1.4.200" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.9" level="project" />
<orderEntry type="library" name="Maven: com.lmax:disruptor:3.4.2" level="project" />
<orderEntry type="library" name="Maven: javax.annotation:javax.annotation-api:1.3.2" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.otter:canal.instance.spring:1.1.6" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.otter:canal.instance.manager:1.1.6" level="project" />
<orderEntry type="library" name="Maven: com.alibaba.otter:connector.core:1.1.6" level="project" />
<orderEntry type="library" name="Maven: com.googlecode.aviator:aviator:2.2.1" level="project" />
<orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils:1.8.2" level="project" />
<orderEntry type="library" name="Maven: oro:oro:2.0.8" level="project" />
<orderEntry type="library" name="Maven: joda-time:joda-time:2.9.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.12.0" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: com.alibaba.otter:canal.prometheus:1.1.6" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.jctools:jctools-core:2.1.2" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.prometheus:simpleclient:0.12.0" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.prometheus:simpleclient_tracer_otel:0.12.0" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.prometheus:simpleclient_tracer_common:0.12.0" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.prometheus:simpleclient_tracer_otel_agent:0.12.0" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.prometheus:simpleclient_hotspot:0.12.0" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.prometheus:simpleclient_httpserver:0.12.0" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.prometheus:simpleclient_common:0.12.0" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: io.prometheus:simpleclient_pushgateway:0.12.0" level="project" />
<orderEntry type="library" name="Maven: javax.xml.bind:jaxb-api:2.3.1" level="project" />
<orderEntry type="library" name="Maven: javax.activation:javax.activation-api:1.2.0" level="project" />
<orderEntry type="library" scope="PROVIDED" name="Maven: com.alibaba.otter:connector.kafka:jar-with-dependencies:1.1.6" level="project" />
<orderEntry type="library" scope="PROVIDED" name="Maven: com.alibaba.otter:connector.rocketmq:jar-with-dependencies:1.1.6" level="project" />
<orderEntry type="library" scope="PROVIDED" name="Maven: com.alibaba.otter:connector.rabbitmq:jar-with-dependencies:1.1.6" level="project" />
<orderEntry type="library" scope="PROVIDED" name="Maven: com.alibaba.otter:connector.pulsarmq:jar-with-dependencies:1.1.6" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-autoconfigure:2.6.9" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot:2.6.9" level="project" />
<orderEntry type="module" module-name="sw-framework-core" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter:2.6.9" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-logging:2.6.9" level="project" />
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-to-slf4j:2.17.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-api:2.17.2" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:jul-to-slf4j:1.7.36" level="project" />
<orderEntry type="library" name="Maven: jakarta.annotation:jakarta.annotation-api:1.3.5" level="project" />
<orderEntry type="library" name="Maven: org.yaml:snakeyaml:1.29" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-web:2.6.9" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-json:2.6.9" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.13.3" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.3" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.module:jackson-module-parameter-names:2.13.3" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-tomcat:2.6.9" level="project" />
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-core:9.0.64" level="project" />
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-el:9.0.64" level="project" />
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-websocket:9.0.64" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-web:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-webmvc:5.3.21" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.5.13" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4.15" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.13.3" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-jdbc:2.6.9" level="project" />
<orderEntry type="library" name="Maven: com.zaxxer:HikariCP:4.0.3" level="project" />
<orderEntry type="library" name="Maven: io.lettuce:lettuce-core:6.1.8.RELEASE" level="project" />
<orderEntry type="library" name="Maven: io.projectreactor:reactor-core:3.4.19" level="project" />
<orderEntry type="library" name="Maven: org.reactivestreams:reactive-streams:1.0.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-pool2:2.11.1" level="project" />
<orderEntry type="library" name="Maven: com.esotericsoftware:kryo:4.0.0" level="project" />
<orderEntry type="library" name="Maven: com.esotericsoftware:reflectasm:1.11.3" level="project" />
<orderEntry type="library" name="Maven: org.ow2.asm:asm:5.0.4" level="project" />
<orderEntry type="library" name="Maven: com.esotericsoftware:minlog:1.3.0" level="project" />
<orderEntry type="library" name="Maven: org.objenesis:objenesis:2.2" level="project" />
<orderEntry type="library" name="Maven: org.projectlombok:lombok:1.18.24" level="project" />
<orderEntry type="library" name="Maven: org.dom4j:dom4j:2.1.3" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-jexl3:3.2.1" level="project" />
<orderEntry type="library" name="Maven: commons-logging:commons-logging:1.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.tika:tika-core:2.1.0" level="project" />
<orderEntry type="library" name="Maven: com.github.ben-manes.caffeine:caffeine:2.9.3" level="project" />
<orderEntry type="library" name="Maven: org.checkerframework:checker-qual:3.19.0" level="project" />
<orderEntry type="library" name="Maven: mysql:mysql-connector-java:8.0.29" level="project" />
<orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2.2" level="project" />
<orderEntry type="module" module-name="sw-system-bpm" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-freemarker:2.6.9" level="project" />
<orderEntry type="library" name="Maven: org.freemarker:freemarker:2.3.31" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-context-support:5.3.21" level="project" />
<orderEntry type="library" name="Maven: net.coobird:thumbnailator:0.4.17" level="project" />
<orderEntry type="library" name="Maven: org.jclarion:image4j:0.7" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.3" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.13.3" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.13.3" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.13.3" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.woodstox:stax2-api:4.2.1" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.woodstox:woodstox-core:6.2.7" level="project" />
<orderEntry type="library" name="Maven: org.apache.velocity:velocity-engine-core:2.3" level="project" />
<orderEntry type="library" name="Maven: com.aliyun:aliyun-java-sdk-core:4.0.6" level="project" />
<orderEntry type="library" name="Maven: com.google.code.gson:gson:2.8.9" level="project" />
<orderEntry type="library" name="Maven: com.sun.xml.bind:jaxb-core:2.1.14" level="project" />
<orderEntry type="library" name="Maven: com.sun.xml.bind:jaxb-impl:2.1" level="project" />
<orderEntry type="library" name="Maven: javax.activation:activation:1.1.1" level="project" />
<orderEntry type="library" name="Maven: commons-net:commons-net:3.6" level="project" />
<orderEntry type="library" name="Maven: com.jcraft:jsch:0.1.53" level="project" />
<orderEntry type="library" name="Maven: commons-fileupload:commons-fileupload:1.4" level="project" />
</component>
</module>

+ 227
- 0
smtweb-framework/canal/server/pom.xml View File

@@ -0,0 +1,227 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.6.9</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>

<groupId>cc.smtweb</groupId>
<artifactId>canal.server</artifactId>
<version>1.1.6</version>

<dependencies>

<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.protocol</artifactId>
<version>1.1.6</version>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.client</artifactId>
<version>1.1.6</version>
</dependency>

<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.server</artifactId>
<version>1.1.6</version>
</dependency>

<!-- 这里指定runtime的metrics provider-->
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.prometheus</artifactId>
<version>1.1.6</version>
<scope>runtime</scope>
</dependency>

<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.kafka</artifactId>
<version>1.1.6</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.rocketmq</artifactId>
<version>1.1.6</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.rabbitmq</artifactId>
<version>1.1.6</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.pulsarmq</artifactId>
<version>1.1.6</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
<dependency>
<groupId>cc.smtweb</groupId>
<artifactId>sw-framework-core</artifactId>
<version>3.1.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>

<dependency>
<groupId>cc.smtweb</groupId>
<artifactId>sw-system-bpm</artifactId>
<version>3.1.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<addMavenDescriptor>true</addMavenDescriptor>
</archive>
<excludes>
<exclude>**/logback.xml</exclude>
<exclude>**/canal.properties</exclude>
<exclude>**/spring/**</exclude>
<exclude>**/example/**</exclude>
<exclude>**/mq.yml</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.10</version>
<executions>
<execution>
<id>copy-dependencies-to-canal-deployer</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<includeClassifiers>jar-with-dependencies</includeClassifiers>
<outputDirectory>${project.basedir}/target/canal/plugin</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<!-- 这是最新版本,推荐使用这个版本 -->
<version>2.2.1</version>
<executions>
<execution>
<id>assemble</id>
<goals>
<goal>single</goal>
</goals>
<phase>package</phase>
</execution>
</executions>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<attach>false</attach>
</configuration>
</plugin>
</plugins>
</build>

<profiles>
<profile>
<id>dev</id>
<activation>
<activeByDefault>true</activeByDefault>
<property>
<name>env</name>
<value>!release</value>
</property>
</activation>

<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<!-- maven assembly插件需要一个描述文件 来告诉插件包的结构以及打包所需的文件来自哪里 -->
<descriptors>
<descriptor>${basedir}/src/main/assembly/dev.xml</descriptor>
</descriptors>
<finalName>canal</finalName>
<outputDirectory>${project.build.directory}</outputDirectory>
</configuration>
</plugin>
</plugins>
</build>

</profile>

<profile>
<id>release</id>
<activation>
<property>
<name>env</name>
<value>release</value>
</property>
</activation>

<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<!-- 发布模式使用的maven assembly插件描述文件 -->
<descriptors>
<descriptor>${basedir}/src/main/assembly/release.xml</descriptor>
</descriptors>
<!-- 如果一个应用的包含多个deploy模块,如果使用同样的包名, 如果把它们复制的一个目录中可能会失败,所以包名加了 artifactId以示区分 -->
<finalName>${project.artifactId}-1.1.6</finalName>
<!-- scm 要求 release 模式打出的包放到顶级目录下的target子目录中 -->
<outputDirectory>${project.parent.build.directory}</outputDirectory>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

+ 54
- 0
smtweb-framework/canal/server/src/main/assembly/dev.xml View File

@@ -0,0 +1,54 @@
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id>dist</id>
<formats>
<format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
<directory>.</directory>
<outputDirectory>/</outputDirectory>
<includes>
<include>README*</include>
</includes>
</fileSet>
<fileSet>
<directory>./src/main/bin</directory>
<outputDirectory>bin</outputDirectory>
<includes>
<include>**/*</include>
</includes>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>./src/main/conf</directory>
<outputDirectory>/conf</outputDirectory>
<includes>
<include>**/*</include>
</includes>
</fileSet>
<fileSet>
<directory>./src/main/resources</directory>
<outputDirectory>/conf</outputDirectory>
<includes>
<include>**/*</include>
</includes>
</fileSet>
<fileSet>
<directory>target</directory>
<outputDirectory>logs</outputDirectory>
<excludes>
<exclude>**/*</exclude>
</excludes>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<outputDirectory>lib</outputDirectory>
<excludes>
<exclude>junit:junit</exclude>
</excludes>
</dependencySet>
</dependencySets>
</assembly>

+ 58
- 0
smtweb-framework/canal/server/src/main/assembly/release.xml View File

@@ -0,0 +1,58 @@
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id>dist</id>
<formats>
<format>tar.gz</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
<directory>.</directory>
<outputDirectory>/</outputDirectory>
<includes>
<include>README*</include>
</includes>
</fileSet>
<fileSet>
<directory>./src/main/bin</directory>
<outputDirectory>bin</outputDirectory>
<includes>
<include>**/*</include>
</includes>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>./src/main/conf</directory>
<outputDirectory>/conf</outputDirectory>
<includes>
<include>**/*</include>
</includes>
</fileSet>
<fileSet>
<directory>./src/main/resources</directory>
<outputDirectory>/conf</outputDirectory>
<includes>
<include>**/*</include>
</includes>
</fileSet>
<fileSet>
<directory>target</directory>
<outputDirectory>logs</outputDirectory>
<excludes>
<exclude>**/*</exclude>
</excludes>
</fileSet>
<fileSet>
<directory>${project.basedir}/target/canal/plugin</directory>
<outputDirectory>/plugin/</outputDirectory>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<outputDirectory>lib</outputDirectory>
<excludes>
<exclude>junit:junit</exclude>
</excludes>
</dependencySet>
</dependencySets>
</assembly>

+ 15
- 0
smtweb-framework/canal/server/src/main/bin/restart.sh View File

@@ -0,0 +1,15 @@
#!/bin/bash

args=$@

case $(uname) in
Linux)
bin_abs_path=$(readlink -f $(dirname $0))
;;
*)
bin_abs_path=$(cd $(dirname $0) ||exit ; pwd)
;;
esac

sh "$bin_abs_path"/stop.sh $args
sh "$bin_abs_path"/startup.sh $args

+ 27
- 0
smtweb-framework/canal/server/src/main/bin/startup.bat View File

@@ -0,0 +1,27 @@
@echo off
@if not "%ECHO%" == "" echo %ECHO%
@if "%OS%" == "Windows_NT" setlocal

set ENV_PATH=.\
if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0%

set conf_dir=%ENV_PATH%\..\conf
set canal_conf=%conf_dir%\canal.properties
@rem set canal_conf=%conf_dir%\canal_local.properties
if "%1" == "local" set canal_conf=%conf_dir%\canal_local.properties
set logback_configurationFile=%conf_dir%\logback.xml

set CLASSPATH=%conf_dir%
set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH%

set JAVA_MEM_OPTS= -Xms128m -Xmx512m -XX:PermSize=128m
set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8
set JAVA_DEBUG_OPT= -server -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=9099,server=y,suspend=n
set CANAL_OPTS= -DappName=otter-canal -Dlogback.configurationFile="%logback_configurationFile%" -Dcanal.conf="%canal_conf%"

set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS%

set CMD_STR= java %JAVA_OPTS% -classpath "%CLASSPATH%" java %JAVA_OPTS% -classpath "%CLASSPATH%" CanalLauncher
echo start cmd : %CMD_STR%

java %JAVA_OPTS% -classpath "%CLASSPATH%" CanalLauncher

+ 126
- 0
smtweb-framework/canal/server/src/main/bin/startup.sh View File

@@ -0,0 +1,126 @@
#!/bin/bash

current_path=`pwd`
case "`uname`" in
Linux)
bin_abs_path=$(readlink -f $(dirname $0))
;;
*)
bin_abs_path=`cd $(dirname $0); pwd`
;;
esac
base=${bin_abs_path}/..
canal_conf=$base/conf/canal.properties
canal_local_conf=$base/conf/canal_local.properties
logback_configurationFile=$base/conf/logback.xml
export LANG=en_US.UTF-8
export BASE=$base

if [ -f $base/bin/canal.pid ] ; then
echo "found canal.pid , Please run stop.sh first ,then startup.sh" 2>&2
exit 1
fi

if [ ! -d $base/logs/canal ] ; then
mkdir -p $base/logs/canal
fi

## set java path
if [ -z "$JAVA" ] ; then
JAVA=$(which java)
fi

ALIBABA_JAVA="/usr/alibaba/java/bin/java"
TAOBAO_JAVA="/opt/taobao/java/bin/java"
if [ -z "$JAVA" ]; then
if [ -f $ALIBABA_JAVA ] ; then
JAVA=$ALIBABA_JAVA
elif [ -f $TAOBAO_JAVA ] ; then
JAVA=$TAOBAO_JAVA
else
echo "Cannot find a Java JDK. Please set either set JAVA or put java (>=1.5) in your PATH." 2>&2
exit 1
fi
fi

case "$#"
in
0 )
;;
1 )
var=$*
if [ "$var" = "local" ]; then
canal_conf=$canal_local_conf
else
if [ -f $var ] ; then
canal_conf=$var
else
echo "THE PARAMETER IS NOT CORRECT.PLEASE CHECK AGAIN."
exit
fi
fi;;
2 )
var=$1
if [ "$var" = "local" ]; then
canal_conf=$canal_local_conf
else
if [ -f $var ] ; then
canal_conf=$var
else
if [ "$1" = "debug" ]; then
DEBUG_PORT=$2
DEBUG_SUSPEND="n"
JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND"
fi
fi
fi;;
* )
echo "THE PARAMETERS MUST BE TWO OR LESS.PLEASE CHECK AGAIN."
exit;;
esac

JavaVersion=`$JAVA -version 2>&1 |awk 'NR==1{ gsub(/"/,""); print $3 }' | awk -F '.' '{print $1}'`
str=`file -L $JAVA | grep 64-bit`
JAVA_OPTS="$JAVA_OPTS -Xss256k -XX:+AggressiveOpts -XX:-UseBiasedLocking -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$base/logs"

if [ $JavaVersion -ge 11 ] ; then
#JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$base_log/gc.log:time "
JAVA_OPTS="$JAVA_OPTS"
else
#JAVA_OPTS="$JAVA_OPTS -Xloggc:$base/logs/canal/gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime"
JAVA_OPTS="$JAVA_OPTS -XX:+UseFastAccessorMethods -XX:+PrintAdaptiveSizePolicy -XX:+PrintTenuringDistribution"
fi

if [ -n "$str" ]; then
# JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
# For G1
JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
else
JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m $JAVA_OPTS"
fi

JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8"
CANAL_OPTS="-DappName=otter-canal -Dlogback.configurationFile=$logback_configurationFile -Dcanal.conf=$canal_conf"

if [ -e $canal_conf -a -e $logback_configurationFile ]
then
for i in $base/lib/*;
do CLASSPATH=$i:"$CLASSPATH";
done
CLASSPATH="$base/conf:$CLASSPATH";
echo "cd to $bin_abs_path for workaround relative path"
cd $bin_abs_path
echo LOG CONFIGURATION : $logback_configurationFile
echo canal conf : $canal_conf
echo CLASSPATH :$CLASSPATH
$JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.deployer.CanalLauncher 1>>$base/logs/canal/canal_stdout.log 2>&1 &
echo $! > $base/bin/canal.pid
echo "cd to $current_path for continue"
cd $current_path
else
echo "canal conf("$canal_conf") OR log configration file($logback_configurationFile) is not exist,please create then first!"
fi

+ 65
- 0
smtweb-framework/canal/server/src/main/bin/stop.sh View File

@@ -0,0 +1,65 @@
#!/bin/bash

cygwin=false;
linux=false;
case "`uname`" in
CYGWIN*)
cygwin=true
;;
Linux*)
linux=true
;;
esac

get_pid() {
STR=$1
PID=$2
if $cygwin; then
JAVA_CMD="$JAVA_HOME\bin\java"
JAVA_CMD=`cygpath --path --unix $JAVA_CMD`
JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'`
else
if $linux; then
if [ ! -z "$PID" ]; then
JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
else
JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'`
fi
else
if [ ! -z "$PID" ]; then
JAVA_PID=`ps aux |grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
else
JAVA_PID=`ps aux |grep "$STR"|grep -v grep|awk '{print $2}'`
fi
fi
fi
echo $JAVA_PID;
}

base=`dirname $0`/..
pidfile=$base/bin/canal.pid
if [ ! -f "$pidfile" ];then
echo "canal is not running. exists"
exit
fi

pid=`cat $pidfile`
if [ "$pid" == "" ] ; then
pid=`get_pid "appName=otter-canal"`
fi

echo -e "`hostname`: stopping canal $pid ... "
kill $pid

LOOPS=0
while (true);
do
gpid=`get_pid "appName=otter-canal" "$pid"`
if [ "$gpid" == "" ] ; then
echo "Oook! cost:$LOOPS"
`rm $pidfile`
break;
fi
let LOOPS=LOOPS+1
sleep 1
done

+ 13
- 0
smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalApplication.java View File

@@ -0,0 +1,13 @@
package cc.smtweb.system.canal.server;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication
public class CanalApplication {

public static void main(String[] args) {
SpringApplication.run(CanalApplication.class, args);
}

}

+ 97
- 0
smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalConstants.java View File

@@ -0,0 +1,97 @@
package cc.smtweb.system.canal.server;

import java.text.MessageFormat;

/**
* 启动常用变量
*
* @author jianghang 2012-11-8 下午03:15:55
* @version 1.0.0
*/
public class CanalConstants {

public static final String MDC_DESTINATION = "destination";
public static final String ROOT = "canal";
public static final String CANAL_ID = ROOT + "." + "id";
public static final String CANAL_IP = ROOT + "." + "ip";
public static final String CANAL_REGISTER_IP = ROOT + "." + "register.ip";
public static final String CANAL_PORT = ROOT + "." + "port";
public static final String CANAL_USER = ROOT + "." + "user";
public static final String CANAL_PASSWD = ROOT + "." + "passwd";
public static final String CANAL_METRICS_PULL_PORT = ROOT + "." + "metrics.pull.port";
public static final String CANAL_ADMIN_MANAGER = ROOT + "." + "admin.manager";
public static final String CANAL_ADMIN_PORT = ROOT + "." + "admin.port";
public static final String CANAL_ADMIN_USER = ROOT + "." + "admin.user";
public static final String CANAL_ADMIN_PASSWD = ROOT + "." + "admin.passwd";
public static final String CANAL_ADMIN_AUTO_REGISTER = ROOT + "." + "admin.register.auto";
public static final String CANAL_ADMIN_AUTO_CLUSTER = ROOT + "." + "admin.register.cluster";
public static final String CANAL_ADMIN_REGISTER_NAME = ROOT + "." + "admin.register.name";
public static final String CANAL_ZKSERVERS = ROOT + "." + "zkServers";
public static final String CANAL_WITHOUT_NETTY = ROOT + "." + "withoutNetty";

public static final String CANAL_DESTINATIONS = ROOT + "." + "destinations";
public static final String CANAL_AUTO_SCAN = ROOT + "." + "auto.scan";
public static final String CANAL_AUTO_SCAN_INTERVAL = ROOT + "." + "auto.scan.interval";
public static final String CANAL_CONF_DIR = ROOT + "." + "conf.dir";
public static final String CANAL_SERVER_MODE = ROOT + "." + "serverMode";

public static final String CANAL_DESTINATION_SPLIT = ",";
public static final String GLOBAL_NAME = "global";

public static final String INSTANCE_MODE_TEMPLATE = ROOT + "." + "instance.{0}.mode";
public static final String INSTANCE_LAZY_TEMPLATE = ROOT + "." + "instance.{0}.lazy";
public static final String INSTANCE_MANAGER_ADDRESS_TEMPLATE = ROOT + "." + "instance.{0}.manager.address";
public static final String INSTANCE_SPRING_XML_TEMPLATE = ROOT + "." + "instance.{0}.spring.xml";

public static final String CANAL_DESTINATION_PROPERTY = ROOT + ".instance.destination";

public static final String CANAL_SOCKETCHANNEL = ROOT + "." + "socketChannel";

public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey";
public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey";

// public static final String CANAL_MQ_SERVERS = ROOT + "." + "mq.servers";
// public static final String CANAL_MQ_RETRIES = ROOT + "." + "mq.retries";
// public static final String CANAL_MQ_BATCHSIZE = ROOT + "." + "mq.batchSize";
// public static final String CANAL_MQ_LINGERMS = ROOT + "." + "mq.lingerMs";
// public static final String CANAL_MQ_MAXREQUESTSIZE = ROOT + "." + "mq.maxRequestSize";
// public static final String CANAL_MQ_BUFFERMEMORY = ROOT + "." + "mq.bufferMemory";
// public static final String CANAL_MQ_CANALBATCHSIZE = ROOT + "." + "mq.canalBatchSize";
// public static final String CANAL_MQ_CANALGETTIMEOUT = ROOT + "." + "mq.canalGetTimeout";
// public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage";
// public static final String CANAL_MQ_PARALLELTHREADSIZE = ROOT + "." + "mq.parallelThreadSize";
// public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType";
// public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks";
// public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction";
// public static final String CANAL_MQ_PRODUCERGROUP = ROOT + "." + "mq.producerGroup";
// public static final String CANAL_MQ_PROPERTIES = ROOT + "." + "mq.properties";
// public static final String CANAL_MQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "mq.enableMessageTrace";
// public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.accessChannel";
// public static final String CANAL_MQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "mq.customizedTraceTopic";
// public static final String CANAL_MQ_NAMESPACE = ROOT + "." + "mq.namespace";
// public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable";
// public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH = ROOT + "." + "mq.kafka.kerberos.krb5FilePath";
// public static final String CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH = ROOT + "." + "mq.kafka.kerberos.jaasFilePath";
// public static final String CANAL_MQ_USERNAME = ROOT + "." + "mq.username";
// public static final String CANAL_MQ_PASSWORD = ROOT + "." + "mq.password";
// public static final String CANAL_MQ_VHOST = ROOT + "." + "mq.vhost";
// public static final String CANAL_MQ_ALIYUN_UID = ROOT + "." + "mq.aliyunuid";
// public static final String CANAL_MQ_EXCHANGE = ROOT + "." + "mq.exchange";
// public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash";

public static String getInstanceModeKey(String destination) {
return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination);
}

public static String getInstanceManagerAddressKey(String destination) {
return MessageFormat.format(INSTANCE_MANAGER_ADDRESS_TEMPLATE, destination);
}

public static String getInstancSpringXmlKey(String destination) {
return MessageFormat.format(INSTANCE_SPRING_XML_TEMPLATE, destination);
}

public static String getInstancLazyKey(String destination) {
return MessageFormat.format(INSTANCE_LAZY_TEMPLATE, destination);
}
}

+ 606
- 0
smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalController.java View File

@@ -0,0 +1,606 @@
package cc.smtweb.system.canal.server;

import java.util.Map;
import java.util.Properties;

import cc.smtweb.system.canal.server.monitor.InstanceAction;
import cc.smtweb.system.canal.server.monitor.InstanceConfigMonitor;
import cc.smtweb.system.canal.server.monitor.ManagerInstanceConfigMonitor;
import cc.smtweb.system.canal.server.monitor.SpringInstanceConfigMonitor;
import org.I0Itec.zkclient.IZkStateListener;
import org.I0Itec.zkclient.exception.ZkNoNodeException;
import org.I0Itec.zkclient.exception.ZkNodeExistsException;
import org.apache.commons.lang.BooleanUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.zookeeper.Watcher.Event.KeeperState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;

import com.alibaba.otter.canal.common.utils.AddressUtils;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningData;
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningListener;
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitor;
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitors;
import cc.smtweb.system.canal.server.InstanceConfig.InstanceMode;
import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
import com.alibaba.otter.canal.instance.manager.PlainCanalInstanceGenerator;
import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient;
import com.alibaba.otter.canal.instance.spring.SpringCanalInstanceGenerator;
import com.alibaba.otter.canal.server.CanalMQStarter;
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
import com.alibaba.otter.canal.server.exception.CanalServerException;
import com.alibaba.otter.canal.server.netty.CanalServerWithNetty;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;

/**
* canal调度控制器
*
* @author jianghang 2012-11-8 下午12:03:11
* @version 1.0.0
*/
public class CanalController {

private static final Logger logger = LoggerFactory.getLogger(CanalController.class);
private String ip;
private String registerIp;
private int port;
private int adminPort;
// 默认使用spring的方式载入
private Map<String, InstanceConfig> instanceConfigs;
private InstanceConfig globalInstanceConfig;
private Map<String, PlainCanalConfigClient> managerClients;
// 监听instance config的变化
private boolean autoScan = true;
private InstanceAction defaultAction;
private Map<InstanceMode, InstanceConfigMonitor> instanceConfigMonitors;
private CanalServerWithEmbedded embededCanalServer;
private CanalServerWithNetty canalServer;

private CanalInstanceGenerator instanceGenerator;
private ZkClientx zkclientx;

private CanalMQStarter canalMQStarter;
private String adminUser;
private String adminPasswd;

public CanalController(){
this(System.getProperties());
}

public CanalController(final Properties properties){
managerClients = MigrateMap.makeComputingMap(this::getManagerClient);

// 初始化全局参数设置
globalInstanceConfig = initGlobalConfig(properties);
instanceConfigs = new MapMaker().makeMap();
// 初始化instance config
initInstanceConfig(properties);

// init socketChannel
String socketChannel = getProperty(properties, CanalConstants.CANAL_SOCKETCHANNEL);
if (StringUtils.isNotEmpty(socketChannel)) {
System.setProperty(CanalConstants.CANAL_SOCKETCHANNEL, socketChannel);
}

// 兼容1.1.0版本的ak/sk参数名
String accesskey = getProperty(properties, "canal.instance.rds.accesskey");
String secretkey = getProperty(properties, "canal.instance.rds.secretkey");
if (StringUtils.isNotEmpty(accesskey)) {
System.setProperty(CanalConstants.CANAL_ALIYUN_ACCESSKEY, accesskey);
}
if (StringUtils.isNotEmpty(secretkey)) {
System.setProperty(CanalConstants.CANAL_ALIYUN_SECRETKEY, secretkey);
}

// 准备canal server
ip = getProperty(properties, CanalConstants.CANAL_IP);
registerIp = getProperty(properties, CanalConstants.CANAL_REGISTER_IP);
port = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_PORT, "11111"));
adminPort = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_ADMIN_PORT, "11110"));
embededCanalServer = CanalServerWithEmbedded.instance();
embededCanalServer.setCanalInstanceGenerator(instanceGenerator);// 设置自定义的instanceGenerator
int metricsPort = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_METRICS_PULL_PORT, "11112"));
embededCanalServer.setMetricsPort(metricsPort);

this.adminUser = getProperty(properties, CanalConstants.CANAL_ADMIN_USER);
this.adminPasswd = getProperty(properties, CanalConstants.CANAL_ADMIN_PASSWD);
embededCanalServer.setUser(getProperty(properties, CanalConstants.CANAL_USER));
embededCanalServer.setPasswd(getProperty(properties, CanalConstants.CANAL_PASSWD));

String canalWithoutNetty = getProperty(properties, CanalConstants.CANAL_WITHOUT_NETTY);
if (canalWithoutNetty == null || "false".equals(canalWithoutNetty)) {
canalServer = CanalServerWithNetty.instance();
canalServer.setIp(ip);
canalServer.setPort(port);
}

// 处理下ip为空,默认使用hostIp暴露到zk中
if (StringUtils.isEmpty(ip) && StringUtils.isEmpty(registerIp)) {
ip = registerIp = AddressUtils.getHostIp();
}

if (StringUtils.isEmpty(ip)) {
ip = AddressUtils.getHostIp();
}

if (StringUtils.isEmpty(registerIp)) {
registerIp = ip; // 兼容以前配置
}
final String zkServers = getProperty(properties, CanalConstants.CANAL_ZKSERVERS);
if (StringUtils.isNotEmpty(zkServers)) {
zkclientx = ZkClientx.getZkClient(zkServers);
// 初始化系统目录
zkclientx.createPersistent(ZookeeperPathUtils.DESTINATION_ROOT_NODE, true);
zkclientx.createPersistent(ZookeeperPathUtils.CANAL_CLUSTER_ROOT_NODE, true);
}

final ServerRunningData serverData = new ServerRunningData(registerIp + ":" + port);
ServerRunningMonitors.setServerData(serverData);
ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap((Function<String, ServerRunningMonitor>) destination -> {
ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
runningMonitor.setDestination(destination);
runningMonitor.setListener(new ServerRunningListener() {

public void processActiveEnter() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.start(destination);
if (canalMQStarter != null) {
canalMQStarter.startDestination(destination);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}

public void processActiveExit() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (canalMQStarter != null) {
canalMQStarter.stopDestination(destination);
}
embededCanalServer.stop(destination);
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}

public void processStart() {
try {
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
registerIp + ":" + port);
initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener() {

public void handleStateChanged(KeeperState state) throws Exception {

}

public void handleNewSession() throws Exception {
initCid(path);
}

@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}

public void processStop() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
registerIp + ":" + port);
releaseCid(path);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}

});
if (zkclientx != null) {
runningMonitor.setZkClient(zkclientx);
}
// 触发创建一下cid节点
runningMonitor.init();
return runningMonitor;
}));

// 初始化monitor机制
autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN));
if (autoScan) {
defaultAction = new InstanceAction() {

public void start(String destination) {
InstanceConfig config = instanceConfigs.get(destination);
if (config == null) {
// 重新读取一下instance config
config = parseInstanceConfig(properties, destination);
instanceConfigs.put(destination, config);
}

if (!embededCanalServer.isStart(destination)) {
// HA机制启动
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
if (!config.getLazy() && !runningMonitor.isStart()) {
runningMonitor.start();
}
}

logger.info("auto notify start {} successful.", destination);
}

public void stop(String destination) {
// 此处的stop,代表强制退出,非HA机制,所以需要退出HA的monitor和配置信息
InstanceConfig config = instanceConfigs.remove(destination);
if (config != null) {
embededCanalServer.stop(destination);
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
if (runningMonitor.isStart()) {
runningMonitor.stop();
}
}

logger.info("auto notify stop {} successful.", destination);
}

public void reload(String destination) {
// 目前任何配置变化,直接重启,简单处理
stop(destination);
start(destination);

logger.info("auto notify reload {} successful.", destination);
}

@Override
public void release(String destination) {
// 此处的release,代表强制释放,主要针对HA机制释放运行,让给其他机器抢占
InstanceConfig config = instanceConfigs.get(destination);
if (config != null) {
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
if (runningMonitor.isStart()) {
boolean release = runningMonitor.release();
if (!release) {
// 如果是单机模式,则直接清除配置
instanceConfigs.remove(destination);
// 停掉服务
runningMonitor.stop();
if (instanceConfigMonitors.containsKey(InstanceConfig.InstanceMode.MANAGER)) {
ManagerInstanceConfigMonitor monitor = (ManagerInstanceConfigMonitor) instanceConfigMonitors.get(InstanceConfig.InstanceMode.MANAGER);
Map<String, InstanceAction> instanceActions = monitor.getActions();
if (instanceActions.containsKey(destination)) {
// 清除内存中的autoScan cache
monitor.release(destination);
}
}
}
}
}

logger.info("auto notify release {} successful.", destination);
}
};

instanceConfigMonitors = MigrateMap.makeComputingMap(mode -> {
int scanInterval = Integer.valueOf(getProperty(properties,
CanalConstants.CANAL_AUTO_SCAN_INTERVAL,
"5"));

if (mode.isSpring()) {
SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor();
monitor.setScanIntervalInSecond(scanInterval);
monitor.setDefaultAction(defaultAction);
// 设置conf目录,默认是user.dir + conf目录组成
String rootDir = getProperty(properties, CanalConstants.CANAL_CONF_DIR);
if (StringUtils.isEmpty(rootDir)) {
rootDir = "../conf";
}

if (StringUtils.equals("otter-canal", System.getProperty("appName"))) {
monitor.setRootConf(rootDir);
} else {
// eclipse debug模式
monitor.setRootConf("src/main/resources/");
}
return monitor;
} else if (mode.isManager()) {
ManagerInstanceConfigMonitor monitor = new ManagerInstanceConfigMonitor();
monitor.setScanIntervalInSecond(scanInterval);
monitor.setDefaultAction(defaultAction);
String managerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER);
monitor.setConfigClient(getManagerClient(managerAddress));
return monitor;
} else {
throw new UnsupportedOperationException("unknow mode :" + mode + " for monitor");
}
});
}
}

private InstanceConfig initGlobalConfig(Properties properties) {
String adminManagerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER);
InstanceConfig globalConfig = new InstanceConfig();
String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(CanalConstants.GLOBAL_NAME));
if (StringUtils.isNotEmpty(adminManagerAddress)) {
// 如果指定了manager地址,则强制适用manager
globalConfig.setMode(InstanceMode.MANAGER);
} else if (StringUtils.isNotEmpty(modeStr)) {
globalConfig.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr)));
}

String lazyStr = getProperty(properties, CanalConstants.getInstancLazyKey(CanalConstants.GLOBAL_NAME));
if (StringUtils.isNotEmpty(lazyStr)) {
globalConfig.setLazy(Boolean.valueOf(lazyStr));
}

String managerAddress = getProperty(properties,
CanalConstants.getInstanceManagerAddressKey(CanalConstants.GLOBAL_NAME));
if (StringUtils.isNotEmpty(managerAddress)) {
if (StringUtils.equals(managerAddress, "${canal.admin.manager}")) {
managerAddress = adminManagerAddress;
}

globalConfig.setManagerAddress(managerAddress);
}

String springXml = getProperty(properties, CanalConstants.getInstancSpringXmlKey(CanalConstants.GLOBAL_NAME));
if (StringUtils.isNotEmpty(springXml)) {
globalConfig.setSpringXml(springXml);
}

instanceGenerator = destination -> {
InstanceConfig config = instanceConfigs.get(destination);
if (config == null) {
throw new CanalServerException("can't find destination:" + destination);
}

if (config.getMode().isManager()) {
PlainCanalInstanceGenerator instanceGenerator = new PlainCanalInstanceGenerator(properties);
instanceGenerator.setCanalConfigClient(managerClients.get(config.getManagerAddress()));
instanceGenerator.setSpringXml(config.getSpringXml());
return instanceGenerator.generate(destination);
} else if (config.getMode().isSpring()) {
SpringCanalInstanceGenerator instanceGenerator = new SpringCanalInstanceGenerator();
instanceGenerator.setSpringXml(config.getSpringXml());
return instanceGenerator.generate(destination);
} else {
throw new UnsupportedOperationException("unknow mode :" + config.getMode());
}

};

return globalConfig;
}

private PlainCanalConfigClient getManagerClient(String managerAddress) {
return new PlainCanalConfigClient(managerAddress, this.adminUser, this.adminPasswd, this.registerIp, adminPort);
}

private void initInstanceConfig(Properties properties) {
String destinationStr = getProperty(properties, CanalConstants.CANAL_DESTINATIONS);
String[] destinations = StringUtils.split(destinationStr, CanalConstants.CANAL_DESTINATION_SPLIT);

for (String destination : destinations) {
InstanceConfig config = parseInstanceConfig(properties, destination);
InstanceConfig oldConfig = instanceConfigs.put(destination, config);

if (oldConfig != null) {
logger.warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config);
}
}
}

private InstanceConfig parseInstanceConfig(Properties properties, String destination) {
String adminManagerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER);
InstanceConfig config = new InstanceConfig(globalInstanceConfig);
String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(destination));
if (StringUtils.isNotEmpty(adminManagerAddress)) {
// 如果指定了manager地址,则强制适用manager
config.setMode(InstanceMode.MANAGER);
} else if (StringUtils.isNotEmpty(modeStr)) {
config.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr)));
}

String lazyStr = getProperty(properties, CanalConstants.getInstancLazyKey(destination));
if (!StringUtils.isEmpty(lazyStr)) {
config.setLazy(Boolean.valueOf(lazyStr));
}

if (config.getMode().isManager()) {
String managerAddress = getProperty(properties, CanalConstants.getInstanceManagerAddressKey(destination));
if (StringUtils.isNotEmpty(managerAddress)) {
if (StringUtils.equals(managerAddress, "${canal.admin.manager}")) {
managerAddress = adminManagerAddress;
}
config.setManagerAddress(managerAddress);
}
}

String springXml = getProperty(properties, CanalConstants.getInstancSpringXmlKey(destination));
if (StringUtils.isNotEmpty(springXml)) {
config.setSpringXml(springXml);
}

return config;
}

public static String getProperty(Properties properties, String key, String defaultValue) {
String value = getProperty(properties, key);
if (StringUtils.isEmpty(value)) {
return defaultValue;
} else {
return value;
}
}

public static String getProperty(Properties properties, String key) {
key = StringUtils.trim(key);
String value = System.getProperty(key);

if (value == null) {
value = System.getenv(key);
}

if (value == null) {
value = properties.getProperty(key);
}

return StringUtils.trim(value);
}

public void start() throws Throwable {
logger.info("## start the canal server[{}({}):{}]", ip, registerIp, port);
// 创建整个canal的工作节点
final String path = ZookeeperPathUtils.getCanalClusterNode(registerIp + ":" + port);
initCid(path);
if (zkclientx != null) {
this.zkclientx.subscribeStateChanges(new IZkStateListener() {

public void handleStateChanged(KeeperState state) throws Exception {

}

public void handleNewSession() throws Exception {
initCid(path);
}

@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
}
// 优先启动embeded服务
embededCanalServer.start();
// 尝试启动一下非lazy状态的通道
for (Map.Entry<String, InstanceConfig> entry : instanceConfigs.entrySet()) {
final String destination = entry.getKey();
InstanceConfig config = entry.getValue();
// 创建destination的工作节点
if (!embededCanalServer.isStart(destination)) {
// HA机制启动
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
if (!config.getLazy() && !runningMonitor.isStart()) {
runningMonitor.start();
}
}

if (autoScan) {
instanceConfigMonitors.get(config.getMode()).register(destination, defaultAction);
}
}

if (autoScan) {
instanceConfigMonitors.get(globalInstanceConfig.getMode()).start();
for (InstanceConfigMonitor monitor : instanceConfigMonitors.values()) {
if (!monitor.isStart()) {
monitor.start();
}
}
}

// 启动网络接口
if (canalServer != null) {
canalServer.start();
}
}

public void stop() throws Throwable {

if (canalServer != null) {
canalServer.stop();
}

if (autoScan) {
for (InstanceConfigMonitor monitor : instanceConfigMonitors.values()) {
if (monitor.isStart()) {
monitor.stop();
}
}
}

for (ServerRunningMonitor runningMonitor : ServerRunningMonitors.getRunningMonitors().values()) {
if (runningMonitor.isStart()) {
runningMonitor.stop();
}
}

// 释放canal的工作节点
releaseCid(ZookeeperPathUtils.getCanalClusterNode(registerIp + ":" + port));
logger.info("## stop the canal server[{}({}):{}]", ip, registerIp, port);

if (zkclientx != null) {
zkclientx.close();
}

// 关闭时清理缓存
if (instanceConfigs != null) {
instanceConfigs.clear();
}
if (managerClients != null) {
managerClients.clear();
}
if (instanceConfigMonitors != null) {
instanceConfigMonitors.clear();
}

ZkClientx.clearClients();
}

private void initCid(String path) {
// logger.info("## init the canalId = {}", cid);
// 初始化系统目录
if (zkclientx != null) {
try {
zkclientx.createEphemeral(path);
} catch (ZkNoNodeException e) {
// 如果父目录不存在,则创建
String parentDir = path.substring(0, path.lastIndexOf('/'));
zkclientx.createPersistent(parentDir, true);
zkclientx.createEphemeral(path);
} catch (ZkNodeExistsException e) {
// ignore
// 因为第一次启动时创建了cid,但在stop/start的时可能会关闭和新建,允许出现NodeExists问题s
}

}
}

private void releaseCid(String path) {
// logger.info("## release the canalId = {}", cid);
// 初始化系统目录
if (zkclientx != null) {
zkclientx.delete(path);
}
}

public CanalMQStarter getCanalMQStarter() {
return canalMQStarter;
}

public void setCanalMQStarter(CanalMQStarter canalMQStarter) {
this.canalMQStarter = canalMQStarter;
}

public Map<InstanceMode, InstanceConfigMonitor> getInstanceConfigMonitors() {
return instanceConfigMonitors;
}

public Map<String, InstanceConfig> getInstanceConfigs() {
return instanceConfigs;
}

}

+ 141
- 0
smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalLauncher.java View File

@@ -0,0 +1,141 @@
package cc.smtweb.system.canal.server;

import com.alibaba.otter.canal.common.utils.AddressUtils;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.alibaba.otter.canal.instance.manager.plain.PlainCanal;
import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient;
import org.apache.commons.lang.BooleanUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileInputStream;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

/**
* canal独立版本启动的入口类
*
* @author jianghang 2012-11-6 下午05:20:49
* @version 1.0.0
*/
public class CanalLauncher {

private static final String CLASSPATH_URL_PREFIX = "classpath:";
private static final Logger logger = LoggerFactory.getLogger(CanalLauncher.class);
public static final CountDownLatch runningLatch = new CountDownLatch(1);
private static ScheduledExecutorService executor = Executors.newScheduledThreadPool(1,
new NamedThreadFactory("canal-server-scan"));

public static void main(String[] args) {
startServer();
}

public static void startServer() {
try {
logger.info("## canal sever start begin");
logger.info("## set default uncaught exception handler");
setGlobalUncaughtExceptionHandler();

// 支持rocketmq client 配置日志路径
System.setProperty("rocketmq.client.logUseSlf4j", "true");

logger.info("## load canal configurations");
String conf = System.getProperty("canal.conf", "classpath:canal.properties");
Properties properties = new Properties();
if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
properties.load(CanalLauncher.class.getClassLoader().getResourceAsStream(conf));
} else {
properties.load(new FileInputStream(conf));
}

final CanalStarter canalStater = new CanalStarter(properties);
String managerAddress = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER);
if (StringUtils.isNotEmpty(managerAddress)) {
String user = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_USER);
String passwd = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PASSWD);
String adminPort = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PORT, "11110");
boolean autoRegister = BooleanUtils.toBoolean(CanalController.getProperty(properties,
CanalConstants.CANAL_ADMIN_AUTO_REGISTER));
String autoCluster = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_AUTO_CLUSTER);
String name = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_REGISTER_NAME);
if (StringUtils.isEmpty(name)) {
name = AddressUtils.getHostName();
}

String registerIp = CanalController.getProperty(properties, CanalConstants.CANAL_REGISTER_IP);
if (StringUtils.isEmpty(registerIp)) {
registerIp = AddressUtils.getHostIp();
}
final PlainCanalConfigClient configClient = new PlainCanalConfigClient(managerAddress,
user,
passwd,
registerIp,
Integer.parseInt(adminPort),
autoRegister,
autoCluster,
name);
PlainCanal canalConfig = configClient.findServer(null);
if (canalConfig == null) {
throw new IllegalArgumentException("managerAddress:" + managerAddress
+ " can't not found config for [" + registerIp + ":" + adminPort
+ "]");
}
Properties managerProperties = canalConfig.getProperties();
// merge local
managerProperties.putAll(properties);
int scanIntervalInSecond = Integer.valueOf(CanalController.getProperty(managerProperties,
CanalConstants.CANAL_AUTO_SCAN_INTERVAL,
"5"));
executor.scheduleWithFixedDelay(new Runnable() {

private PlainCanal lastCanalConfig;

public void run() {
try {
if (lastCanalConfig == null) {
lastCanalConfig = configClient.findServer(null);
} else {
PlainCanal newCanalConfig = configClient.findServer(lastCanalConfig.getMd5());
if (newCanalConfig != null) {
// 远程配置canal.properties修改重新加载整个应用
canalStater.stop();
Properties managerProperties = newCanalConfig.getProperties();
// merge local
managerProperties.putAll(properties);
canalStater.setProperties(managerProperties);
canalStater.start();

lastCanalConfig = newCanalConfig;
}
}

} catch (Throwable e) {
logger.error("scan failed", e);
}
}

}, 0, scanIntervalInSecond, TimeUnit.SECONDS);
canalStater.setProperties(managerProperties);
} else {
canalStater.setProperties(properties);
}

canalStater.start();
logger.info("## canal server start success");
runningLatch.await();
executor.shutdownNow();
} catch (Throwable e) {
logger.error("## Something goes wrong when starting up the canal Server:", e);
}
}

private static void setGlobalUncaughtExceptionHandler() {
Thread.setDefaultUncaughtExceptionHandler((t, e) -> logger.error("UnCaughtException", e));
}

}

+ 39
- 0
smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStartedListener.java View File

@@ -0,0 +1,39 @@
package cc.smtweb.system.canal.server;

import cc.smtweb.framework.core.annotation.SwStartListener;
import cc.smtweb.framework.core.common.SwConsts;
import cc.smtweb.framework.core.mvc.controller.IStartListener;
import cc.smtweb.framework.core.systask.SysThreadPool;
import cc.smtweb.framework.core.systask.SysThreadWorker;
import lombok.extern.slf4j.Slf4j;

/**
* @Author yaoq
* @Date 2022年09月06日 10:31
* @Description
*/
@Slf4j
@SwStartListener
public class CanalStartedListener implements IStartListener {

@Override
public int order() {
return SwConsts.DEFAULT_ORDER + 2;
}


@Override
public void init() {

}

@Override
public void run() {
SysThreadPool.getInstance().addTask(new SysThreadWorker("canal server") {
@Override
public void localWork() throws Exception {
CanalLauncher.startServer();
}
});
}
}

+ 169
- 0
smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/CanalStarter.java View File

@@ -0,0 +1,169 @@
package cc.smtweb.system.canal.server;

import java.util.Properties;

import com.alibaba.otter.canal.connector.core.config.MQProperties;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.alibaba.otter.canal.admin.netty.CanalAdminWithNetty;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
import cc.smtweb.system.canal.server.admin.CanalAdminController;
import com.alibaba.otter.canal.server.CanalMQStarter;

/**
* Canal server 启动类
*
* @author rewerma 2020-01-27
* @version 1.0.2
*/
public class CanalStarter {

private static final Logger logger = LoggerFactory.getLogger(CanalStarter.class);

private static final String CONNECTOR_SPI_DIR = "/plugin";
private static final String CONNECTOR_STANDBY_SPI_DIR = "/canal/plugin";

private CanalController controller = null;
private CanalMQProducer canalMQProducer = null;
private Thread shutdownThread = null;
private CanalMQStarter canalMQStarter = null;
private volatile Properties properties;
private volatile boolean running = false;

private CanalAdminWithNetty canalAdmin;

public CanalStarter(Properties properties){
this.properties = properties;
}

public boolean isRunning() {
return running;
}

public Properties getProperties() {
return properties;
}

public void setProperties(Properties properties) {
this.properties = properties;
}

public CanalController getController() {
return controller;
}

/**
* 启动方法
*
* @throws Throwable
*/
public synchronized void start() throws Throwable {
String serverMode = CanalController.getProperty(properties, CanalConstants.CANAL_SERVER_MODE);
if (!"tcp".equalsIgnoreCase(serverMode)) {
ExtensionLoader<CanalMQProducer> loader = ExtensionLoader.getExtensionLoader(CanalMQProducer.class);
canalMQProducer = loader
.getExtension(serverMode.toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR);
if (canalMQProducer != null) {
ClassLoader cl = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(canalMQProducer.getClass().getClassLoader());
canalMQProducer.init(properties);
Thread.currentThread().setContextClassLoader(cl);
}
}

if (canalMQProducer != null) {
MQProperties mqProperties = canalMQProducer.getMqProperties();
// disable netty
System.setProperty(CanalConstants.CANAL_WITHOUT_NETTY, "true");
if (mqProperties.isFlatMessage()) {
// 设置为raw避免ByteString->Entry的二次解析
System.setProperty("canal.instance.memory.rawEntry", "false");
}
}

logger.info("## start the canal server.");
controller = new CanalController(properties);
controller.start();
logger.info("## the canal server is running now ......");
shutdownThread = new Thread(() -> {
try {
logger.info("## stop the canal server");
controller.stop();
CanalLauncher.runningLatch.countDown();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping canal Server:", e);
} finally {
logger.info("## canal server is down.");
}
});
Runtime.getRuntime().addShutdownHook(shutdownThread);

if (canalMQProducer != null) {
canalMQStarter = new CanalMQStarter(canalMQProducer);
String destinations = CanalController.getProperty(properties, CanalConstants.CANAL_DESTINATIONS);
canalMQStarter.start(destinations);
controller.setCanalMQStarter(canalMQStarter);
}

// start canalAdmin
String port = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PORT);
if (canalAdmin == null && StringUtils.isNotEmpty(port)) {
String user = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_USER);
String passwd = CanalController.getProperty(properties, CanalConstants.CANAL_ADMIN_PASSWD);
CanalAdminController canalAdmin = new CanalAdminController(this);
canalAdmin.setUser(user);
canalAdmin.setPasswd(passwd);

String ip = CanalController.getProperty(properties, CanalConstants.CANAL_IP);

logger.debug("canal admin port:{}, canal admin user:{}, canal admin password: {}, canal ip:{}",
port,
user,
passwd,
ip);

CanalAdminWithNetty canalAdminWithNetty = CanalAdminWithNetty.instance();
canalAdminWithNetty.setCanalAdmin(canalAdmin);
canalAdminWithNetty.setPort(Integer.parseInt(port));
canalAdminWithNetty.setIp(ip);
canalAdminWithNetty.start();
this.canalAdmin = canalAdminWithNetty;
}

running = true;
}

public synchronized void stop() throws Throwable {
stop(false);
}

/**
* 销毁方法,远程配置变更时调用
*
* @throws Throwable
*/
public synchronized void stop(boolean stopByAdmin) throws Throwable {
if (!stopByAdmin && canalAdmin != null) {
canalAdmin.stop();
canalAdmin = null;
}

if (controller != null) {
controller.stop();
controller = null;
}
if (shutdownThread != null) {
Runtime.getRuntime().removeShutdownHook(shutdownThread);
shutdownThread = null;
}
if (canalMQProducer != null && canalMQStarter != null) {
canalMQStarter.destroy();
canalMQStarter = null;
canalMQProducer = null;
}
running = false;
}
}

+ 93
- 0
smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/InstanceConfig.java View File

@@ -0,0 +1,93 @@
package cc.smtweb.system.canal.server;

import org.apache.commons.lang.builder.ToStringBuilder;

import com.alibaba.otter.canal.common.utils.CanalToStringStyle;

/**
* 启动的相关配置
*
* @author jianghang 2012-11-8 下午02:50:54
* @version 1.0.0
*/
public class InstanceConfig {

private InstanceConfig globalConfig;
private InstanceMode mode;
private Boolean lazy;
private String managerAddress;
private String springXml;

public InstanceConfig(){

}

public InstanceConfig(InstanceConfig globalConfig){
this.globalConfig = globalConfig;
}

public static enum InstanceMode {
SPRING, MANAGER;

public boolean isSpring() {
return this == InstanceMode.SPRING;
}

public boolean isManager() {
return this == InstanceMode.MANAGER;
}
}

public Boolean getLazy() {
if (lazy == null && globalConfig != null) {
return globalConfig.getLazy();
} else {
return lazy;
}
}

public void setLazy(Boolean lazy) {
this.lazy = lazy;
}

public InstanceMode getMode() {
if (mode == null && globalConfig != null) {
return globalConfig.getMode();
} else {
return mode;
}
}

public void setMode(InstanceMode mode) {
this.mode = mode;
}

public String getManagerAddress() {
if (managerAddress == null && globalConfig != null) {
return globalConfig.getManagerAddress();
} else {
return managerAddress;
}
}

public void setManagerAddress(String managerAddress) {
this.managerAddress = managerAddress;
}

public String getSpringXml() {
if (springXml == null && globalConfig != null) {
return globalConfig.getSpringXml();
} else {
return springXml;
}
}

public void setSpringXml(String springXml) {
this.springXml = springXml;
}

public String toString() {
return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
}

}

+ 256
- 0
smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/admin/CanalAdminController.java View File

@@ -0,0 +1,256 @@
package cc.smtweb.system.canal.server.admin;

import java.io.File;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import cc.smtweb.system.canal.server.CanalStarter;
import cc.smtweb.system.canal.server.monitor.InstanceConfigMonitor;
import cc.smtweb.system.canal.server.monitor.ManagerInstanceConfigMonitor;
import cc.smtweb.system.canal.server.monitor.SpringInstanceConfigMonitor;
import cc.smtweb.system.canal.server.monitor.InstanceAction;
import org.apache.commons.io.filefilter.TrueFileFilter;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.alibaba.otter.canal.admin.CanalAdmin;
import com.alibaba.otter.canal.common.utils.FileUtils;
import cc.smtweb.system.canal.server.InstanceConfig;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.protocol.SecurityUtil;
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
import com.google.common.base.Joiner;

/**
* 提供canal admin的管理操作
*
* @author agapple 2019年8月24日 下午11:39:01
* @since 1.1.4
*/
public class CanalAdminController implements CanalAdmin {

private static final Logger logger = LoggerFactory.getLogger(CanalAdminController.class);
private String user;
private String passwd;
private CanalStarter canalStater;

public CanalAdminController(CanalStarter canalStater){
this.canalStater = canalStater;
}

@Override
public boolean check() {
return canalStater.isRunning();
}

@Override
public synchronized boolean start() {
try {
if (!canalStater.isRunning()) {
canalStater.start();
return true;
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return false;
}

@Override
public synchronized boolean stop() {
try {
if (canalStater.isRunning()) {
canalStater.stop(true);
return true;
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return false;
}

@Override
public synchronized boolean restart() {
stop();
return start();
}

@Override
public boolean auth(String user, String passwd, byte[] seed) {
// 如果user/passwd密码为空,则任何用户账户都能登录
if ((StringUtils.isEmpty(this.user) || StringUtils.equals(this.user, user))) {
if (StringUtils.isEmpty(this.passwd)) {
return true;
} else if (StringUtils.isEmpty(passwd)) {
// 如果server密码有配置,客户端密码为空,则拒绝
return false;
}

try {
byte[] passForClient = SecurityUtil.hexStr2Bytes(passwd);
return SecurityUtil.scrambleServerAuth(passForClient, SecurityUtil.hexStr2Bytes(this.passwd), seed);
} catch (NoSuchAlgorithmException e) {
return false;
}
}

return false;
}

@Override
public String getRunningInstances() {
try {
Map<String, CanalInstance> instances = CanalServerWithEmbedded.instance().getCanalInstances();
List<String> runningInstances = new ArrayList<>();
instances.forEach((destination, instance) -> {
if (instance.isStart()) {
runningInstances.add(destination);
}
});

return Joiner.on(",").join(runningInstances);
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return "";
}

@Override
public boolean checkInstance(String destination) {
Map<String, CanalInstance> instances = CanalServerWithEmbedded.instance().getCanalInstances();
if (instances == null || !instances.containsKey(destination)) {
return false;
} else {
CanalInstance instance = instances.get(destination);
return instance.isStart();
}
}

@Override
public boolean startInstance(String destination) {
try {
InstanceAction instanceAction = getInstanceAction(destination);
if (instanceAction != null) {
instanceAction.start(destination);
return true;
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return false;
}

@Override
public boolean stopInstance(String destination) {
try {
InstanceAction instanceAction = getInstanceAction(destination);
if (instanceAction != null) {
instanceAction.stop(destination);
return true;
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return false;
}

@Override
public boolean releaseInstance(String destination) {
try {
InstanceAction instanceAction = getInstanceAction(destination);
if (instanceAction != null) {
instanceAction.release(destination);
return true;
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return false;
}

@Override
public boolean restartInstance(String destination) {
try {
InstanceAction instanceAction = getInstanceAction(destination);
if (instanceAction != null) {
instanceAction.reload(destination);
return true;
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return false;
}

@Override
public String listCanalLog() {
Collection<File> files = org.apache.commons.io.FileUtils.listFiles(new File("../logs/canal/"),
TrueFileFilter.TRUE,
TrueFileFilter.TRUE);
List<String> names = files.stream().map(File::getName).collect(Collectors.toList());
return Joiner.on(",").join(names);
}

@Override
public String canalLog(int lines) {
return FileUtils.readFileFromOffset("../logs/canal/canal.log", lines, "UTF-8");
}

@Override
public String listInstanceLog(String destination) {
Collection<File> files = org.apache.commons.io.FileUtils.listFiles(new File("../logs/" + destination + "/"),
TrueFileFilter.TRUE,
TrueFileFilter.TRUE);
List<String> names = files.stream().map(File::getName).collect(Collectors.toList());
return Joiner.on(",").join(names);
}

@Override
public String instanceLog(String destination, String fileName, int lines) {
if (StringUtils.isEmpty(fileName)) {
fileName = destination + ".log";
}
return FileUtils.readFileFromOffset("../logs/" + destination + "/" + fileName, lines, "UTF-8");
}

private InstanceAction getInstanceAction(String destination) {
Map<InstanceConfig.InstanceMode, InstanceConfigMonitor> monitors = canalStater.getController()
.getInstanceConfigMonitors();

InstanceAction instanceAction = null;
if (monitors.containsKey(InstanceConfig.InstanceMode.SPRING)) {
SpringInstanceConfigMonitor monitor = (SpringInstanceConfigMonitor) monitors.get(InstanceConfig.InstanceMode.SPRING);
Map<String, InstanceAction> instanceActions = monitor.getActions();
instanceAction = instanceActions.get(destination);
}

if (instanceAction != null) {
return instanceAction;
}

if (monitors.containsKey(InstanceConfig.InstanceMode.MANAGER)) {
ManagerInstanceConfigMonitor monitor = (ManagerInstanceConfigMonitor) monitors.get(InstanceConfig.InstanceMode.MANAGER);
Map<String, InstanceAction> instanceActions = monitor.getActions();
instanceAction = instanceActions.get(destination);
}
return instanceAction;
}

public void setUser(String user) {
this.user = user;
}

public void setPasswd(String passwd) {
this.passwd = passwd;
}

public void setCanalStater(CanalStarter canalStater) {
this.canalStater = canalStater;
}

}

+ 30
- 0
smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceAction.java View File

@@ -0,0 +1,30 @@
package cc.smtweb.system.canal.server.monitor;

/**
* config配置变化
*
* @author jianghang 2013-2-18 下午01:19:29
* @version 1.0.1
*/
public interface InstanceAction {

/**
* 启动destination
*/
void start(String destination);

/**
* 主动释放destination运行
*/
void release(String destination);

/**
* 停止destination
*/
void stop(String destination);

/**
* 重载destination,可能需要stop,start操作,或者只是更新下内存配置
*/
void reload(String destination);
}

+ 16
- 0
smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/InstanceConfigMonitor.java View File

@@ -0,0 +1,16 @@
package cc.smtweb.system.canal.server.monitor;

import com.alibaba.otter.canal.common.CanalLifeCycle;

/**
* 监听instance file的文件变化,触发instance start/stop等操作
*
* @author jianghang 2013-2-6 下午06:19:56
* @version 1.0.1
*/
public interface InstanceConfigMonitor extends CanalLifeCycle {

void register(String destination, InstanceAction action);

void unregister(String destination);
}

+ 184
- 0
smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/ManagerInstanceConfigMonitor.java View File

@@ -0,0 +1,184 @@
package cc.smtweb.system.canal.server.monitor;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.common.CanalLifeCycle;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.alibaba.otter.canal.instance.manager.plain.PlainCanal;
import com.alibaba.otter.canal.instance.manager.plain.PlainCanalConfigClient;
import com.google.common.collect.Lists;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;

/**
* 基于manager配置的实现
*
* @author agapple 2019年8月26日 下午10:00:20
* @since 1.1.4
*/
public class ManagerInstanceConfigMonitor extends AbstractCanalLifeCycle implements InstanceConfigMonitor, CanalLifeCycle {

private static final Logger logger = LoggerFactory.getLogger(ManagerInstanceConfigMonitor.class);
private long scanIntervalInSecond = 5;
private InstanceAction defaultAction = null;
private Map<String, InstanceAction> actions = new MapMaker().makeMap();
private Map<String, PlainCanal> configs = MigrateMap.makeComputingMap(destination -> new PlainCanal());
private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1,
new NamedThreadFactory("canal-instance-scan"));

private volatile boolean isFirst = true;
private PlainCanalConfigClient configClient;

public void start() {
super.start();
executor.scheduleWithFixedDelay(() -> {
try {
scan();
if (isFirst) {
isFirst = false;
}
} catch (Throwable e) {
logger.error("scan failed", e);
}
}, 0, scanIntervalInSecond, TimeUnit.SECONDS);
}

public void stop() {
super.stop();
executor.shutdownNow();
actions.clear();
}

public void register(String destination, InstanceAction action) {
if (action != null) {
actions.put(destination, action);
} else {
actions.put(destination, defaultAction);
}
}

public void unregister(String destination) {
actions.remove(destination);
}

private void scan() {
String instances = configClient.findInstances(null);
if (instances == null) {
return;
}

final List<String> is = Lists.newArrayList(StringUtils.split(instances, ','));
List<String> start = new ArrayList<>();
List<String> stop = new ArrayList<>();
List<String> restart = new ArrayList<>();
for (String instance : is) {
if (!configs.containsKey(instance)) {
PlainCanal newPlainCanal = configClient.findInstance(instance, null);
if (newPlainCanal != null) {
configs.put(instance, newPlainCanal);
start.add(instance);
}
} else {
PlainCanal plainCanal = configs.get(instance);
PlainCanal newPlainCanal = configClient.findInstance(instance, plainCanal.getMd5());
if (newPlainCanal != null) {
// 配置有变化
restart.add(instance);
configs.put(instance, newPlainCanal);
}
}
}

configs.forEach((instance, plainCanal) -> {
if (!is.contains(instance)) {
stop.add(instance);
}
});

stop.forEach(instance -> {
notifyStop(instance);
});

restart.forEach(instance -> {
notifyReload(instance);
});

start.forEach(instance -> {
notifyStart(instance);
});

}

private void notifyStart(String destination) {
try {
defaultAction.start(destination);
actions.put(destination, defaultAction);
// 启动成功后记录配置文件信息
} catch (Throwable e) {
logger.error(String.format("scan add found[%s] but start failed", destination), e);
}
}

private void notifyStop(String destination) {
InstanceAction action = actions.remove(destination);
if (action != null) {
try {
action.stop(destination);
configs.remove(destination);
} catch (Throwable e) {
logger.error(String.format("scan delete found[%s] but stop failed", destination), e);
actions.put(destination, action);// 再重新加回去,下一次scan时再执行删除
}
}
}

private void notifyReload(String destination) {
InstanceAction action = actions.get(destination);
if (action != null) {
try {
action.reload(destination);
} catch (Throwable e) {
logger.error(String.format("scan reload found[%s] but reload failed", destination), e);
}
}
}

public void release(String destination) {
InstanceAction action = actions.remove(destination);
if (action != null) {
try {
configs.remove(destination);
} catch (Throwable e) {
logger.error(String.format("scan delete found[%s] but stop failed", destination), e);
actions.put(destination, action);// 再重新加回去,下一次scan时再执行删除
}
}
}

public void setDefaultAction(InstanceAction defaultAction) {
this.defaultAction = defaultAction;
}

public void setScanIntervalInSecond(long scanIntervalInSecond) {
this.scanIntervalInSecond = scanIntervalInSecond;
}

public void setConfigClient(PlainCanalConfigClient configClient) {
this.configClient = configClient;
}

public Map<String, InstanceAction> getActions() {
return actions;
}

}

+ 297
- 0
smtweb-framework/canal/server/src/main/java/cc/smtweb/system/canal/server/monitor/SpringInstanceConfigMonitor.java View File

@@ -0,0 +1,297 @@
package cc.smtweb.system.canal.server.monitor;

import java.io.File;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import org.springframework.util.CollectionUtils;

import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.common.CanalLifeCycle;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;

/**
* 监听基于spring配置的instance变化
*
* @author jianghang 2013-2-6 下午06:23:55
* @version 1.0.1
*/
public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implements InstanceConfigMonitor, CanalLifeCycle {

private static final Logger logger = LoggerFactory.getLogger(SpringInstanceConfigMonitor.class);
private String rootConf;
// 扫描周期,单位秒
private long scanIntervalInSecond = 5;
private InstanceAction defaultAction = null;
private Map<String, InstanceAction> actions = new MapMaker().makeMap();
private Map<String, InstanceConfigFiles> lastFiles = MigrateMap.makeComputingMap(InstanceConfigFiles::new);
private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1,
new NamedThreadFactory("canal-instance-scan"));

private volatile boolean isFirst = true;

public Map<String, InstanceAction> getActions() {
return actions;
}

public void start() {
super.start();
Assert.notNull(rootConf, "root conf dir is null!");

executor.scheduleWithFixedDelay(() -> {
try {
scan();
if (isFirst) {
isFirst = false;
}
} catch (Throwable e) {
logger.error("scan failed", e);
}
}, 0, scanIntervalInSecond, TimeUnit.SECONDS);
}

public void stop() {
super.stop();
executor.shutdownNow();
actions.clear();
lastFiles.clear();
}

public void register(String destination, InstanceAction action) {
if (action != null) {
actions.put(destination, action);
} else {
actions.put(destination, defaultAction);
}
}

public void unregister(String destination) {
actions.remove(destination);
}

public void setRootConf(String rootConf) {
this.rootConf = rootConf;
}

private void scan() {
File rootdir = new File(rootConf);
if (!rootdir.exists()) {
return;
}

File[] instanceDirs = rootdir.listFiles(pathname -> {
String filename = pathname.getName();
return pathname.isDirectory() && !"spring".equalsIgnoreCase(filename);
});

// 扫描目录的新增
Set<String> currentInstanceNames = new HashSet<>();

// 判断目录内文件的变化
for (File instanceDir : instanceDirs) {
String destination = instanceDir.getName();
currentInstanceNames.add(destination);
File[] instanceConfigs = instanceDir.listFiles((dir, name) -> {
// return !StringUtils.endsWithIgnoreCase(name, ".dat");
// 限制一下,只针对instance.properties文件,避免因为.svn或者其他生成的临时文件导致出现reload
return StringUtils.equalsIgnoreCase(name, "instance.properties");
});

if (!actions.containsKey(destination) && instanceConfigs.length > 0) {
// 存在合法的instance.properties,并且第一次添加时,进行启动操作
notifyStart(instanceDir, destination, instanceConfigs);
} else if (actions.containsKey(destination)) {
// 历史已经启动过
if (instanceConfigs.length == 0) { // 如果不存在合法的instance.properties
notifyStop(destination);
} else {
InstanceConfigFiles lastFile = lastFiles.get(destination);
// 历史启动过 所以配置文件信息必然存在
if (!isFirst && CollectionUtils.isEmpty(lastFile.getInstanceFiles())) {
logger.error("[{}] is started, but not found instance file info.", destination);
}

boolean hasChanged = judgeFileChanged(instanceConfigs, lastFile.getInstanceFiles());
// 通知变化
if (hasChanged) {
notifyReload(destination);
}

if (hasChanged || CollectionUtils.isEmpty(lastFile.getInstanceFiles())) {
// 更新内容
List<FileInfo> newFileInfo = new ArrayList<>();
for (File instanceConfig : instanceConfigs) {
newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified()));
}

lastFile.setInstanceFiles(newFileInfo);
}
}
}

}

// 判断目录是否删除
Set<String> deleteInstanceNames = new HashSet<>();
for (String destination : actions.keySet()) {
if (!currentInstanceNames.contains(destination)) {
deleteInstanceNames.add(destination);
}
}
for (String deleteInstanceName : deleteInstanceNames) {
notifyStop(deleteInstanceName);
}
}

private void notifyStart(File instanceDir, String destination, File[] instanceConfigs) {
try {
defaultAction.start(destination);
actions.put(destination, defaultAction);

// 启动成功后记录配置文件信息
InstanceConfigFiles lastFile = lastFiles.get(destination);
List<FileInfo> newFileInfo = new ArrayList<>();
for (File instanceConfig : instanceConfigs) {
newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified()));
}
lastFile.setInstanceFiles(newFileInfo);
} catch (Throwable e) {
logger.error(String.format("scan add found[%s] but start failed", destination), e);
}
}

private void notifyStop(String destination) {
InstanceAction action = actions.remove(destination);
if (action != null) {
try {
action.stop(destination);
lastFiles.remove(destination);
} catch (Throwable e) {
logger.error(String.format("scan delete found[%s] but stop failed", destination), e);
actions.put(destination, action);// 再重新加回去,下一次scan时再执行删除
}
}
}

private void notifyReload(String destination) {
InstanceAction action = actions.get(destination);
if (action != null) {
try {
action.reload(destination);
} catch (Throwable e) {
logger.error(String.format("scan reload found[%s] but reload failed", destination), e);
}
}
}

private boolean judgeFileChanged(File[] instanceConfigs, List<FileInfo> fileInfos) {
boolean hasChanged = false;
for (File instanceConfig : instanceConfigs) {
for (FileInfo fileInfo : fileInfos) {
if (instanceConfig.getName().equals(fileInfo.getName())) {
hasChanged |= (instanceConfig.lastModified() != fileInfo.getLastModified());
if (hasChanged) {
return hasChanged;
}
}
}
}

return hasChanged;
}

public void setDefaultAction(InstanceAction defaultAction) {
this.defaultAction = defaultAction;
}

public void setScanIntervalInSecond(long scanIntervalInSecond) {
this.scanIntervalInSecond = scanIntervalInSecond;
}

public static class InstanceConfigFiles {

private String destination; // instance
// name
private List<FileInfo> springFile = new ArrayList<>(); // spring的instance
// xml
private FileInfo rootFile; // canal.properties
private List<FileInfo> instanceFiles = new ArrayList<>(); // instance对应的配置

public InstanceConfigFiles(String destination){
this.destination = destination;
}

public String getDestination() {
return destination;
}

public void setDestination(String destination) {
this.destination = destination;
}

public List<FileInfo> getSpringFile() {
return springFile;
}

public void setSpringFile(List<FileInfo> springFile) {
this.springFile = springFile;
}

public FileInfo getRootFile() {
return rootFile;
}

public void setRootFile(FileInfo rootFile) {
this.rootFile = rootFile;
}

public List<FileInfo> getInstanceFiles() {
return instanceFiles;
}

public void setInstanceFiles(List<FileInfo> instanceFiles) {
this.instanceFiles = instanceFiles;
}

}

public static class FileInfo {

private String name;
private long lastModified = 0;

public FileInfo(String name, long lastModified){
this.name = name;
this.lastModified = lastModified;
}

public String getName() {
return name;
}

public void setName(String name) {
this.name = name;
}

public long getLastModified() {
return lastModified;
}

public void setLastModified(long lastModified) {
this.lastModified = lastModified;
}

}
}

+ 182
- 0
smtweb-framework/canal/server/src/main/resources/canal.properties View File

@@ -0,0 +1,182 @@
#################################################
######### common argument #############
#################################################
# tcp bind ip
canal.ip =
# register ip to zookeeper
canal.register.ip =
canal.port = 11111
canal.metrics.pull.port = 11112
# canal instance user/passwd
# canal.user = canal
# canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458

# canal admin config
#canal.admin.manager = 127.0.0.1:8089
canal.admin.port = 11110
canal.admin.user = admin
canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
# admin auto register
#canal.admin.register.auto = true
#canal.admin.register.cluster =
#canal.admin.register.name =

canal.zkServers =
# flush data to zk
canal.zookeeper.flush.period = 1000
canal.withoutNetty = false
# tcp, kafka, rocketMQ, rabbitMQ, pulsarMQ
canal.serverMode = tcp
# flush meta cursor/parse position to file
canal.file.data.dir = ${canal.conf.dir}
canal.file.flush.period = 1000
## memory store RingBuffer size, should be Math.pow(2,n)
canal.instance.memory.buffer.size = 16384
## memory store RingBuffer used memory unit size , default 1kb
canal.instance.memory.buffer.memunit = 1024
## meory store gets mode used MEMSIZE or ITEMSIZE
canal.instance.memory.batch.mode = MEMSIZE
canal.instance.memory.rawEntry = true

## detecing config
canal.instance.detecting.enable = false
#canal.instance.detecting.sql = insert into retl.xdual values(1,now()) on duplicate key update x=now()
canal.instance.detecting.sql = select 1
canal.instance.detecting.interval.time = 3
canal.instance.detecting.retry.threshold = 3
canal.instance.detecting.heartbeatHaEnable = false

# support maximum transaction size, more than the size of the transaction will be cut into multiple transactions delivery
canal.instance.transaction.size = 1024
# mysql fallback connected to new master should fallback times
canal.instance.fallbackIntervalInSeconds = 60

# network config
canal.instance.network.receiveBufferSize = 16384
canal.instance.network.sendBufferSize = 16384
canal.instance.network.soTimeout = 30

# binlog filter config
canal.instance.filter.druid.ddl = true
canal.instance.filter.query.dcl = false
canal.instance.filter.query.dml = false
canal.instance.filter.query.ddl = false
canal.instance.filter.table.error = false
canal.instance.filter.rows = false
canal.instance.filter.transaction.entry = false
canal.instance.filter.dml.insert = false
canal.instance.filter.dml.update = false
canal.instance.filter.dml.delete = false

# binlog format/image check
canal.instance.binlog.format = ROW,STATEMENT,MIXED
canal.instance.binlog.image = FULL,MINIMAL,NOBLOB

# binlog ddl isolation
canal.instance.get.ddl.isolation = false

# parallel parser config
canal.instance.parser.parallel = true
## concurrent thread number, default 60% available processors, suggest not to exceed Runtime.getRuntime().availableProcessors()
#canal.instance.parser.parallelThreadSize = 16
## disruptor ringbuffer size, must be power of 2
canal.instance.parser.parallelBufferSize = 256

# table meta tsdb info
canal.instance.tsdb.enable = true
canal.instance.tsdb.dir = ${canal.file.data.dir:../conf}/${canal.instance.destination:}
canal.instance.tsdb.url = jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
canal.instance.tsdb.dbUsername = canal
canal.instance.tsdb.dbPassword = canal
# dump snapshot interval, default 24 hour
canal.instance.tsdb.snapshot.interval = 24
# purge snapshot expire , default 360 hour(15 days)
canal.instance.tsdb.snapshot.expire = 360

#################################################
######### destinations #############
#################################################
canal.destinations = example
# conf root dir
canal.conf.dir = ../conf
# auto scan instance dir add/remove and start/stop instance
canal.auto.scan = true
canal.auto.scan.interval = 5
# set this value to 'true' means that when binlog pos not found, skip to latest.
# WARN: pls keep 'false' in production env, or if you know what you want.
canal.auto.reset.latest.pos.mode = false

canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml
#canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml

canal.instance.global.mode = spring
canal.instance.global.lazy = false
canal.instance.global.manager.address = ${canal.admin.manager}
#canal.instance.global.spring.xml = classpath:spring/memory-instance.xml
canal.instance.global.spring.xml = classpath:spring/file-instance.xml
#canal.instance.global.spring.xml = classpath:spring/default-instance.xml

##################################################
######### MQ Properties #############
##################################################
# aliyun ak/sk , support rds/mq
canal.aliyun.accessKey =
canal.aliyun.secretKey =
canal.aliyun.uid=

canal.mq.flatMessage = true
canal.mq.canalBatchSize = 50
canal.mq.canalGetTimeout = 100
# Set this value to "cloud", if you want open message trace feature in aliyun.
canal.mq.accessChannel = local

canal.mq.database.hash = true
canal.mq.send.thread.size = 30
canal.mq.build.thread.size = 8

##################################################
######### Kafka #############
##################################################
kafka.bootstrap.servers = 127.0.0.1:9092
kafka.acks = all
kafka.compression.type = none
kafka.batch.size = 16384
kafka.linger.ms = 1
kafka.max.request.size = 1048576
kafka.buffer.memory = 33554432
kafka.max.in.flight.requests.per.connection = 1
kafka.retries = 0

kafka.kerberos.enable = false
kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"

##################################################
######### RocketMQ #############
##################################################
rocketmq.producer.group = test
rocketmq.enable.message.trace = false
rocketmq.customized.trace.topic =
rocketmq.namespace =
rocketmq.namesrv.addr = 127.0.0.1:9876
rocketmq.retry.times.when.send.failed = 0
rocketmq.vip.channel.enabled = false
rocketmq.tag =

##################################################
######### RabbitMQ #############
##################################################
rabbitmq.host =
rabbitmq.virtual.host =
rabbitmq.exchange =
rabbitmq.username =
rabbitmq.password =
rabbitmq.deliveryMode =


##################################################
######### Pulsar #############
##################################################
pulsarmq.serverUrl =
pulsarmq.roleToken =
pulsarmq.topicTenantPrefix =

+ 12
- 0
smtweb-framework/canal/server/src/main/resources/canal_local.properties View File

@@ -0,0 +1,12 @@
# register ip
canal.register.ip =

# canal admin config
canal.admin.manager = 127.0.0.1:8089
canal.admin.port = 11110
canal.admin.user = admin
canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
# admin auto register
canal.admin.register.auto = true
canal.admin.register.cluster =
canal.admin.register.name =

+ 59
- 0
smtweb-framework/canal/server/src/main/resources/example/instance.properties View File

@@ -0,0 +1,59 @@
#################################################
## mysql serverId , v1.0.26+ will autoGen
canal.instance.mysql.slaveId=2

# enable gtid use true/false
canal.instance.gtidon=false

# position info
canal.instance.master.address=172.28.123.25:3306
#canal.instance.master.journal.name=
#canal.instance.master.position=
#canal.instance.master.timestamp=
#canal.instance.master.gtid=

# rds oss binlog
canal.instance.rds.accesskey=
canal.instance.rds.secretkey=
canal.instance.rds.instanceId=

# table meta tsdb info
canal.instance.tsdb.enable=true
#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
#canal.instance.tsdb.dbUsername=canal
#canal.instance.tsdb.dbPassword=canal

#canal.instance.standby.address =
#canal.instance.standby.journal.name =
#canal.instance.standby.position =
#canal.instance.standby.timestamp =
#canal.instance.standby.gtid=

# username/password
canal.instance.dbUsername=root
canal.instance.dbPassword=root
canal.instance.connectionCharset = UTF-8
# enable druid Decrypt database password
canal.instance.enableDruid=false
#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==

# table regex
canal.instance.filter.regex=scmz\\..*
# table black regex
canal.instance.filter.black.regex=mysql\\.slave_.*
# table field filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)
#canal.instance.filter.field=test1.t_product:id/subject/keywords,test2.t_company:id/name/contact/ch
# table field black filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)
#canal.instance.filter.black.field=test1.t_product:subject/product_image,test2.t_company:id/name/contact/ch

# mq config
canal.mq.topic=example
# dynamic topic route by schema or table regex
#canal.mq.dynamicTopic=mytest1.user,topic2:mytest2\\..*,.*\\..*
canal.mq.partition=0
# hash partition config
#canal.mq.enableDynamicQueuePartition=false
#canal.mq.partitionsNum=3
#canal.mq.dynamicTopicPartitionNum=test.*:4,mycanal:6
#canal.mq.partitionHash=test.table:id^name,.*\\..*
#################################################

+ 117
- 0
smtweb-framework/canal/server/src/main/resources/logback.xml.bak View File

@@ -0,0 +1,117 @@
<configuration scan="true" scanPeriod=" 5 seconds">
<jmxConfigurator />
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
</pattern>
</encoder>
</appender>

<appender name="CANAL-ROOT" class="ch.qos.logback.classic.sift.SiftingAppender">
<discriminator>
<Key>destination</Key>
<DefaultValue>canal</DefaultValue>
</discriminator>
<sift>
<appender name="FILE-${destination}" class="ch.qos.logback.core.rolling.RollingFileAppender">
<File>../logs/${destination}/${destination}.log</File>
<rollingPolicy
class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- rollover daily -->
<fileNamePattern>../logs/${destination}/%d{yyyy-MM-dd}/${destination}-%d{yyyy-MM-dd}-%i.log.gz</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- or whenever the file size reaches 100MB -->
<maxFileSize>512MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>60</maxHistory>
</rollingPolicy>
<encoder>
<pattern>
%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
</pattern>
</encoder>
</appender>
</sift>
</appender>

<appender name="CANAL-META" class="ch.qos.logback.classic.sift.SiftingAppender">
<discriminator>
<Key>destination</Key>
<DefaultValue>canal</DefaultValue>
</discriminator>
<sift>
<appender name="META-FILE-${destination}" class="ch.qos.logback.core.rolling.RollingFileAppender">
<File>../logs/${destination}/meta.log</File>
<rollingPolicy
class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- rollover daily -->
<fileNamePattern>../logs/${destination}/%d{yyyy-MM-dd}/meta-%d{yyyy-MM-dd}-%i.log.gz</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- or whenever the file size reaches 100MB -->
<maxFileSize>32MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>60</maxHistory>
</rollingPolicy>
<encoder>
<pattern>
%d{yyyy-MM-dd HH:mm:ss.SSS} - %msg%n
</pattern>
</encoder>
</appender>
</sift>
</appender>

<appender name="RocketmqClientAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>../logs/canal/rocketmq_client.log</file>
<rollingPolicy
class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- rollover daily -->
<fileNamePattern>../logs/canal/%d{yyyy-MM-dd}/rocketmq_client-%d{yyyy-MM-dd}-%i.log.gz</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- or whenever the file size reaches 100MB -->
<maxFileSize>512MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>60</maxHistory>
</rollingPolicy>
<encoder charset="UTF-8">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n</pattern>
</encoder>
</appender>

<logger name="com.alibaba.otter.canal.instance" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-ROOT" />
</logger>
<logger name="com.alibaba.otter.canal.deployer" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-ROOT" />
</logger>
<logger name="com.alibaba.otter.canal.meta.FileMixedMetaManager" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-META" />
</logger>
<logger name="com.alibaba.otter.canal.connector.kafka" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-ROOT" />
</logger>
<logger name="com.alibaba.otter.canal.connector.rocketmq" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-ROOT" />
</logger>
<logger name="com.alibaba.otter.canal.connector.rabbitmq" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-ROOT" />
</logger>
<logger name="RocketmqClient" additivity="false">
<level value="INFO" />
<appender-ref ref="RocketmqClientAppender" />
</logger>
<logger name="com.alibaba.otter.canal.connector.pulsarmq" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-ROOT" />
</logger>
<root level="WARN">
<!-- <appender-ref ref="STDOUT"/> -->
<appender-ref ref="CANAL-ROOT" />
</root>
</configuration>

+ 1375
- 0
smtweb-framework/canal/server/src/main/resources/metrics/Canal_instances_tmpl.json
File diff suppressed because it is too large
View File


+ 39
- 0
smtweb-framework/canal/server/src/main/resources/spring/base-instance.xml View File

@@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">

<!-- properties -->
<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
<property name="ignoreResourceNotFound" value="true" />
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
<property name="locationNames">
<list>
<value>classpath:canal.properties</value>
<value>classpath:${canal.instance.destination:}/instance.properties</value>
</list>
</property>
</bean>
<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer">
<property name="propertyEditorRegistrars">
<list>
<ref bean="socketAddressEditor" />
</list>
</property>
</bean>
<bean id="baseEventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogEventParserProxy" abstract="true">
<property name="accesskey" value="${canal.aliyun.accesskey:}" />
<property name="secretkey" value="${canal.aliyun.secretkey:}" />
<property name="instanceId" value="${canal.instance.rds.instanceId:}" />
</bean>
</beans>

+ 211
- 0
smtweb-framework/canal/server/src/main/resources/spring/default-instance.xml View File

@@ -0,0 +1,211 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">

<import resource="classpath:spring/base-instance.xml" />

<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
<property name="destination" value="${canal.instance.destination}" />
<property name="eventParser">
<ref bean="eventParser" />
</property>
<property name="eventSink">
<ref bean="eventSink" />
</property>
<property name="eventStore">
<ref bean="eventStore" />
</property>
<property name="metaManager">
<ref bean="metaManager" />
</property>
<property name="alarmHandler">
<ref bean="alarmHandler" />
</property>
<property name="mqConfig">
<ref bean="mqConfig" />
</property>
</bean>

<!-- 报警处理类 -->
<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.LogAlarmHandler" />

<bean id="zkClientx" class="org.springframework.beans.factory.config.MethodInvokingFactoryBean" >
<property name="targetClass" value="com.alibaba.otter.canal.common.zookeeper.ZkClientx" />
<property name="targetMethod" value="getZkClient" />
<property name="arguments">
<list>
<value>${canal.zkServers:127.0.0.1:2181}</value>
</list>
</property>
</bean>

<bean id="metaManager" class="com.alibaba.otter.canal.meta.PeriodMixedMetaManager">
<property name="zooKeeperMetaManager">
<bean class="com.alibaba.otter.canal.meta.ZooKeeperMetaManager">
<property name="zkClientx" ref="zkClientx" />
</bean>
</property>
<property name="period" value="${canal.zookeeper.flush.period:1000}" />
</bean>

<bean id="eventStore" class="com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer">
<property name="bufferSize" value="${canal.instance.memory.buffer.size:16384}" />
<property name="bufferMemUnit" value="${canal.instance.memory.buffer.memunit:1024}" />
<property name="batchMode" value="${canal.instance.memory.batch.mode:MEMSIZE}" />
<property name="ddlIsolation" value="${canal.instance.get.ddl.isolation:false}" />
<property name="raw" value="${canal.instance.memory.rawEntry:true}" />
</bean>

<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
</bean>

<bean id="eventParser" parent="baseEventParser" >
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
<property name="detectingIntervalInSeconds" value="${canal.instance.detecting.interval.time:5}" />
<property name="haController">
<bean class="com.alibaba.otter.canal.parse.ha.HeartBeatHAController">
<property name="detectingRetryTimes" value="${canal.instance.detecting.retry.threshold:3}" />
<property name="switchEnable" value="${canal.instance.detecting.heartbeatHaEnable:false}" />
</bean>
</property>

<property name="alarmHandler" ref="alarmHandler" />

<!-- 解析过滤处理 -->
<property name="eventFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
</bean>
</property>

<property name="eventBlackFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.black.regex:}" />
<constructor-arg index="1" value="false" />
</bean>
</property>
<property name="fieldFilter" value="${canal.instance.filter.field}" />
<property name="fieldBlackFilter" value="${canal.instance.filter.black.field}" />

<!-- 最大事务解析大小,超过该大小后事务将被切分为多个事务投递 -->
<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />

<!-- 网络链接参数 -->
<property name="receiveBufferSize" value="${canal.instance.network.receiveBufferSize:16384}" />
<property name="sendBufferSize" value="${canal.instance.network.sendBufferSize:16384}" />
<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />

<!-- 解析编码 -->
<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />

<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</constructor-arg>
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
<constructor-arg ref="metaManager"/>
</bean>
</constructor-arg>
</bean>
</property>

<!-- failover切换时回退的时间 -->
<property name="fallbackIntervalInSeconds" value="${canal.instance.fallbackIntervalInSeconds:60}" />

<!-- 解析数据库信息 -->
<property name="masterInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.master.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>
<property name="standbyInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.standby.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>

<!-- 解析起始位点 -->
<property name="masterPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.master.journal.name}" />
<property name="position" value="${canal.instance.master.position}" />
<property name="timestamp" value="${canal.instance.master.timestamp}" />
<property name="gtid" value="${canal.instance.master.gtid}" />
</bean>
</property>
<property name="standbyPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.standby.journal.name}" />
<property name="position" value="${canal.instance.standby.position}" />
<property name="timestamp" value="${canal.instance.standby.timestamp}" />
<property name="gtid" value="${canal.instance.standby.gtid}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
<property name="filterDmlInsert" value="${canal.instance.filter.dml.insert:false}" />
<property name="filterDmlUpdate" value="${canal.instance.filter.dml.update:false}" />
<property name="filterDmlDelete" value="${canal.instance.filter.dml.delete:false}" />
<property name="filterRows" value="${canal.instance.filter.rows:false}" />
<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />

<!--表结构相关-->
<property name="enableTsdb" value="${canal.instance.tsdb.enable:true}"/>
<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
<property name="tsdbSnapshotInterval" value="${canal.instance.tsdb.snapshot.interval:24}" />
<property name="tsdbSnapshotExpire" value="${canal.instance.tsdb.snapshot.expire:360}" />

<!--是否启用GTID模式-->
<property name="isGTIDMode" value="${canal.instance.gtidon:false}"/>

<!-- parallel parser -->
<property name="parallel" value="${canal.instance.parser.parallel:true}" />
<property name="parallelThreadSize" value="${canal.instance.parser.parallelThreadSize}" />
<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />

<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
</bean>

<bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">
<property name="topic" value="${canal.mq.topic}" />
<property name="dynamicTopic" value="${canal.mq.dynamicTopic}" />
<property name="partition" value="${canal.mq.partition}" />
<property name="partitionsNum" value="${canal.mq.partitionsNum}" />
<property name="partitionHash" value="${canal.mq.partitionHash}" />
<property name="dynamicTopicPartitionNum" value="${canal.mq.dynamicTopicPartitionNum}" />
<property name="enableDynamicQueuePartition" value="${canal.mq.enableDynamicQueuePartition}" />
</bean>
</beans>

+ 197
- 0
smtweb-framework/canal/server/src/main/resources/spring/file-instance.xml View File

@@ -0,0 +1,197 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">

<import resource="classpath:spring/base-instance.xml" />

<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
<property name="destination" value="${canal.instance.destination}" />
<property name="eventParser">
<ref bean="eventParser" />
</property>
<property name="eventSink">
<ref bean="eventSink" />
</property>
<property name="eventStore">
<ref bean="eventStore" />
</property>
<property name="metaManager">
<ref bean="metaManager" />
</property>
<property name="alarmHandler">
<ref bean="alarmHandler" />
</property>
<property name="mqConfig">
<ref bean="mqConfig" />
</property>
</bean>

<!-- 报警处理类 -->
<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.LogAlarmHandler" />

<bean id="metaManager" class="com.alibaba.otter.canal.meta.FileMixedMetaManager">
<property name="dataDir" value="${canal.file.data.dir:../conf}" />
<property name="period" value="${canal.file.flush.period:1000}" />
</bean>

<bean id="eventStore" class="com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer">
<property name="bufferSize" value="${canal.instance.memory.buffer.size:16384}" />
<property name="bufferMemUnit" value="${canal.instance.memory.buffer.memunit:1024}" />
<property name="batchMode" value="${canal.instance.memory.batch.mode:MEMSIZE}" />
<property name="ddlIsolation" value="${canal.instance.get.ddl.isolation:false}" />
<property name="raw" value="${canal.instance.memory.rawEntry:true}" />
</bean>

<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
</bean>

<bean id="eventParser" parent="baseEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
<property name="detectingIntervalInSeconds" value="${canal.instance.detecting.interval.time:5}" />
<property name="haController">
<bean class="com.alibaba.otter.canal.parse.ha.HeartBeatHAController">
<property name="detectingRetryTimes" value="${canal.instance.detecting.retry.threshold:3}" />
<property name="switchEnable" value="${canal.instance.detecting.heartbeatHaEnable:false}" />
</bean>
</property>

<property name="alarmHandler" ref="alarmHandler" />

<!-- 解析过滤处理 -->
<property name="eventFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
</bean>
</property>

<property name="eventBlackFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.black.regex:}" />
<constructor-arg index="1" value="false" />
</bean>
</property>
<property name="fieldFilter" value="${canal.instance.filter.field}" />
<property name="fieldBlackFilter" value="${canal.instance.filter.black.field}" />
<!-- 最大事务解析大小,超过该大小后事务将被切分为多个事务投递 -->
<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />

<!-- 网络链接参数 -->
<property name="receiveBufferSize" value="${canal.instance.network.receiveBufferSize:16384}" />
<property name="sendBufferSize" value="${canal.instance.network.sendBufferSize:16384}" />
<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />

<!-- 解析编码 -->
<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />

<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</constructor-arg>
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
<constructor-arg ref="metaManager"/>
</bean>
</constructor-arg>
</bean>
</property>

<!-- failover切换时回退的时间 -->
<property name="fallbackIntervalInSeconds" value="${canal.instance.fallbackIntervalInSeconds:60}" />

<!-- 解析数据库信息 -->
<property name="masterInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.master.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>
<property name="standbyInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.standby.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>

<!-- 解析起始位点 -->
<property name="masterPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.master.journal.name}" />
<property name="position" value="${canal.instance.master.position}" />
<property name="timestamp" value="${canal.instance.master.timestamp}" />
<property name="gtid" value="${canal.instance.master.gtid}" />
</bean>
</property>
<property name="standbyPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.standby.journal.name}" />
<property name="position" value="${canal.instance.standby.position}" />
<property name="timestamp" value="${canal.instance.standby.timestamp}" />
<property name="gtid" value="${canal.instance.standby.gtid}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
<property name="filterDmlInsert" value="${canal.instance.filter.dml.insert:false}" />
<property name="filterDmlUpdate" value="${canal.instance.filter.dml.update:false}" />
<property name="filterDmlDelete" value="${canal.instance.filter.dml.delete:false}" />
<property name="filterRows" value="${canal.instance.filter.rows:false}" />
<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />

<!--表结构相关-->
<property name="enableTsdb" value="${canal.instance.tsdb.enable:true}"/>
<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
<property name="tsdbSnapshotInterval" value="${canal.instance.tsdb.snapshot.interval:24}" />
<property name="tsdbSnapshotExpire" value="${canal.instance.tsdb.snapshot.expire:360}" />

<!--是否启用GTID模式-->
<property name="isGTIDMode" value="${canal.instance.gtidon:false}"/>

<!-- parallel parser -->
<property name="parallel" value="${canal.instance.parser.parallel:true}" />
<property name="parallelThreadSize" value="${canal.instance.parser.parallelThreadSize}" />
<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />

<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
</bean>

<bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">
<property name="topic" value="${canal.mq.topic}" />
<property name="dynamicTopic" value="${canal.mq.dynamicTopic}" />
<property name="partition" value="${canal.mq.partition}" />
<property name="partitionsNum" value="${canal.mq.partitionsNum}" />
<property name="partitionHash" value="${canal.mq.partitionHash}" />
<property name="dynamicTopicPartitionNum" value="${canal.mq.dynamicTopicPartitionNum}" />
<property name="enableDynamicQueuePartition" value="${canal.mq.enableDynamicQueuePartition}" />
</bean>
</beans>

+ 292
- 0
smtweb-framework/canal/server/src/main/resources/spring/group-instance.xml View File

@@ -0,0 +1,292 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">

<import resource="classpath:spring/base-instance.xml" />

<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
<property name="destination" value="${canal.instance.destination}" />
<property name="eventParser">
<ref bean="eventParser" />
</property>
<property name="eventSink">
<ref bean="eventSink" />
</property>
<property name="eventStore">
<ref bean="eventStore" />
</property>
<property name="metaManager">
<ref bean="metaManager" />
</property>
<property name="alarmHandler">
<ref bean="alarmHandler" />
</property>
<property name="mqConfig">
<ref bean="mqConfig" />
</property>
</bean>

<!-- 报警处理类 -->
<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.LogAlarmHandler" />

<bean id="metaManager" class="com.alibaba.otter.canal.meta.MemoryMetaManager" />

<bean id="eventStore" class="com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer">
<property name="bufferSize" value="${canal.instance.memory.buffer.size:16384}" />
<property name="bufferMemUnit" value="${canal.instance.memory.buffer.memunit:1024}" />
<property name="batchMode" value="${canal.instance.memory.batch.mode:MEMSIZE}" />
<property name="ddlIsolation" value="${canal.instance.get.ddl.isolation:false}" />
<property name="raw" value="${canal.instance.memory.rawEntry:true}" />
</bean>

<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
</bean>

<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.group.GroupEventParser">
<property name="eventParsers">
<list>
<ref bean="eventParser1" />
<ref bean="eventParser2" />
</list>
</property>
</bean>

<bean id="eventParser1" parent="baseEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
<property name="detectingIntervalInSeconds" value="${canal.instance.detecting.interval.time:5}" />
<property name="haController">
<bean class="com.alibaba.otter.canal.parse.ha.HeartBeatHAController">
<property name="detectingRetryTimes" value="${canal.instance.detecting.retry.threshold:3}" />
<property name="switchEnable" value="${canal.instance.detecting.heartbeatHaEnable:false}" />
</bean>
</property>

<property name="alarmHandler" ref="alarmHandler" />

<!-- 解析过滤处理 -->
<property name="eventFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
</bean>
</property>

<property name="eventBlackFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.black.regex:}" />
<constructor-arg index="1" value="false" />
</bean>
</property>
<property name="fieldFilter" value="${canal.instance.filter.field}" />
<property name="fieldBlackFilter" value="${canal.instance.filter.black.field}" />
<!-- 最大事务解析大小,超过该大小后事务将被切分为多个事务投递 -->
<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />

<!-- 网络链接参数 -->
<property name="receiveBufferSize" value="${canal.instance.network.receiveBufferSize:16384}" />
<property name="sendBufferSize" value="${canal.instance.network.sendBufferSize:16384}" />
<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />

<!-- 解析编码 -->
<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />

<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</property>

<!-- failover切换时回退的时间 -->
<property name="fallbackIntervalInSeconds" value="${canal.instance.fallbackIntervalInSeconds:60}" />

<!-- 解析数据库信息 -->
<property name="masterInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.master1.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>
<property name="standbyInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.standby1.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>

<!-- 解析起始位点 -->
<property name="masterPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.master1.journal.name}" />
<property name="position" value="${canal.instance.master1.position}" />
<property name="timestamp" value="${canal.instance.master1.timestamp}" />
<property name="gtid" value="${canal.instance.master1.gtid}" />
</bean>
</property>
<property name="standbyPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.standby1.journal.name}" />
<property name="position" value="${canal.instance.standby1.position}" />
<property name="timestamp" value="${canal.instance.standby1.timestamp}" />
<property name="gtid" value="${canal.instance.standby1.gtid}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
<property name="filterDmlInsert" value="${canal.instance.filter.dml.insert:false}" />
<property name="filterDmlUpdate" value="${canal.instance.filter.dml.update:false}" />
<property name="filterDmlDelete" value="${canal.instance.filter.dml.delete:false}" />
<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />

<!-- parallel parser -->
<property name="parallel" value="${canal.instance.parser.parallel:true}" />
<property name="parallelThreadSize" value="${canal.instance.parser.parallelThreadSize}" />
<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />

<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
</bean>

<bean id="eventParser2" parent="baseEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
<property name="detectingIntervalInSeconds" value="${canal.instance.detecting.interval.time:5}" />
<property name="haController">
<bean class="com.alibaba.otter.canal.parse.ha.HeartBeatHAController">
<property name="detectingRetryTimes" value="${canal.instance.detecting.retry.threshold:3}" />
<property name="switchEnable" value="${canal.instance.detecting.heartbeatHaEnable:false}" />
</bean>
</property>

<property name="alarmHandler" ref="alarmHandler" />

<!-- 解析过滤处理 -->
<property name="eventFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
</bean>
</property>

<property name="eventBlackFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.black.regex:}" />
<constructor-arg index="1" value="false" />
</bean>
</property>
<!-- 最大事务解析大小,超过该大小后事务将被切分为多个事务投递 -->
<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />

<!-- 网络链接参数 -->
<property name="receiveBufferSize" value="${canal.instance.network.receiveBufferSize:16384}" />
<property name="sendBufferSize" value="${canal.instance.network.sendBufferSize:16384}" />
<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />

<!-- 解析编码 -->
<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />

<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</property>

<!-- failover切换时回退的时间 -->
<property name="fallbackIntervalInSeconds" value="${canal.instance.fallbackIntervalInSeconds:60}" />

<!-- 解析数据库信息 -->
<property name="masterInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.master2.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>
<property name="standbyInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.standby2.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>

<!-- 解析起始位点 -->
<property name="masterPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.master2.journal.name}" />
<property name="position" value="${canal.instance.master2.position}" />
<property name="timestamp" value="${canal.instance.master2.timestamp}" />
<property name="gtid" value="${canal.instance.master2.gtid}" />
</bean>
</property>
<property name="standbyPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.standby2.journal.name}" />
<property name="position" value="${canal.instance.standby2.position}" />
<property name="timestamp" value="${canal.instance.standby2.timestamp}" />
<property name="gtid" value="${canal.instance.standby2.gtid}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
<property name="filterDmlInsert" value="${canal.instance.filter.dml.insert:false}" />
<property name="filterDmlUpdate" value="${canal.instance.filter.dml.update:false}" />
<property name="filterDmlDelete" value="${canal.instance.filter.dml.delete:false}" />
<property name="filterRows" value="${canal.instance.filter.rows:false}" />
<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />

<!-- parallel parser -->
<property name="parallel" value="${canal.instance.parser.parallel:true}" />
<property name="parallelThreadSize" value="${canal.instance.parser.parallelThreadSize}" />
<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />

<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
</bean>

<bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">
<property name="topic" value="${canal.mq.topic}" />
<property name="dynamicTopic" value="${canal.mq.dynamicTopic}" />
<property name="partition" value="${canal.mq.partition}" />
<property name="partitionsNum" value="${canal.mq.partitionsNum}" />
<property name="partitionHash" value="${canal.mq.partitionHash}" />
<property name="dynamicTopicPartitionNum" value="${canal.mq.dynamicTopicPartitionNum}" />
<property name="enableDynamicQueuePartition" value="${canal.mq.enableDynamicQueuePartition}" />
</bean>
</beans>

+ 185
- 0
smtweb-framework/canal/server/src/main/resources/spring/memory-instance.xml View File

@@ -0,0 +1,185 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">

<import resource="classpath:spring/base-instance.xml" />

<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
<property name="destination" value="${canal.instance.destination}" />
<property name="eventParser">
<ref bean="eventParser" />
</property>
<property name="eventSink">
<ref bean="eventSink" />
</property>
<property name="eventStore">
<ref bean="eventStore" />
</property>
<property name="metaManager">
<ref bean="metaManager" />
</property>
<property name="alarmHandler">
<ref bean="alarmHandler" />
</property>
<property name="mqConfig">
<ref bean="mqConfig" />
</property>
</bean>

<!-- 报警处理类 -->
<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.LogAlarmHandler" />

<bean id="metaManager" class="com.alibaba.otter.canal.meta.MemoryMetaManager" />

<bean id="eventStore" class="com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer">
<property name="bufferSize" value="${canal.instance.memory.buffer.size:16384}" />
<property name="bufferMemUnit" value="${canal.instance.memory.buffer.memunit:1024}" />
<property name="batchMode" value="${canal.instance.memory.batch.mode:MEMSIZE}" />
<property name="ddlIsolation" value="${canal.instance.get.ddl.isolation:false}" />
<property name="raw" value="${canal.instance.memory.rawEntry:true}" />
</bean>

<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
</bean>

<bean id="eventParser" parent="baseEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
<property name="detectingIntervalInSeconds" value="${canal.instance.detecting.interval.time:5}" />
<property name="haController">
<bean class="com.alibaba.otter.canal.parse.ha.HeartBeatHAController">
<property name="detectingRetryTimes" value="${canal.instance.detecting.retry.threshold:3}" />
<property name="switchEnable" value="${canal.instance.detecting.heartbeatHaEnable:false}" />
</bean>
</property>

<property name="alarmHandler" ref="alarmHandler" />

<!-- 解析过滤处理 -->
<property name="eventFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
</bean>
</property>

<property name="eventBlackFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.black.regex:}" />
<constructor-arg index="1" value="false" />
</bean>
</property>
<property name="fieldFilter" value="${canal.instance.filter.field}" />
<property name="fieldBlackFilter" value="${canal.instance.filter.black.field}" />
<!-- 最大事务解析大小,超过该大小后事务将被切分为多个事务投递 -->
<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />

<!-- 网络链接参数 -->
<property name="receiveBufferSize" value="${canal.instance.network.receiveBufferSize:16384}" />
<property name="sendBufferSize" value="${canal.instance.network.sendBufferSize:16384}" />
<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />

<!-- 解析编码 -->
<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />

<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</property>

<!-- failover切换时回退的时间 -->
<property name="fallbackIntervalInSeconds" value="${canal.instance.fallbackIntervalInSeconds:60}" />

<!-- 解析数据库信息 -->
<property name="masterInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.master.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>
<property name="standbyInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.standby.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>

<!-- 解析起始位点 -->
<property name="masterPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.master.journal.name}" />
<property name="position" value="${canal.instance.master.position}" />
<property name="timestamp" value="${canal.instance.master.timestamp}" />
<property name="gtid" value="${canal.instance.master.gtid}" />
</bean>
</property>
<property name="standbyPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.standby.journal.name}" />
<property name="position" value="${canal.instance.standby.position}" />
<property name="timestamp" value="${canal.instance.standby.timestamp}" />
<property name="gtid" value="${canal.instance.standby.gtid}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
<property name="filterDmlInsert" value="${canal.instance.filter.dml.insert:false}" />
<property name="filterDmlUpdate" value="${canal.instance.filter.dml.update:false}" />
<property name="filterDmlDelete" value="${canal.instance.filter.dml.delete:false}" />
<property name="filterRows" value="${canal.instance.filter.rows:false}" />
<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />

<!--表结构相关-->
<property name="enableTsdb" value="${canal.instance.tsdb.enable:false}"/>
<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
<property name="tsdbSnapshotInterval" value="${canal.instance.tsdb.snapshot.interval:24}" />
<property name="tsdbSnapshotExpire" value="${canal.instance.tsdb.snapshot.expire:360}" />

<!--是否启用GTID模式-->
<property name="isGTIDMode" value="${canal.instance.gtidon:false}"/>

<!-- parallel parser -->
<property name="parallel" value="${canal.instance.parser.parallel:true}" />
<property name="parallelThreadSize" value="${canal.instance.parser.parallelThreadSize}" />
<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />

<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
</bean>

<bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">
<property name="topic" value="${canal.mq.topic}" />
<property name="dynamicTopic" value="${canal.mq.dynamicTopic}" />
<property name="partition" value="${canal.mq.partition}" />
<property name="partitionsNum" value="${canal.mq.partitionsNum}" />
<property name="partitionHash" value="${canal.mq.partitionHash}" />
<property name="dynamicTopicPartitionNum" value="${canal.mq.dynamicTopicPartitionNum}" />
<property name="enableDynamicQueuePartition" value="${canal.mq.enableDynamicQueuePartition}" />
</bean>
</beans>

+ 61
- 0
smtweb-framework/canal/server/src/main/resources/spring/tsdb/h2-tsdb.xml View File

@@ -0,0 +1,61 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">
<!-- properties -->
<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
<property name="ignoreResourceNotFound" value="true" />
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
<property name="locationNames">
<list>
<value>classpath:canal.properties</value>
<value>classpath:${canal.instance.destination:}/instance.properties</value>
</list>
</property>
</bean>
<!-- 基于db的实现 -->
<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta" destroy-method="destory">
<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
</bean>
<bean id="dataSource" class="com.alibaba.druid.pool.DruidDataSource" destroy-method="close">
<property name="driverClassName" value="org.h2.Driver" />
<property name="url" value="${canal.instance.tsdb.url:}" />
<property name="username" value="${canal.instance.tsdb.dbUsername:}" />
<property name="password" value="${canal.instance.tsdb.dbPassword:}" />
<property name="maxActive" value="30" />
<property name="initialSize" value="0" />
<property name="minIdle" value="1" />
<property name="maxWait" value="10000" />
<property name="timeBetweenEvictionRunsMillis" value="60000" />
<property name="minEvictableIdleTimeMillis" value="300000" />
<property name="testWhileIdle" value="true" />
<property name="testOnBorrow" value="false" />
<property name="testOnReturn" value="false" />
<property name="useUnfairLock" value="true" />
<property name="validationQuery" value="SELECT 1" />
</bean>

<bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">
<property name="dataSource" ref="dataSource"/>
<property name="configLocation" value="classpath:spring/tsdb/sql-map/sqlmap-config.xml"/>
</bean>

<bean id="metaHistoryDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO">
<property name="sqlSessionFactory" ref="sqlSessionFactory"/>
</bean>

<bean id="metaSnapshotDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO">
<property name="sqlSessionFactory" ref="sqlSessionFactory"/>
</bean>
</beans>

+ 63
- 0
smtweb-framework/canal/server/src/main/resources/spring/tsdb/mysql-tsdb.xml View File

@@ -0,0 +1,63 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">
<!-- properties -->
<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
<property name="ignoreResourceNotFound" value="true" />
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
<property name="locationNames">
<list>
<value>classpath:canal.properties</value>
<value>classpath:${canal.instance.destination:}/instance.properties</value>
</list>
</property>
</bean>
<!-- 基于db的实现 -->
<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta" destroy-method="destory">
<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
</bean>
<bean id="dataSource" class="com.alibaba.druid.pool.DruidDataSource" destroy-method="close">
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
<property name="url" value="${canal.instance.tsdb.url:}" />
<property name="username" value="${canal.instance.tsdb.dbUsername:}" />
<property name="password" value="${canal.instance.tsdb.dbPassword:}" />
<property name="maxActive" value="30" />
<property name="initialSize" value="0" />
<property name="minIdle" value="1" />
<property name="maxWait" value="10000" />
<property name="timeBetweenEvictionRunsMillis" value="60000" />
<property name="minEvictableIdleTimeMillis" value="300000" />
<property name="validationQuery" value="SELECT 1" />
<property name="exceptionSorterClassName" value="com.alibaba.druid.pool.vendor.MySqlExceptionSorter" />
<property name="validConnectionCheckerClassName" value="com.alibaba.druid.pool.vendor.MySqlValidConnectionChecker" />
<property name="testWhileIdle" value="true" />
<property name="testOnBorrow" value="false" />
<property name="testOnReturn" value="false" />
<property name="useUnfairLock" value="true" />
</bean>

<bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">
<property name="dataSource" ref="dataSource"/>
<property name="configLocation" value="classpath:spring/tsdb/sql-map/sqlmap-config.xml"/>
</bean>

<bean id="metaHistoryDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO">
<property name="sqlSessionFactory" ref="sqlSessionFactory"/>
</bean>

<bean id="metaSnapshotDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO">
<property name="sqlSessionFactory" ref="sqlSessionFactory"/>
</bean>
</beans>

+ 14
- 0
smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap-config.xml View File

@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE configuration PUBLIC "-//mybatis.org//DTD Config 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-config.dtd">
<configuration>
<typeAliases>
<typeAlias alias="metaHistoryDO" type="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDO"/>
<typeAlias alias="metaSnapshotDO" type="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDO"/>
</typeAliases>

<mappers>
<mapper resource="spring/tsdb/sql-map/sqlmap_history.xml"/>
<mapper resource="spring/tsdb/sql-map/sqlmap_snapshot.xml"/>
</mappers>
</configuration>

+ 44
- 0
smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml View File

@@ -0,0 +1,44 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryMapper">
<sql id="allColumns">
<![CDATA[
gmt_create,gmt_modified,destination,binlog_file,binlog_offest,binlog_master_id,binlog_timestamp,use_schema,sql_schema,sql_table,sql_text,sql_type,extra
]]>
</sql>
<sql id="allVOColumns">
<![CDATA[
a.id as id,a.gmt_create as gmtCreate,a.gmt_modified as gmtModified,
a.destination as destination,a.binlog_file as binlogFile,a.binlog_offest as binlogOffest,a.binlog_master_id as binlogMasterId,a.binlog_timestamp as binlogTimestamp,
a.use_schema as useSchema,a.sql_schema as sqlSchema,a.sql_table as sqlTable,a.sql_text as sqlText,a.sql_type as sqlType,a.extra as extra
]]>
</sql>

<select id="findByTimestamp" parameterType="java.util.Map" resultType="metaHistoryDO">
select
<include refid="allVOColumns"/>
from meta_history a
<![CDATA[
where destination = #{destination} and binlog_timestamp >= #{snapshotTimestamp} and binlog_timestamp <= #{timestamp}
order by binlog_timestamp asc,id asc
]]>
</select>

<insert id="insert" parameterType="metaHistoryDO">
insert into meta_history (<include refid="allColumns"/>)
values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#{destination},#{binlogFile},#{binlogOffest},#{binlogMasterId},#{binlogTimestamp},#{useSchema},#{sqlSchema},#{sqlTable},#{sqlText},#{sqlType},#{extra})
</insert>
<delete id="deleteByName" parameterType="java.util.Map">
delete from meta_history
where destination=#{destination}
</delete>


<delete id="deleteByTimestamp" parameterType="java.util.Map">
<![CDATA[
delete from meta_history
where destination=#{destination} and binlog_timestamp < #{timestamp}
]]>
</delete>
</mapper>

+ 48
- 0
smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml View File

@@ -0,0 +1,48 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotMapper">
<sql id="allColumns">
<![CDATA[
gmt_create,gmt_modified,destination,binlog_file,binlog_offest,binlog_master_id,binlog_timestamp,data,extra
]]>
</sql>
<sql id="allVOColumns">
<![CDATA[
a.id as id,a.gmt_create as gmtCreate,a.gmt_modified as gmtModified,
a.destination as destination,a.binlog_file as binlogFile,a.binlog_offest as binlogOffest,a.binlog_master_id as binlogMasterId,a.binlog_timestamp as binlogTimestamp,a.data as data,a.extra as extra
]]>
</sql>

<select id="findByTimestamp" parameterType="java.util.Map" resultType="metaSnapshotDO">
select <include refid="allVOColumns"/>
<![CDATA[
from meta_snapshot a
where destination = #{destination} and binlog_timestamp < #{timestamp}
order by binlog_timestamp desc,id desc
limit 1
]]>
</select>

<insert id="insert" parameterType="metaSnapshotDO">
insert into meta_snapshot (<include refid="allColumns"/>)
values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#{destination},#{binlogFile},#{binlogOffest},#{binlogMasterId},#{binlogTimestamp},#{data},#{extra})
</insert>

<update id="update" parameterType="metaSnapshotDO">
update meta_snapshot set gmt_modified=now(),
binlog_file=#{binlogFile},binlog_offest=#{binlogOffest},binlog_master_id=#{binlogMasterId},binlog_timestamp=#{binlogTimestamp},data=#{data},extra=#{extra}
where destination=#{destination} and binlog_timestamp=0
</update>

<delete id="deleteByName" parameterType="java.util.Map">
delete from meta_snapshot
where destination=#{destination}
</delete>

<delete id="deleteByTimestamp" parameterType="java.util.Map">
<![CDATA[
delete from meta_snapshot
where destination=#{destination} and binlog_timestamp < #{timestamp} and binlog_timestamp > 0
]]>
</delete>
</mapper>

+ 39
- 0
smtweb-framework/canal/server/src/main/resources/spring/tsdb/sql/create_table.sql View File

@@ -0,0 +1,39 @@
CREATE TABLE IF NOT EXISTS `meta_snapshot` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',
`gmt_create` datetime NOT NULL COMMENT '创建时间',
`gmt_modified` datetime NOT NULL COMMENT '修改时间',
`destination` varchar(128) DEFAULT NULL COMMENT '通道名称',
`binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名',
`binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量',
`binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id',
`binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳',
`data` longtext DEFAULT NULL COMMENT '表结构数据',
`extra` text DEFAULT NULL COMMENT '额外的扩展信息',
PRIMARY KEY (`id`),
UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`),
KEY `destination` (`destination`),
KEY `destination_timestamp` (`destination`,`binlog_timestamp`),
KEY `gmt_modified` (`gmt_modified`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='表结构记录表快照表';

CREATE TABLE IF NOT EXISTS `meta_history` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',
`gmt_create` datetime NOT NULL COMMENT '创建时间',
`gmt_modified` datetime NOT NULL COMMENT '修改时间',
`destination` varchar(128) DEFAULT NULL COMMENT '通道名称',
`binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名',
`binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量',
`binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id',
`binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳',
`use_schema` varchar(1024) DEFAULT NULL COMMENT '执行sql时对应的schema',
`sql_schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema',
`sql_table` varchar(1024) DEFAULT NULL COMMENT '对应的table',
`sql_text` longtext DEFAULT NULL COMMENT '执行的sql',
`sql_type` varchar(256) DEFAULT NULL COMMENT 'sql类型',
`extra` text DEFAULT NULL COMMENT '额外的扩展信息',
PRIMARY KEY (`id`),
UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`),
KEY `destination` (`destination`),
KEY `destination_timestamp` (`destination`,`binlog_timestamp`),
KEY `gmt_modified` (`gmt_modified`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='表结构变化明细表';

+ 182
- 0
smtweb-framework/canal/server/target/classes/canal.properties View File

@@ -0,0 +1,182 @@
#################################################
######### common argument #############
#################################################
# tcp bind ip
canal.ip =
# register ip to zookeeper
canal.register.ip =
canal.port = 11111
canal.metrics.pull.port = 11112
# canal instance user/passwd
# canal.user = canal
# canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458

# canal admin config
#canal.admin.manager = 127.0.0.1:8089
canal.admin.port = 11110
canal.admin.user = admin
canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
# admin auto register
#canal.admin.register.auto = true
#canal.admin.register.cluster =
#canal.admin.register.name =

canal.zkServers =
# flush data to zk
canal.zookeeper.flush.period = 1000
canal.withoutNetty = false
# tcp, kafka, rocketMQ, rabbitMQ, pulsarMQ
canal.serverMode = tcp
# flush meta cursor/parse position to file
canal.file.data.dir = ${canal.conf.dir}
canal.file.flush.period = 1000
## memory store RingBuffer size, should be Math.pow(2,n)
canal.instance.memory.buffer.size = 16384
## memory store RingBuffer used memory unit size , default 1kb
canal.instance.memory.buffer.memunit = 1024
## meory store gets mode used MEMSIZE or ITEMSIZE
canal.instance.memory.batch.mode = MEMSIZE
canal.instance.memory.rawEntry = true

## detecing config
canal.instance.detecting.enable = false
#canal.instance.detecting.sql = insert into retl.xdual values(1,now()) on duplicate key update x=now()
canal.instance.detecting.sql = select 1
canal.instance.detecting.interval.time = 3
canal.instance.detecting.retry.threshold = 3
canal.instance.detecting.heartbeatHaEnable = false

# support maximum transaction size, more than the size of the transaction will be cut into multiple transactions delivery
canal.instance.transaction.size = 1024
# mysql fallback connected to new master should fallback times
canal.instance.fallbackIntervalInSeconds = 60

# network config
canal.instance.network.receiveBufferSize = 16384
canal.instance.network.sendBufferSize = 16384
canal.instance.network.soTimeout = 30

# binlog filter config
canal.instance.filter.druid.ddl = true
canal.instance.filter.query.dcl = false
canal.instance.filter.query.dml = false
canal.instance.filter.query.ddl = false
canal.instance.filter.table.error = false
canal.instance.filter.rows = false
canal.instance.filter.transaction.entry = false
canal.instance.filter.dml.insert = false
canal.instance.filter.dml.update = false
canal.instance.filter.dml.delete = false

# binlog format/image check
canal.instance.binlog.format = ROW,STATEMENT,MIXED
canal.instance.binlog.image = FULL,MINIMAL,NOBLOB

# binlog ddl isolation
canal.instance.get.ddl.isolation = false

# parallel parser config
canal.instance.parser.parallel = true
## concurrent thread number, default 60% available processors, suggest not to exceed Runtime.getRuntime().availableProcessors()
#canal.instance.parser.parallelThreadSize = 16
## disruptor ringbuffer size, must be power of 2
canal.instance.parser.parallelBufferSize = 256

# table meta tsdb info
canal.instance.tsdb.enable = true
canal.instance.tsdb.dir = ${canal.file.data.dir:../conf}/${canal.instance.destination:}
canal.instance.tsdb.url = jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
canal.instance.tsdb.dbUsername = canal
canal.instance.tsdb.dbPassword = canal
# dump snapshot interval, default 24 hour
canal.instance.tsdb.snapshot.interval = 24
# purge snapshot expire , default 360 hour(15 days)
canal.instance.tsdb.snapshot.expire = 360

#################################################
######### destinations #############
#################################################
canal.destinations = example
# conf root dir
canal.conf.dir = ../conf
# auto scan instance dir add/remove and start/stop instance
canal.auto.scan = true
canal.auto.scan.interval = 5
# set this value to 'true' means that when binlog pos not found, skip to latest.
# WARN: pls keep 'false' in production env, or if you know what you want.
canal.auto.reset.latest.pos.mode = false

canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml
#canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml

canal.instance.global.mode = spring
canal.instance.global.lazy = false
canal.instance.global.manager.address = ${canal.admin.manager}
#canal.instance.global.spring.xml = classpath:spring/memory-instance.xml
canal.instance.global.spring.xml = classpath:spring/file-instance.xml
#canal.instance.global.spring.xml = classpath:spring/default-instance.xml

##################################################
######### MQ Properties #############
##################################################
# aliyun ak/sk , support rds/mq
canal.aliyun.accessKey =
canal.aliyun.secretKey =
canal.aliyun.uid=

canal.mq.flatMessage = true
canal.mq.canalBatchSize = 50
canal.mq.canalGetTimeout = 100
# Set this value to "cloud", if you want open message trace feature in aliyun.
canal.mq.accessChannel = local

canal.mq.database.hash = true
canal.mq.send.thread.size = 30
canal.mq.build.thread.size = 8

##################################################
######### Kafka #############
##################################################
kafka.bootstrap.servers = 127.0.0.1:9092
kafka.acks = all
kafka.compression.type = none
kafka.batch.size = 16384
kafka.linger.ms = 1
kafka.max.request.size = 1048576
kafka.buffer.memory = 33554432
kafka.max.in.flight.requests.per.connection = 1
kafka.retries = 0

kafka.kerberos.enable = false
kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"

##################################################
######### RocketMQ #############
##################################################
rocketmq.producer.group = test
rocketmq.enable.message.trace = false
rocketmq.customized.trace.topic =
rocketmq.namespace =
rocketmq.namesrv.addr = 127.0.0.1:9876
rocketmq.retry.times.when.send.failed = 0
rocketmq.vip.channel.enabled = false
rocketmq.tag =

##################################################
######### RabbitMQ #############
##################################################
rabbitmq.host =
rabbitmq.virtual.host =
rabbitmq.exchange =
rabbitmq.username =
rabbitmq.password =
rabbitmq.deliveryMode =


##################################################
######### Pulsar #############
##################################################
pulsarmq.serverUrl =
pulsarmq.roleToken =
pulsarmq.topicTenantPrefix =

+ 12
- 0
smtweb-framework/canal/server/target/classes/canal_local.properties View File

@@ -0,0 +1,12 @@
# register ip
canal.register.ip =

# canal admin config
canal.admin.manager = 127.0.0.1:8089
canal.admin.port = 11110
canal.admin.user = admin
canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
# admin auto register
canal.admin.register.auto = true
canal.admin.register.cluster =
canal.admin.register.name =

+ 59
- 0
smtweb-framework/canal/server/target/classes/example/instance.properties View File

@@ -0,0 +1,59 @@
#################################################
## mysql serverId , v1.0.26+ will autoGen
canal.instance.mysql.slaveId=2

# enable gtid use true/false
canal.instance.gtidon=false

# position info
canal.instance.master.address=172.28.123.25:3306
#canal.instance.master.journal.name=
#canal.instance.master.position=
#canal.instance.master.timestamp=
#canal.instance.master.gtid=

# rds oss binlog
canal.instance.rds.accesskey=
canal.instance.rds.secretkey=
canal.instance.rds.instanceId=

# table meta tsdb info
canal.instance.tsdb.enable=true
#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
#canal.instance.tsdb.dbUsername=canal
#canal.instance.tsdb.dbPassword=canal

#canal.instance.standby.address =
#canal.instance.standby.journal.name =
#canal.instance.standby.position =
#canal.instance.standby.timestamp =
#canal.instance.standby.gtid=

# username/password
canal.instance.dbUsername=root
canal.instance.dbPassword=root
canal.instance.connectionCharset = UTF-8
# enable druid Decrypt database password
canal.instance.enableDruid=false
#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==

# table regex
canal.instance.filter.regex=scmz\\..*
# table black regex
canal.instance.filter.black.regex=mysql\\.slave_.*
# table field filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)
#canal.instance.filter.field=test1.t_product:id/subject/keywords,test2.t_company:id/name/contact/ch
# table field black filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)
#canal.instance.filter.black.field=test1.t_product:subject/product_image,test2.t_company:id/name/contact/ch

# mq config
canal.mq.topic=example
# dynamic topic route by schema or table regex
#canal.mq.dynamicTopic=mytest1.user,topic2:mytest2\\..*,.*\\..*
canal.mq.partition=0
# hash partition config
#canal.mq.enableDynamicQueuePartition=false
#canal.mq.partitionsNum=3
#canal.mq.dynamicTopicPartitionNum=test.*:4,mycanal:6
#canal.mq.partitionHash=test.table:id^name,.*\\..*
#################################################

+ 117
- 0
smtweb-framework/canal/server/target/classes/logback.xml.bak View File

@@ -0,0 +1,117 @@
<configuration scan="true" scanPeriod=" 5 seconds">
<jmxConfigurator />
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
</pattern>
</encoder>
</appender>

<appender name="CANAL-ROOT" class="ch.qos.logback.classic.sift.SiftingAppender">
<discriminator>
<Key>destination</Key>
<DefaultValue>canal</DefaultValue>
</discriminator>
<sift>
<appender name="FILE-${destination}" class="ch.qos.logback.core.rolling.RollingFileAppender">
<File>../logs/${destination}/${destination}.log</File>
<rollingPolicy
class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- rollover daily -->
<fileNamePattern>../logs/${destination}/%d{yyyy-MM-dd}/${destination}-%d{yyyy-MM-dd}-%i.log.gz</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- or whenever the file size reaches 100MB -->
<maxFileSize>512MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>60</maxHistory>
</rollingPolicy>
<encoder>
<pattern>
%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
</pattern>
</encoder>
</appender>
</sift>
</appender>

<appender name="CANAL-META" class="ch.qos.logback.classic.sift.SiftingAppender">
<discriminator>
<Key>destination</Key>
<DefaultValue>canal</DefaultValue>
</discriminator>
<sift>
<appender name="META-FILE-${destination}" class="ch.qos.logback.core.rolling.RollingFileAppender">
<File>../logs/${destination}/meta.log</File>
<rollingPolicy
class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- rollover daily -->
<fileNamePattern>../logs/${destination}/%d{yyyy-MM-dd}/meta-%d{yyyy-MM-dd}-%i.log.gz</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- or whenever the file size reaches 100MB -->
<maxFileSize>32MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>60</maxHistory>
</rollingPolicy>
<encoder>
<pattern>
%d{yyyy-MM-dd HH:mm:ss.SSS} - %msg%n
</pattern>
</encoder>
</appender>
</sift>
</appender>

<appender name="RocketmqClientAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>../logs/canal/rocketmq_client.log</file>
<rollingPolicy
class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- rollover daily -->
<fileNamePattern>../logs/canal/%d{yyyy-MM-dd}/rocketmq_client-%d{yyyy-MM-dd}-%i.log.gz</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- or whenever the file size reaches 100MB -->
<maxFileSize>512MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>60</maxHistory>
</rollingPolicy>
<encoder charset="UTF-8">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n</pattern>
</encoder>
</appender>

<logger name="com.alibaba.otter.canal.instance" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-ROOT" />
</logger>
<logger name="com.alibaba.otter.canal.deployer" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-ROOT" />
</logger>
<logger name="com.alibaba.otter.canal.meta.FileMixedMetaManager" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-META" />
</logger>
<logger name="com.alibaba.otter.canal.connector.kafka" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-ROOT" />
</logger>
<logger name="com.alibaba.otter.canal.connector.rocketmq" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-ROOT" />
</logger>
<logger name="com.alibaba.otter.canal.connector.rabbitmq" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-ROOT" />
</logger>
<logger name="RocketmqClient" additivity="false">
<level value="INFO" />
<appender-ref ref="RocketmqClientAppender" />
</logger>
<logger name="com.alibaba.otter.canal.connector.pulsarmq" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-ROOT" />
</logger>
<root level="WARN">
<!-- <appender-ref ref="STDOUT"/> -->
<appender-ref ref="CANAL-ROOT" />
</root>
</configuration>

+ 1375
- 0
smtweb-framework/canal/server/target/classes/metrics/Canal_instances_tmpl.json
File diff suppressed because it is too large
View File


+ 39
- 0
smtweb-framework/canal/server/target/classes/spring/base-instance.xml View File

@@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">

<!-- properties -->
<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
<property name="ignoreResourceNotFound" value="true" />
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
<property name="locationNames">
<list>
<value>classpath:canal.properties</value>
<value>classpath:${canal.instance.destination:}/instance.properties</value>
</list>
</property>
</bean>
<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer">
<property name="propertyEditorRegistrars">
<list>
<ref bean="socketAddressEditor" />
</list>
</property>
</bean>
<bean id="baseEventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogEventParserProxy" abstract="true">
<property name="accesskey" value="${canal.aliyun.accesskey:}" />
<property name="secretkey" value="${canal.aliyun.secretkey:}" />
<property name="instanceId" value="${canal.instance.rds.instanceId:}" />
</bean>
</beans>

+ 211
- 0
smtweb-framework/canal/server/target/classes/spring/default-instance.xml View File

@@ -0,0 +1,211 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">

<import resource="classpath:spring/base-instance.xml" />

<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
<property name="destination" value="${canal.instance.destination}" />
<property name="eventParser">
<ref bean="eventParser" />
</property>
<property name="eventSink">
<ref bean="eventSink" />
</property>
<property name="eventStore">
<ref bean="eventStore" />
</property>
<property name="metaManager">
<ref bean="metaManager" />
</property>
<property name="alarmHandler">
<ref bean="alarmHandler" />
</property>
<property name="mqConfig">
<ref bean="mqConfig" />
</property>
</bean>

<!-- 报警处理类 -->
<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.LogAlarmHandler" />

<bean id="zkClientx" class="org.springframework.beans.factory.config.MethodInvokingFactoryBean" >
<property name="targetClass" value="com.alibaba.otter.canal.common.zookeeper.ZkClientx" />
<property name="targetMethod" value="getZkClient" />
<property name="arguments">
<list>
<value>${canal.zkServers:127.0.0.1:2181}</value>
</list>
</property>
</bean>

<bean id="metaManager" class="com.alibaba.otter.canal.meta.PeriodMixedMetaManager">
<property name="zooKeeperMetaManager">
<bean class="com.alibaba.otter.canal.meta.ZooKeeperMetaManager">
<property name="zkClientx" ref="zkClientx" />
</bean>
</property>
<property name="period" value="${canal.zookeeper.flush.period:1000}" />
</bean>

<bean id="eventStore" class="com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer">
<property name="bufferSize" value="${canal.instance.memory.buffer.size:16384}" />
<property name="bufferMemUnit" value="${canal.instance.memory.buffer.memunit:1024}" />
<property name="batchMode" value="${canal.instance.memory.batch.mode:MEMSIZE}" />
<property name="ddlIsolation" value="${canal.instance.get.ddl.isolation:false}" />
<property name="raw" value="${canal.instance.memory.rawEntry:true}" />
</bean>

<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
</bean>

<bean id="eventParser" parent="baseEventParser" >
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
<property name="detectingIntervalInSeconds" value="${canal.instance.detecting.interval.time:5}" />
<property name="haController">
<bean class="com.alibaba.otter.canal.parse.ha.HeartBeatHAController">
<property name="detectingRetryTimes" value="${canal.instance.detecting.retry.threshold:3}" />
<property name="switchEnable" value="${canal.instance.detecting.heartbeatHaEnable:false}" />
</bean>
</property>

<property name="alarmHandler" ref="alarmHandler" />

<!-- 解析过滤处理 -->
<property name="eventFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
</bean>
</property>

<property name="eventBlackFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.black.regex:}" />
<constructor-arg index="1" value="false" />
</bean>
</property>
<property name="fieldFilter" value="${canal.instance.filter.field}" />
<property name="fieldBlackFilter" value="${canal.instance.filter.black.field}" />

<!-- 最大事务解析大小,超过该大小后事务将被切分为多个事务投递 -->
<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />

<!-- 网络链接参数 -->
<property name="receiveBufferSize" value="${canal.instance.network.receiveBufferSize:16384}" />
<property name="sendBufferSize" value="${canal.instance.network.sendBufferSize:16384}" />
<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />

<!-- 解析编码 -->
<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />

<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</constructor-arg>
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
<constructor-arg ref="metaManager"/>
</bean>
</constructor-arg>
</bean>
</property>

<!-- failover切换时回退的时间 -->
<property name="fallbackIntervalInSeconds" value="${canal.instance.fallbackIntervalInSeconds:60}" />

<!-- 解析数据库信息 -->
<property name="masterInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.master.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>
<property name="standbyInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.standby.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>

<!-- 解析起始位点 -->
<property name="masterPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.master.journal.name}" />
<property name="position" value="${canal.instance.master.position}" />
<property name="timestamp" value="${canal.instance.master.timestamp}" />
<property name="gtid" value="${canal.instance.master.gtid}" />
</bean>
</property>
<property name="standbyPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.standby.journal.name}" />
<property name="position" value="${canal.instance.standby.position}" />
<property name="timestamp" value="${canal.instance.standby.timestamp}" />
<property name="gtid" value="${canal.instance.standby.gtid}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
<property name="filterDmlInsert" value="${canal.instance.filter.dml.insert:false}" />
<property name="filterDmlUpdate" value="${canal.instance.filter.dml.update:false}" />
<property name="filterDmlDelete" value="${canal.instance.filter.dml.delete:false}" />
<property name="filterRows" value="${canal.instance.filter.rows:false}" />
<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />

<!--表结构相关-->
<property name="enableTsdb" value="${canal.instance.tsdb.enable:true}"/>
<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
<property name="tsdbSnapshotInterval" value="${canal.instance.tsdb.snapshot.interval:24}" />
<property name="tsdbSnapshotExpire" value="${canal.instance.tsdb.snapshot.expire:360}" />

<!--是否启用GTID模式-->
<property name="isGTIDMode" value="${canal.instance.gtidon:false}"/>

<!-- parallel parser -->
<property name="parallel" value="${canal.instance.parser.parallel:true}" />
<property name="parallelThreadSize" value="${canal.instance.parser.parallelThreadSize}" />
<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />

<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
</bean>

<bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">
<property name="topic" value="${canal.mq.topic}" />
<property name="dynamicTopic" value="${canal.mq.dynamicTopic}" />
<property name="partition" value="${canal.mq.partition}" />
<property name="partitionsNum" value="${canal.mq.partitionsNum}" />
<property name="partitionHash" value="${canal.mq.partitionHash}" />
<property name="dynamicTopicPartitionNum" value="${canal.mq.dynamicTopicPartitionNum}" />
<property name="enableDynamicQueuePartition" value="${canal.mq.enableDynamicQueuePartition}" />
</bean>
</beans>

+ 197
- 0
smtweb-framework/canal/server/target/classes/spring/file-instance.xml View File

@@ -0,0 +1,197 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">

<import resource="classpath:spring/base-instance.xml" />

<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
<property name="destination" value="${canal.instance.destination}" />
<property name="eventParser">
<ref bean="eventParser" />
</property>
<property name="eventSink">
<ref bean="eventSink" />
</property>
<property name="eventStore">
<ref bean="eventStore" />
</property>
<property name="metaManager">
<ref bean="metaManager" />
</property>
<property name="alarmHandler">
<ref bean="alarmHandler" />
</property>
<property name="mqConfig">
<ref bean="mqConfig" />
</property>
</bean>

<!-- 报警处理类 -->
<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.LogAlarmHandler" />

<bean id="metaManager" class="com.alibaba.otter.canal.meta.FileMixedMetaManager">
<property name="dataDir" value="${canal.file.data.dir:../conf}" />
<property name="period" value="${canal.file.flush.period:1000}" />
</bean>

<bean id="eventStore" class="com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer">
<property name="bufferSize" value="${canal.instance.memory.buffer.size:16384}" />
<property name="bufferMemUnit" value="${canal.instance.memory.buffer.memunit:1024}" />
<property name="batchMode" value="${canal.instance.memory.batch.mode:MEMSIZE}" />
<property name="ddlIsolation" value="${canal.instance.get.ddl.isolation:false}" />
<property name="raw" value="${canal.instance.memory.rawEntry:true}" />
</bean>

<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
</bean>

<bean id="eventParser" parent="baseEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
<property name="detectingIntervalInSeconds" value="${canal.instance.detecting.interval.time:5}" />
<property name="haController">
<bean class="com.alibaba.otter.canal.parse.ha.HeartBeatHAController">
<property name="detectingRetryTimes" value="${canal.instance.detecting.retry.threshold:3}" />
<property name="switchEnable" value="${canal.instance.detecting.heartbeatHaEnable:false}" />
</bean>
</property>

<property name="alarmHandler" ref="alarmHandler" />

<!-- 解析过滤处理 -->
<property name="eventFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
</bean>
</property>

<property name="eventBlackFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.black.regex:}" />
<constructor-arg index="1" value="false" />
</bean>
</property>
<property name="fieldFilter" value="${canal.instance.filter.field}" />
<property name="fieldBlackFilter" value="${canal.instance.filter.black.field}" />
<!-- 最大事务解析大小,超过该大小后事务将被切分为多个事务投递 -->
<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />

<!-- 网络链接参数 -->
<property name="receiveBufferSize" value="${canal.instance.network.receiveBufferSize:16384}" />
<property name="sendBufferSize" value="${canal.instance.network.sendBufferSize:16384}" />
<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />

<!-- 解析编码 -->
<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />

<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</constructor-arg>
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
<constructor-arg ref="metaManager"/>
</bean>
</constructor-arg>
</bean>
</property>

<!-- failover切换时回退的时间 -->
<property name="fallbackIntervalInSeconds" value="${canal.instance.fallbackIntervalInSeconds:60}" />

<!-- 解析数据库信息 -->
<property name="masterInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.master.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>
<property name="standbyInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.standby.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>

<!-- 解析起始位点 -->
<property name="masterPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.master.journal.name}" />
<property name="position" value="${canal.instance.master.position}" />
<property name="timestamp" value="${canal.instance.master.timestamp}" />
<property name="gtid" value="${canal.instance.master.gtid}" />
</bean>
</property>
<property name="standbyPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.standby.journal.name}" />
<property name="position" value="${canal.instance.standby.position}" />
<property name="timestamp" value="${canal.instance.standby.timestamp}" />
<property name="gtid" value="${canal.instance.standby.gtid}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
<property name="filterDmlInsert" value="${canal.instance.filter.dml.insert:false}" />
<property name="filterDmlUpdate" value="${canal.instance.filter.dml.update:false}" />
<property name="filterDmlDelete" value="${canal.instance.filter.dml.delete:false}" />
<property name="filterRows" value="${canal.instance.filter.rows:false}" />
<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />

<!--表结构相关-->
<property name="enableTsdb" value="${canal.instance.tsdb.enable:true}"/>
<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
<property name="tsdbSnapshotInterval" value="${canal.instance.tsdb.snapshot.interval:24}" />
<property name="tsdbSnapshotExpire" value="${canal.instance.tsdb.snapshot.expire:360}" />

<!--是否启用GTID模式-->
<property name="isGTIDMode" value="${canal.instance.gtidon:false}"/>

<!-- parallel parser -->
<property name="parallel" value="${canal.instance.parser.parallel:true}" />
<property name="parallelThreadSize" value="${canal.instance.parser.parallelThreadSize}" />
<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />

<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
</bean>

<bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">
<property name="topic" value="${canal.mq.topic}" />
<property name="dynamicTopic" value="${canal.mq.dynamicTopic}" />
<property name="partition" value="${canal.mq.partition}" />
<property name="partitionsNum" value="${canal.mq.partitionsNum}" />
<property name="partitionHash" value="${canal.mq.partitionHash}" />
<property name="dynamicTopicPartitionNum" value="${canal.mq.dynamicTopicPartitionNum}" />
<property name="enableDynamicQueuePartition" value="${canal.mq.enableDynamicQueuePartition}" />
</bean>
</beans>

+ 292
- 0
smtweb-framework/canal/server/target/classes/spring/group-instance.xml View File

@@ -0,0 +1,292 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">

<import resource="classpath:spring/base-instance.xml" />

<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
<property name="destination" value="${canal.instance.destination}" />
<property name="eventParser">
<ref bean="eventParser" />
</property>
<property name="eventSink">
<ref bean="eventSink" />
</property>
<property name="eventStore">
<ref bean="eventStore" />
</property>
<property name="metaManager">
<ref bean="metaManager" />
</property>
<property name="alarmHandler">
<ref bean="alarmHandler" />
</property>
<property name="mqConfig">
<ref bean="mqConfig" />
</property>
</bean>

<!-- 报警处理类 -->
<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.LogAlarmHandler" />

<bean id="metaManager" class="com.alibaba.otter.canal.meta.MemoryMetaManager" />

<bean id="eventStore" class="com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer">
<property name="bufferSize" value="${canal.instance.memory.buffer.size:16384}" />
<property name="bufferMemUnit" value="${canal.instance.memory.buffer.memunit:1024}" />
<property name="batchMode" value="${canal.instance.memory.batch.mode:MEMSIZE}" />
<property name="ddlIsolation" value="${canal.instance.get.ddl.isolation:false}" />
<property name="raw" value="${canal.instance.memory.rawEntry:true}" />
</bean>

<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
</bean>

<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.group.GroupEventParser">
<property name="eventParsers">
<list>
<ref bean="eventParser1" />
<ref bean="eventParser2" />
</list>
</property>
</bean>

<bean id="eventParser1" parent="baseEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
<property name="detectingIntervalInSeconds" value="${canal.instance.detecting.interval.time:5}" />
<property name="haController">
<bean class="com.alibaba.otter.canal.parse.ha.HeartBeatHAController">
<property name="detectingRetryTimes" value="${canal.instance.detecting.retry.threshold:3}" />
<property name="switchEnable" value="${canal.instance.detecting.heartbeatHaEnable:false}" />
</bean>
</property>

<property name="alarmHandler" ref="alarmHandler" />

<!-- 解析过滤处理 -->
<property name="eventFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
</bean>
</property>

<property name="eventBlackFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.black.regex:}" />
<constructor-arg index="1" value="false" />
</bean>
</property>
<property name="fieldFilter" value="${canal.instance.filter.field}" />
<property name="fieldBlackFilter" value="${canal.instance.filter.black.field}" />
<!-- 最大事务解析大小,超过该大小后事务将被切分为多个事务投递 -->
<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />

<!-- 网络链接参数 -->
<property name="receiveBufferSize" value="${canal.instance.network.receiveBufferSize:16384}" />
<property name="sendBufferSize" value="${canal.instance.network.sendBufferSize:16384}" />
<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />

<!-- 解析编码 -->
<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />

<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</property>

<!-- failover切换时回退的时间 -->
<property name="fallbackIntervalInSeconds" value="${canal.instance.fallbackIntervalInSeconds:60}" />

<!-- 解析数据库信息 -->
<property name="masterInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.master1.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>
<property name="standbyInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.standby1.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>

<!-- 解析起始位点 -->
<property name="masterPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.master1.journal.name}" />
<property name="position" value="${canal.instance.master1.position}" />
<property name="timestamp" value="${canal.instance.master1.timestamp}" />
<property name="gtid" value="${canal.instance.master1.gtid}" />
</bean>
</property>
<property name="standbyPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.standby1.journal.name}" />
<property name="position" value="${canal.instance.standby1.position}" />
<property name="timestamp" value="${canal.instance.standby1.timestamp}" />
<property name="gtid" value="${canal.instance.standby1.gtid}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
<property name="filterDmlInsert" value="${canal.instance.filter.dml.insert:false}" />
<property name="filterDmlUpdate" value="${canal.instance.filter.dml.update:false}" />
<property name="filterDmlDelete" value="${canal.instance.filter.dml.delete:false}" />
<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />

<!-- parallel parser -->
<property name="parallel" value="${canal.instance.parser.parallel:true}" />
<property name="parallelThreadSize" value="${canal.instance.parser.parallelThreadSize}" />
<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />

<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
</bean>

<bean id="eventParser2" parent="baseEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
<property name="detectingIntervalInSeconds" value="${canal.instance.detecting.interval.time:5}" />
<property name="haController">
<bean class="com.alibaba.otter.canal.parse.ha.HeartBeatHAController">
<property name="detectingRetryTimes" value="${canal.instance.detecting.retry.threshold:3}" />
<property name="switchEnable" value="${canal.instance.detecting.heartbeatHaEnable:false}" />
</bean>
</property>

<property name="alarmHandler" ref="alarmHandler" />

<!-- 解析过滤处理 -->
<property name="eventFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
</bean>
</property>

<property name="eventBlackFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.black.regex:}" />
<constructor-arg index="1" value="false" />
</bean>
</property>
<!-- 最大事务解析大小,超过该大小后事务将被切分为多个事务投递 -->
<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />

<!-- 网络链接参数 -->
<property name="receiveBufferSize" value="${canal.instance.network.receiveBufferSize:16384}" />
<property name="sendBufferSize" value="${canal.instance.network.sendBufferSize:16384}" />
<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />

<!-- 解析编码 -->
<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />

<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</property>

<!-- failover切换时回退的时间 -->
<property name="fallbackIntervalInSeconds" value="${canal.instance.fallbackIntervalInSeconds:60}" />

<!-- 解析数据库信息 -->
<property name="masterInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.master2.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>
<property name="standbyInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.standby2.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>

<!-- 解析起始位点 -->
<property name="masterPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.master2.journal.name}" />
<property name="position" value="${canal.instance.master2.position}" />
<property name="timestamp" value="${canal.instance.master2.timestamp}" />
<property name="gtid" value="${canal.instance.master2.gtid}" />
</bean>
</property>
<property name="standbyPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.standby2.journal.name}" />
<property name="position" value="${canal.instance.standby2.position}" />
<property name="timestamp" value="${canal.instance.standby2.timestamp}" />
<property name="gtid" value="${canal.instance.standby2.gtid}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
<property name="filterDmlInsert" value="${canal.instance.filter.dml.insert:false}" />
<property name="filterDmlUpdate" value="${canal.instance.filter.dml.update:false}" />
<property name="filterDmlDelete" value="${canal.instance.filter.dml.delete:false}" />
<property name="filterRows" value="${canal.instance.filter.rows:false}" />
<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />

<!-- parallel parser -->
<property name="parallel" value="${canal.instance.parser.parallel:true}" />
<property name="parallelThreadSize" value="${canal.instance.parser.parallelThreadSize}" />
<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />

<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
</bean>

<bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">
<property name="topic" value="${canal.mq.topic}" />
<property name="dynamicTopic" value="${canal.mq.dynamicTopic}" />
<property name="partition" value="${canal.mq.partition}" />
<property name="partitionsNum" value="${canal.mq.partitionsNum}" />
<property name="partitionHash" value="${canal.mq.partitionHash}" />
<property name="dynamicTopicPartitionNum" value="${canal.mq.dynamicTopicPartitionNum}" />
<property name="enableDynamicQueuePartition" value="${canal.mq.enableDynamicQueuePartition}" />
</bean>
</beans>

+ 185
- 0
smtweb-framework/canal/server/target/classes/spring/memory-instance.xml View File

@@ -0,0 +1,185 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">

<import resource="classpath:spring/base-instance.xml" />

<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
<property name="destination" value="${canal.instance.destination}" />
<property name="eventParser">
<ref bean="eventParser" />
</property>
<property name="eventSink">
<ref bean="eventSink" />
</property>
<property name="eventStore">
<ref bean="eventStore" />
</property>
<property name="metaManager">
<ref bean="metaManager" />
</property>
<property name="alarmHandler">
<ref bean="alarmHandler" />
</property>
<property name="mqConfig">
<ref bean="mqConfig" />
</property>
</bean>

<!-- 报警处理类 -->
<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.LogAlarmHandler" />

<bean id="metaManager" class="com.alibaba.otter.canal.meta.MemoryMetaManager" />

<bean id="eventStore" class="com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer">
<property name="bufferSize" value="${canal.instance.memory.buffer.size:16384}" />
<property name="bufferMemUnit" value="${canal.instance.memory.buffer.memunit:1024}" />
<property name="batchMode" value="${canal.instance.memory.batch.mode:MEMSIZE}" />
<property name="ddlIsolation" value="${canal.instance.get.ddl.isolation:false}" />
<property name="raw" value="${canal.instance.memory.rawEntry:true}" />
</bean>

<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
</bean>

<bean id="eventParser" parent="baseEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
<property name="detectingIntervalInSeconds" value="${canal.instance.detecting.interval.time:5}" />
<property name="haController">
<bean class="com.alibaba.otter.canal.parse.ha.HeartBeatHAController">
<property name="detectingRetryTimes" value="${canal.instance.detecting.retry.threshold:3}" />
<property name="switchEnable" value="${canal.instance.detecting.heartbeatHaEnable:false}" />
</bean>
</property>

<property name="alarmHandler" ref="alarmHandler" />

<!-- 解析过滤处理 -->
<property name="eventFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
</bean>
</property>

<property name="eventBlackFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.black.regex:}" />
<constructor-arg index="1" value="false" />
</bean>
</property>
<property name="fieldFilter" value="${canal.instance.filter.field}" />
<property name="fieldBlackFilter" value="${canal.instance.filter.black.field}" />
<!-- 最大事务解析大小,超过该大小后事务将被切分为多个事务投递 -->
<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />

<!-- 网络链接参数 -->
<property name="receiveBufferSize" value="${canal.instance.network.receiveBufferSize:16384}" />
<property name="sendBufferSize" value="${canal.instance.network.sendBufferSize:16384}" />
<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />

<!-- 解析编码 -->
<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />

<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</property>

<!-- failover切换时回退的时间 -->
<property name="fallbackIntervalInSeconds" value="${canal.instance.fallbackIntervalInSeconds:60}" />

<!-- 解析数据库信息 -->
<property name="masterInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.master.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>
<property name="standbyInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo" init-method="initPwd">
<property name="address" value="${canal.instance.standby.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
</bean>
</property>

<!-- 解析起始位点 -->
<property name="masterPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.master.journal.name}" />
<property name="position" value="${canal.instance.master.position}" />
<property name="timestamp" value="${canal.instance.master.timestamp}" />
<property name="gtid" value="${canal.instance.master.gtid}" />
</bean>
</property>
<property name="standbyPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.standby.journal.name}" />
<property name="position" value="${canal.instance.standby.position}" />
<property name="timestamp" value="${canal.instance.standby.timestamp}" />
<property name="gtid" value="${canal.instance.standby.gtid}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
<property name="filterDmlInsert" value="${canal.instance.filter.dml.insert:false}" />
<property name="filterDmlUpdate" value="${canal.instance.filter.dml.update:false}" />
<property name="filterDmlDelete" value="${canal.instance.filter.dml.delete:false}" />
<property name="filterRows" value="${canal.instance.filter.rows:false}" />
<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />

<!--表结构相关-->
<property name="enableTsdb" value="${canal.instance.tsdb.enable:false}"/>
<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
<property name="tsdbSnapshotInterval" value="${canal.instance.tsdb.snapshot.interval:24}" />
<property name="tsdbSnapshotExpire" value="${canal.instance.tsdb.snapshot.expire:360}" />

<!--是否启用GTID模式-->
<property name="isGTIDMode" value="${canal.instance.gtidon:false}"/>

<!-- parallel parser -->
<property name="parallel" value="${canal.instance.parser.parallel:true}" />
<property name="parallelThreadSize" value="${canal.instance.parser.parallelThreadSize}" />
<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />

<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
</bean>

<bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">
<property name="topic" value="${canal.mq.topic}" />
<property name="dynamicTopic" value="${canal.mq.dynamicTopic}" />
<property name="partition" value="${canal.mq.partition}" />
<property name="partitionsNum" value="${canal.mq.partitionsNum}" />
<property name="partitionHash" value="${canal.mq.partitionHash}" />
<property name="dynamicTopicPartitionNum" value="${canal.mq.dynamicTopicPartitionNum}" />
<property name="enableDynamicQueuePartition" value="${canal.mq.enableDynamicQueuePartition}" />
</bean>
</beans>

+ 61
- 0
smtweb-framework/canal/server/target/classes/spring/tsdb/h2-tsdb.xml View File

@@ -0,0 +1,61 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">
<!-- properties -->
<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
<property name="ignoreResourceNotFound" value="true" />
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
<property name="locationNames">
<list>
<value>classpath:canal.properties</value>
<value>classpath:${canal.instance.destination:}/instance.properties</value>
</list>
</property>
</bean>
<!-- 基于db的实现 -->
<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta" destroy-method="destory">
<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
</bean>
<bean id="dataSource" class="com.alibaba.druid.pool.DruidDataSource" destroy-method="close">
<property name="driverClassName" value="org.h2.Driver" />
<property name="url" value="${canal.instance.tsdb.url:}" />
<property name="username" value="${canal.instance.tsdb.dbUsername:}" />
<property name="password" value="${canal.instance.tsdb.dbPassword:}" />
<property name="maxActive" value="30" />
<property name="initialSize" value="0" />
<property name="minIdle" value="1" />
<property name="maxWait" value="10000" />
<property name="timeBetweenEvictionRunsMillis" value="60000" />
<property name="minEvictableIdleTimeMillis" value="300000" />
<property name="testWhileIdle" value="true" />
<property name="testOnBorrow" value="false" />
<property name="testOnReturn" value="false" />
<property name="useUnfairLock" value="true" />
<property name="validationQuery" value="SELECT 1" />
</bean>

<bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">
<property name="dataSource" ref="dataSource"/>
<property name="configLocation" value="classpath:spring/tsdb/sql-map/sqlmap-config.xml"/>
</bean>

<bean id="metaHistoryDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO">
<property name="sqlSessionFactory" ref="sqlSessionFactory"/>
</bean>

<bean id="metaSnapshotDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO">
<property name="sqlSessionFactory" ref="sqlSessionFactory"/>
</bean>
</beans>

+ 63
- 0
smtweb-framework/canal/server/target/classes/spring/tsdb/mysql-tsdb.xml View File

@@ -0,0 +1,63 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">
<!-- properties -->
<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
<property name="ignoreResourceNotFound" value="true" />
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
<property name="locationNames">
<list>
<value>classpath:canal.properties</value>
<value>classpath:${canal.instance.destination:}/instance.properties</value>
</list>
</property>
</bean>
<!-- 基于db的实现 -->
<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta" destroy-method="destory">
<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
</bean>
<bean id="dataSource" class="com.alibaba.druid.pool.DruidDataSource" destroy-method="close">
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
<property name="url" value="${canal.instance.tsdb.url:}" />
<property name="username" value="${canal.instance.tsdb.dbUsername:}" />
<property name="password" value="${canal.instance.tsdb.dbPassword:}" />
<property name="maxActive" value="30" />
<property name="initialSize" value="0" />
<property name="minIdle" value="1" />
<property name="maxWait" value="10000" />
<property name="timeBetweenEvictionRunsMillis" value="60000" />
<property name="minEvictableIdleTimeMillis" value="300000" />
<property name="validationQuery" value="SELECT 1" />
<property name="exceptionSorterClassName" value="com.alibaba.druid.pool.vendor.MySqlExceptionSorter" />
<property name="validConnectionCheckerClassName" value="com.alibaba.druid.pool.vendor.MySqlValidConnectionChecker" />
<property name="testWhileIdle" value="true" />
<property name="testOnBorrow" value="false" />
<property name="testOnReturn" value="false" />
<property name="useUnfairLock" value="true" />
</bean>

<bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">
<property name="dataSource" ref="dataSource"/>
<property name="configLocation" value="classpath:spring/tsdb/sql-map/sqlmap-config.xml"/>
</bean>

<bean id="metaHistoryDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO">
<property name="sqlSessionFactory" ref="sqlSessionFactory"/>
</bean>

<bean id="metaSnapshotDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO">
<property name="sqlSessionFactory" ref="sqlSessionFactory"/>
</bean>
</beans>

+ 14
- 0
smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap-config.xml View File

@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE configuration PUBLIC "-//mybatis.org//DTD Config 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-config.dtd">
<configuration>
<typeAliases>
<typeAlias alias="metaHistoryDO" type="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDO"/>
<typeAlias alias="metaSnapshotDO" type="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDO"/>
</typeAliases>

<mappers>
<mapper resource="spring/tsdb/sql-map/sqlmap_history.xml"/>
<mapper resource="spring/tsdb/sql-map/sqlmap_snapshot.xml"/>
</mappers>
</configuration>

+ 44
- 0
smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_history.xml View File

@@ -0,0 +1,44 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryMapper">
<sql id="allColumns">
<![CDATA[
gmt_create,gmt_modified,destination,binlog_file,binlog_offest,binlog_master_id,binlog_timestamp,use_schema,sql_schema,sql_table,sql_text,sql_type,extra
]]>
</sql>
<sql id="allVOColumns">
<![CDATA[
a.id as id,a.gmt_create as gmtCreate,a.gmt_modified as gmtModified,
a.destination as destination,a.binlog_file as binlogFile,a.binlog_offest as binlogOffest,a.binlog_master_id as binlogMasterId,a.binlog_timestamp as binlogTimestamp,
a.use_schema as useSchema,a.sql_schema as sqlSchema,a.sql_table as sqlTable,a.sql_text as sqlText,a.sql_type as sqlType,a.extra as extra
]]>
</sql>

<select id="findByTimestamp" parameterType="java.util.Map" resultType="metaHistoryDO">
select
<include refid="allVOColumns"/>
from meta_history a
<![CDATA[
where destination = #{destination} and binlog_timestamp >= #{snapshotTimestamp} and binlog_timestamp <= #{timestamp}
order by binlog_timestamp asc,id asc
]]>
</select>

<insert id="insert" parameterType="metaHistoryDO">
insert into meta_history (<include refid="allColumns"/>)
values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#{destination},#{binlogFile},#{binlogOffest},#{binlogMasterId},#{binlogTimestamp},#{useSchema},#{sqlSchema},#{sqlTable},#{sqlText},#{sqlType},#{extra})
</insert>
<delete id="deleteByName" parameterType="java.util.Map">
delete from meta_history
where destination=#{destination}
</delete>


<delete id="deleteByTimestamp" parameterType="java.util.Map">
<![CDATA[
delete from meta_history
where destination=#{destination} and binlog_timestamp < #{timestamp}
]]>
</delete>
</mapper>

+ 48
- 0
smtweb-framework/canal/server/target/classes/spring/tsdb/sql-map/sqlmap_snapshot.xml View File

@@ -0,0 +1,48 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotMapper">
<sql id="allColumns">
<![CDATA[
gmt_create,gmt_modified,destination,binlog_file,binlog_offest,binlog_master_id,binlog_timestamp,data,extra
]]>
</sql>
<sql id="allVOColumns">
<![CDATA[
a.id as id,a.gmt_create as gmtCreate,a.gmt_modified as gmtModified,
a.destination as destination,a.binlog_file as binlogFile,a.binlog_offest as binlogOffest,a.binlog_master_id as binlogMasterId,a.binlog_timestamp as binlogTimestamp,a.data as data,a.extra as extra
]]>
</sql>

<select id="findByTimestamp" parameterType="java.util.Map" resultType="metaSnapshotDO">
select <include refid="allVOColumns"/>
<![CDATA[
from meta_snapshot a
where destination = #{destination} and binlog_timestamp < #{timestamp}
order by binlog_timestamp desc,id desc
limit 1
]]>
</select>

<insert id="insert" parameterType="metaSnapshotDO">
insert into meta_snapshot (<include refid="allColumns"/>)
values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#{destination},#{binlogFile},#{binlogOffest},#{binlogMasterId},#{binlogTimestamp},#{data},#{extra})
</insert>

<update id="update" parameterType="metaSnapshotDO">
update meta_snapshot set gmt_modified=now(),
binlog_file=#{binlogFile},binlog_offest=#{binlogOffest},binlog_master_id=#{binlogMasterId},binlog_timestamp=#{binlogTimestamp},data=#{data},extra=#{extra}
where destination=#{destination} and binlog_timestamp=0
</update>

<delete id="deleteByName" parameterType="java.util.Map">
delete from meta_snapshot
where destination=#{destination}
</delete>

<delete id="deleteByTimestamp" parameterType="java.util.Map">
<![CDATA[
delete from meta_snapshot
where destination=#{destination} and binlog_timestamp < #{timestamp} and binlog_timestamp > 0
]]>
</delete>
</mapper>

+ 39
- 0
smtweb-framework/canal/server/target/classes/spring/tsdb/sql/create_table.sql View File

@@ -0,0 +1,39 @@
CREATE TABLE IF NOT EXISTS `meta_snapshot` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',
`gmt_create` datetime NOT NULL COMMENT '创建时间',
`gmt_modified` datetime NOT NULL COMMENT '修改时间',
`destination` varchar(128) DEFAULT NULL COMMENT '通道名称',
`binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名',
`binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量',
`binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id',
`binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳',
`data` longtext DEFAULT NULL COMMENT '表结构数据',
`extra` text DEFAULT NULL COMMENT '额外的扩展信息',
PRIMARY KEY (`id`),
UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`),
KEY `destination` (`destination`),
KEY `destination_timestamp` (`destination`,`binlog_timestamp`),
KEY `gmt_modified` (`gmt_modified`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='表结构记录表快照表';

CREATE TABLE IF NOT EXISTS `meta_history` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',
`gmt_create` datetime NOT NULL COMMENT '创建时间',
`gmt_modified` datetime NOT NULL COMMENT '修改时间',
`destination` varchar(128) DEFAULT NULL COMMENT '通道名称',
`binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名',
`binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量',
`binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id',
`binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳',
`use_schema` varchar(1024) DEFAULT NULL COMMENT '执行sql时对应的schema',
`sql_schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema',
`sql_table` varchar(1024) DEFAULT NULL COMMENT '对应的table',
`sql_text` longtext DEFAULT NULL COMMENT '执行的sql',
`sql_type` varchar(256) DEFAULT NULL COMMENT 'sql类型',
`extra` text DEFAULT NULL COMMENT '额外的扩展信息',
PRIMARY KEY (`id`),
UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`),
KEY `destination` (`destination`),
KEY `destination_timestamp` (`destination`,`binlog_timestamp`),
KEY `gmt_modified` (`gmt_modified`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='表结构变化明细表';

+ 22
- 0
smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst View File

@@ -0,0 +1,22 @@
cc\smtweb\system\canal\CanalController.class
cc\smtweb\system\canal\CanalConstants.class
cc\smtweb\system\canal\InstanceConfig.class
cc\smtweb\system\canal\CanalController$1.class
cc\smtweb\system\canal\monitor\SpringInstanceConfigMonitor.class
cc\smtweb\system\canal\CanalLauncher.class
cc\smtweb\system\canal\monitor\SpringInstanceConfigMonitor$FileInfo.class
cc\smtweb\system\canal\client\CanalClient.class
cc\smtweb\system\canal\CanalController$3.class
cc\smtweb\system\canal\admin\CanalAdminController.class
cc\smtweb\system\canal\CanalLauncher$1.class
cc\smtweb\system\canal\monitor\InstanceConfigMonitor.class
cc\smtweb\system\canal\CanalStarter.class
cc\smtweb\system\canal\CanalApplication.class
cc\smtweb\system\canal\CanalController$1$1.class
cc\smtweb\system\canal\monitor\InstanceAction.class
cc\smtweb\system\canal\client\CanalClient$1.class
cc\smtweb\system\canal\CanalController$2.class
cc\smtweb\system\canal\monitor\SpringInstanceConfigMonitor$InstanceConfigFiles.class
cc\smtweb\system\canal\client\CanalVO.class
cc\smtweb\system\canal\InstanceConfig$InstanceMode.class
cc\smtweb\system\canal\monitor\ManagerInstanceConfigMonitor.class

+ 13
- 0
smtweb-framework/canal/server/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst View File

@@ -0,0 +1,13 @@
E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\CanalConstants.java
E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\CanalStarter.java
E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\InstanceConfig.java
E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\CanalApplication.java
E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\CanalLauncher.java
E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\client\CanalClient.java
E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\monitor\InstanceAction.java
E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\monitor\InstanceConfigMonitor.java
E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\CanalController.java
E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\monitor\ManagerInstanceConfigMonitor.java
E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\admin\CanalAdminController.java
E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\monitor\SpringInstanceConfigMonitor.java
E:\JJKJ\Code\VisualDesign\smtweb2\smtweb-framework\canal\src\main\java\cc\smtweb\system\canal\client\CanalVO.java

+ 34
- 0
smtweb-framework/core/src/main/java/cc/smtweb/framework/core/CoreApplicationClosedListener.java View File

@@ -0,0 +1,34 @@
package cc.smtweb.framework.core;

import cc.smtweb.framework.core.common.SwConsts;
import cc.smtweb.framework.core.mvc.controller.IStartListener;
import cc.smtweb.framework.core.mvc.controller.scan.ApplicationScanner;
import cc.smtweb.framework.core.mvc.controller.scan.BeanManager;
import cc.smtweb.framework.core.systask.SysServiceFactory;
import cc.smtweb.framework.core.systask.WebStartedEvent;
import lombok.SneakyThrows;
import org.springframework.boot.context.event.ApplicationStartedEvent;
import org.springframework.context.ApplicationListener;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.event.ContextClosedEvent;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;

import java.util.Comparator;
import java.util.List;

/**
* 服务停止监听
*/
@Component
public class CoreApplicationClosedListener implements ApplicationListener<ContextClosedEvent> {
@Override
public void onApplicationEvent(ContextClosedEvent event) {
SysServiceFactory.getInstance().stop();
List<IStartListener> list = BeanManager.getInstance().getStartListeners();
list.sort(Comparator.comparingInt(IStartListener::order));
for (IStartListener sl : list) {
sl.close();
}
}
}

+ 2
- 0
smtweb-framework/core/src/main/java/cc/smtweb/framework/core/CoreApplicationStartedListener.java View File

@@ -4,6 +4,7 @@ import cc.smtweb.framework.core.common.SwConsts;
import cc.smtweb.framework.core.mvc.controller.IStartListener;
import cc.smtweb.framework.core.mvc.controller.scan.ApplicationScanner;
import cc.smtweb.framework.core.mvc.controller.scan.BeanManager;
import cc.smtweb.framework.core.systask.SysServiceFactory;
import cc.smtweb.framework.core.systask.WebStartedEvent;
import lombok.SneakyThrows;
import org.springframework.boot.context.event.ApplicationStartedEvent;
@@ -38,6 +39,7 @@ public class CoreApplicationStartedListener implements ApplicationListener<Appli
for (IStartListener sl : list) {
sl.run();
}
SysServiceFactory.getInstance().start();
// 通知 controller 正式使用
applicationContext.publishEvent(new WebStartedEvent());
SwConsts.SysParam.SYS_STARTED = true;


+ 0
- 5
smtweb-framework/core/src/main/java/cc/smtweb/framework/core/CoreAutoConfiguration.java View File

@@ -37,9 +37,4 @@ public class CoreAutoConfiguration implements WebMvcConfigurer {
public ControllerConfig coreControllerConfig() {
return new ControllerConfig("core", "cc.smtweb.framework.core");
}

@Override
public void addInterceptors(InterceptorRegistry registry) {
registry.addInterceptor(new CoreInterceptor());
}
}

+ 9
- 1
smtweb-framework/core/src/main/java/cc/smtweb/framework/core/common/SwConsts.java View File

@@ -16,7 +16,15 @@ public interface SwConsts {
//是否执行定时任务
public static boolean enableJob = false;
}

//事件常量
class EventConst {
// 登录映射
public static final String LOGIN_MAPPING = "LOGIN_MAPPING";
//登录成功后事件
public static final String LOGIN_SUCCESS = "LOGIN_SUCCESS";
//登录失败后事件
public static final String LOGIN_FAIL = "LOGIN_FAIL";
}
//错误码
interface ErrorCode {
//未登录


+ 16
- 0
smtweb-framework/core/src/main/java/cc/smtweb/framework/core/container/base/IListenerIntf.java View File

@@ -0,0 +1,16 @@
package cc.smtweb.framework.core.container.base;

/**
* Created by Akmm at 12-3-22 下午4:52
* 事件监听
*/
public interface IListenerIntf {
/**
* 监听动作
*
* @param parameter 输入参数
* @param reslt 返回参数
* @throws Exception
*/
void doWork(SwParameter parameter, SwReturn result) throws Exception;
}

+ 40
- 0
smtweb-framework/core/src/main/java/cc/smtweb/framework/core/container/base/SwParameter.java View File

@@ -0,0 +1,40 @@
package cc.smtweb.framework.core.container.base;

/**
* Created by Akmm at 2011-1-12 9:41:41
* 输入参数
*/
public class SwParameter extends java.util.HashMap<String, Object> {

private Object singleParam;

public <T> T getSingleParam() {
return (T)singleParam;
}

public void setSingleParam(Object singleParam) {
this.singleParam = singleParam;
}

/**
* 得到参数值
*
* @param name 参数名
* @param <T> 参数类型
* @return 参数值
*/
@SuppressWarnings({"unchecked"})
public <T> T getParameter(String name) {
return (T) get(name);
}

/**
* 设置参数值
*
* @param name 参数名
* @param value 参数值
*/
public void setParameter(String name, Object value) {
put(name, value);
}
}

+ 166
- 0
smtweb-framework/core/src/main/java/cc/smtweb/framework/core/container/base/SwReturn.java View File

@@ -0,0 +1,166 @@
package cc.smtweb.framework.core.container.base;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
* Created by Akmm at 2011-1-12 9:45:04
* 插件服务方法的返回参数
*/
@SuppressWarnings("UnusedDeclaration")
public class SwReturn {
/**
* 是否执行成功
*/
private boolean isSuccess;
/**
* 提示信息列表
*/
private List<String> messages = null;
/**
* 执行结果参数
*/
private Map<String, Object> map = new HashMap<>();

/**
* @param isSuccess
* @param message
*/
public SwReturn(boolean isSuccess, String message) {
this.isSuccess = isSuccess;
this.addMessage(message);
}

public SwReturn(Integer errCode, String message) {
this.isSuccess = false;
this.addMessage(message);
this.setParameter("errCode", errCode);
}

public SwReturn(String errCode, String message) {
this.isSuccess = false;
this.addMessage(message);
this.setParameter("errCode", errCode);
}

/**
*/
public SwReturn() {
this(true, null);
}

/**
* 设置参数
*
* @param name
* @param value
* @pdOid 590f34f1-b661-4a64-8dbb-b1c9a4ec931e
*/
public void setParameter(String name, Object value) {
map.put(name, value);
}

/**
* 获取参数
*
* @param name
* @pdOid a0df6724-734d-4c5d-b156-e96010436ee8
*/
@SuppressWarnings({"unchecked"})
public <T> T getParameter(String name) {
return (T) map.get(name);
}

public boolean containsParameter(String name) {
return map.containsKey(name);
}

/**
* 是否执行成功
*/
public boolean isSuccess() {
return isSuccess;
}

/**
* @param success
* @pdOid bf4fdbfe-a134-43b7-9496-578fb61ab106
*/
public void setSuccess(boolean success) {
isSuccess = success;
}

/**
* 得到提示信息,字符串格式,有多个提示,则以回车符分隔
*/
public String getMessageStr() {
if (this.messages == null) return null;
StringBuilder sb = new StringBuilder(32);
for (String s : messages) {
sb.append("\n").append(s);
}
return sb.substring(1);
}

/**
* 得到结果参数Map
*/
public Map<String, Object> getMap() {
return map;
}

/**
* 增加一条提示信息
*
* @param msg
*/
public void addMessage(String msg) {
if (this.messages == null)
this.messages = new ArrayList<>();
if (msg != null) this.messages.add(msg);
}

/**
* 设置消息
*
* @param msg
*/
public void setMessage(String msg) {
clearMessage();
addMessage(msg);
}

/**
* 清空所有提示信息
*/
public void clearMessage() {
if (this.messages != null) this.messages.clear();
}

/**
* 得到提示信息列表
*
* @return
*/
public List<String> getMessages() {
return messages;
}

/**
* 设置为执行失败,且增加错误信息
*
* @param errmsg
*/
public void setError(String errmsg) {
this.isSuccess = false;
addMessage(errmsg);
}
public int getErrorCode() {
if (!isSuccess() && containsParameter("errCode")) {
return (Integer) getParameter("errCode");
}
return 0;
}
}

+ 113
- 0
smtweb-framework/core/src/main/java/cc/smtweb/framework/core/container/factory/ListenerFactory.java View File

@@ -0,0 +1,113 @@
package cc.smtweb.framework.core.container.factory;

import cc.smtweb.framework.core.container.base.IListenerIntf;
import cc.smtweb.framework.core.container.base.SwParameter;
import cc.smtweb.framework.core.container.base.SwReturn;
import cc.smtweb.framework.core.systask.SysThreadPool;
import cc.smtweb.framework.core.systask.SysThreadWorker;

import java.util.List;
import java.util.Map;
import java.util.Vector;
import java.util.concurrent.ConcurrentHashMap;

/**
* Created by Akmm at 12-3-22 下午4:45
* 事件监听工厂
*/
public class ListenerFactory {
private static Map<String, List<IListenerIntf>> map;

static {
map = new ConcurrentHashMap<>(64);
}

/**
* 注册监听
*
* @param event 事件名
* @param listener 监听类
*/
public static void reg(String event, IListenerIntf listener) {
List<IListenerIntf> list = map.get(event);
if (list == null) {
synchronized (ListenerFactory.class) {
list = map.get(event);
if (list == null) {
list = new Vector<>();
map.put(event, list);
}
}
}
list.add(listener);
}

/**
* 取消注册监听
*
* @param event 事件名
* @param listener 监听类
*/
public static void remove(String event, IListenerIntf listener) {
List<IListenerIntf> list = map.get(event);
if (list != null) list.remove(listener);
}

/**
* 取消指定事件所有监听
*
* @param event 事件名
*/
public static void clear(String event) {
map.remove(event);
}

/**
* 执行监听
*
* @param event 事件名
* @param param 事件参数
* @param result 监听程序的执行结果
* @throws Exception
*/
public static void fireListener(String event, SwParameter param, SwReturn result)throws Exception{
List<IListenerIntf> list = map.get(event);
if (list == null) return;
for (IListenerIntf listener : list) {
listener.doWork(param, result);
if (!result.isSuccess()) return;
}
}

/**
* 执行监听
*
* @param event 事件名
* @param param 事件参数
* @throws Exception
*/
public static void fireListener(String event, SwParameter param) throws Exception {
SwReturn result = new SwReturn();
fireListener(event, param, result);
if (!result.isSuccess()) throw new Exception(result.getMessageStr());
}
/**
* 执行监听\异步执行
*
* @param event 事件名
* @param param 事件参数
* @throws Exception
*/
public static void fireListenerAsyc(String event, SwParameter param){
SysThreadPool.getInstance().addTask(new SysThreadWorker() {
@Override
public void localWork() throws Exception {
try {
fireListener(event, param);
} catch (Exception e) {
e.printStackTrace();
}
}
});
}
}

+ 13
- 2
smtweb-framework/core/src/main/java/cc/smtweb/framework/core/db/jdbc/JdbcEngine.java View File

@@ -356,7 +356,7 @@ public class JdbcEngine {
}

//独立事务
public void doTransSingle(IDbWorker worker) throws Exception {
public void doTransSingle(IDbWorker worker) throws SwException {
JdbcTrans jdbcTrans = openTrans();
try {
worker.work();
@@ -364,11 +364,22 @@ public class JdbcEngine {
} catch (Exception e) {
jdbcTrans.rollback();
worker.doAfterDbRollback();
throw e;
throw new SwException(e);
}
worker.doAfterDbCommit();
}

public void doTransSingle(ISimpleWorker worker) throws SwException {
JdbcTrans jdbcTrans = openTrans();
try {
worker.work();
jdbcTrans.commit();
} catch (Exception e) {
jdbcTrans.rollback();
throw new SwException(e);
}
}


/**
* 获取原始数据库连接执行命令


+ 7
- 7
smtweb-framework/core/src/main/java/cc/smtweb/framework/core/mvc/config/WebMvcConfig.java View File

@@ -1,5 +1,6 @@
package cc.smtweb.framework.core.mvc.config;

import cc.smtweb.framework.core.CoreInterceptor;
import cc.smtweb.framework.core.cache.CacheManager;
import cc.smtweb.framework.core.cache.redis.RedisManager;
import cc.smtweb.framework.core.db.jdbc.IdGenerator;
@@ -19,10 +20,7 @@ import org.springframework.context.annotation.Primary;
import org.springframework.http.MediaType;
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
import org.springframework.web.method.support.HandlerMethodArgumentResolver;
import org.springframework.web.servlet.config.annotation.ContentNegotiationConfigurer;
import org.springframework.web.servlet.config.annotation.CorsRegistry;
import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
import org.springframework.web.servlet.config.annotation.*;

import java.util.List;

@@ -49,9 +47,6 @@ public class WebMvcConfig implements WebMvcConfigurer {

private static final String TRUE_VALUE = "true";

@Autowired
private RedisManager redisManager;

@Bean
public SessionManager sessionManager(RedisManager redisManager, IdGenerator idGenerator) {
return new SessionManager(redisManager, idGenerator);
@@ -68,6 +63,11 @@ public class WebMvcConfig implements WebMvcConfigurer {
}

@Override
public void addInterceptors(InterceptorRegistry registry) {
registry.addInterceptor(new CoreInterceptor());
}

@Override
public void addCorsMappings(CorsRegistry registry) {
// 2.4 以前使用 .allowedOrigins("*")
registry.addMapping("/**")


+ 3
- 0
smtweb-framework/core/src/main/java/cc/smtweb/framework/core/mvc/controller/IStartListener.java View File

@@ -13,4 +13,7 @@ public interface IStartListener {

//启动事件
void run();

//服务停止时
default void close(){}
}

+ 1
- 1
smtweb-framework/core/src/main/java/cc/smtweb/framework/core/session/UserSession.java View File

@@ -51,7 +51,7 @@ public class UserSession implements Serializable {
this.terminalType = terminalType;

HttpSession session = request.getSession(true);
if (user_id > 0) session.setAttribute(SwConsts._LOGIN_USER_ID_IN_SESSION, getUserKey());
if (user_id > 0L) session.setAttribute(SwConsts._LOGIN_USER_ID_IN_SESSION, getUserKey());
this.sessionId = session.getId();
this.loginTimeMillis = System.currentTimeMillis();
this.lastTimeMillis = this.loginTimeMillis;


+ 9
- 12
smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/BaseSysService.java View File

@@ -23,17 +23,14 @@ public abstract class BaseSysService implements Runnable {
public void run() {
try {
final String id = this.getClass().getName();
SingleRequestHelper.singleRequest(SwConsts.REDIS_KEY_BASE_JOB, id, "任务执行中,本次忽略.........", new SingleRequestHelper.ISingleWork() {
@Override
public void doSingleWork() throws Exception {
long time = System.currentTimeMillis();
if (isPrintLog()) {
log.debug(getTitle() + ":::开始执行.........");
}
work();
if (isPrintLog()) {
log.debug(getTitle() + ":::执行完成,耗时:" + (System.currentTimeMillis() - time) + "毫秒.........");
}
SingleRequestHelper.singleRequest(SwConsts.REDIS_KEY_BASE_JOB, id, "任务执行中,本次忽略.........", () -> {
long time = System.currentTimeMillis();
if (isPrintLog()) {
log.debug(getTitle() + ":::开始执行.........");
}
work();
if (isPrintLog()) {
log.debug(getTitle() + ":::执行完成,耗时:" + (System.currentTimeMillis() - time) + "毫秒.........");
}
});
} catch (Exception e) {
@@ -42,5 +39,5 @@ public abstract class BaseSysService implements Runnable {
}

//主要要干的活
protected abstract void work() throws Exception;
protected abstract void work();
}

+ 4
- 3
smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SingleRequestHelper.java View File

@@ -5,18 +5,19 @@ import cc.smtweb.framework.core.exception.SwException;
import cc.smtweb.framework.core.util.DateUtil;
import cc.smtweb.framework.core.util.StringUtil;

//redis控制的互斥执行,一般用于定时任务,避免多服务器重复执行
public class SingleRequestHelper {

public static interface ISingleWork {
void doSingleWork() throws Exception;
void doSingleWork();

}

public static void singleRequest(String region_key, String key, ISingleWork singleWork) throws Exception {
public static void singleRequest(String region_key, String key, ISingleWork singleWork) {
singleRequest(region_key, key, "其他人正在执行该操作,请等待!", singleWork);
}

public static void singleRequest(String region_key, String key, String tip_msg, ISingleWork singleWork) throws Exception {
public static void singleRequest(String region_key, String key, String tip_msg, ISingleWork singleWork) {
boolean clearCache = false;
try {
String lastTime = RedisManager.getInstance().hGet(region_key, key, String.class);


+ 2
- 2
smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysServiceFactory.java View File

@@ -39,7 +39,7 @@ public class SysServiceFactory {


//启动任务
protected void start() {
public void start() {
if (schedule != null) {
stop();
}
@@ -52,7 +52,7 @@ public class SysServiceFactory {
}

//停止任务
protected void stop() {
public void stop() {
schedule.shutdown();
}
}

+ 1
- 1
smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadPool.java View File

@@ -77,7 +77,7 @@ public class SysThreadPool {
public static void main(String[] args) {
for (int i = 1; i < 100; i++) {
int j = i;
SysThreadPool.getInstance().addTask(new SysThreadWorker(null, null) {
SysThreadPool.getInstance().addTask(new SysThreadWorker() {
@Override
public void localWork() throws Exception {
Thread.sleep(1000);


+ 3
- 11
smtweb-framework/core/src/main/java/cc/smtweb/framework/core/systask/SysThreadWorker.java View File

@@ -6,20 +6,14 @@ import lombok.extern.slf4j.Slf4j;

@Slf4j
public abstract class SysThreadWorker implements Runnable {
private String loginPartyId;

private String loginOwnerPartyId;

private String bizName;

protected SysThreadWorker(String loginPartyId, String loginOwnerPartyId) {
this.loginPartyId = loginPartyId;
this.loginOwnerPartyId = loginOwnerPartyId;
protected SysThreadWorker() {

}

protected SysThreadWorker(String loginPartyId, String loginOwnerPartyId, String bizName) {
this.loginPartyId = loginPartyId;
this.loginOwnerPartyId = loginOwnerPartyId;
protected SysThreadWorker(String bizName) {
this.bizName = StringUtil.checkNull(bizName);
}

@@ -27,8 +21,6 @@ public abstract class SysThreadWorker implements Runnable {
@Override
public void run() {
try {
// SysParams.setLoginOwnerPartyId(loginOwnerPartyId);
// SysParams.setLoginPartyId(loginPartyId);
localWork();
} catch (Exception e) {
log.error("DfpPool handle error:", e);


Loading…
Cancel
Save