From 37ed31e3d845cb3907377e6b33ca1a14f556a50a Mon Sep 17 00:00:00 2001 From: GuangJie1 Date: Mon, 8 Sep 2025 16:34:37 +0800 Subject: [PATCH] update canal --- Others/canal/1.1.8/24.03-lts-sp2/Dockerfile | 24 +++ .../1.1.8/24.03-lts-sp2/instance.properties | 75 ++++++++++ Others/canal/1.1.8/24.03-lts-sp2/startup.sh | 137 ++++++++++++++++++ Others/canal/README.md | 87 +++-------- Others/canal/doc/image-info.yml | 88 ++++------- Others/canal/meta.yml | 2 + 6 files changed, 286 insertions(+), 127 deletions(-) create mode 100644 Others/canal/1.1.8/24.03-lts-sp2/Dockerfile create mode 100644 Others/canal/1.1.8/24.03-lts-sp2/instance.properties create mode 100644 Others/canal/1.1.8/24.03-lts-sp2/startup.sh diff --git a/Others/canal/1.1.8/24.03-lts-sp2/Dockerfile b/Others/canal/1.1.8/24.03-lts-sp2/Dockerfile new file mode 100644 index 00000000..affeec23 --- /dev/null +++ b/Others/canal/1.1.8/24.03-lts-sp2/Dockerfile @@ -0,0 +1,24 @@ +ARG BASE=openeuler/openeuler:24.03-lts-sp2 +FROM ${BASE} +ARG VERSION=1.1.8 + +RUN yum update -y && \ + yum install -y java-1.8.0-openjdk wget file hostname && \ + yum clean all + +RUN mkdir -p /opt/canal && \ + wget https://github.com/alibaba/canal/releases/download/canal-${VERSION}/canal.deployer-${VERSION}.tar.gz && \ + tar zxvf canal.deployer-${VERSION}.tar.gz -C /opt/canal && \ + rm -f canal.deployer-${VERSION}.tar.gz + +COPY instance.properties /opt/canal/conf/example/instance.properties + +WORKDIR /opt/canal + +EXPOSE 11110 11111 11112 + +COPY startup.sh /opt/canal/bin/startup.sh + +RUN chmod +x /opt/canal/bin/startup.sh + +ENTRYPOINT /opt/canal/bin/startup.sh \ No newline at end of file diff --git a/Others/canal/1.1.8/24.03-lts-sp2/instance.properties b/Others/canal/1.1.8/24.03-lts-sp2/instance.properties new file mode 100644 index 00000000..e60554eb --- /dev/null +++ b/Others/canal/1.1.8/24.03-lts-sp2/instance.properties @@ -0,0 +1,75 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +################################################# +## mysql serverId , v1.0.26+ will autoGen +# canal.instance.mysql.slaveId=0 + +# enable gtid use true/false +canal.instance.gtidon=false + +# rds oss binlog +canal.instance.rds.accesskey= +canal.instance.rds.secretkey= +canal.instance.rds.instanceId= + +# position info +canal.instance.master.address=127.0.0.1:3306 +canal.instance.master.journal.name= +canal.instance.master.position= +canal.instance.master.timestamp= +canal.instance.master.gtid= + +# multi stream for polardbx +canal.instance.multi.stream.on=false + +# ssl +#canal.instance.master.sslMode=DISABLED +#canal.instance.master.tlsVersions= +#canal.instance.master.trustCertificateKeyStoreType= +#canal.instance.master.trustCertificateKeyStoreUrl= +#canal.instance.master.trustCertificateKeyStorePassword= +#canal.instance.master.clientCertificateKeyStoreType= +#canal.instance.master.clientCertificateKeyStoreUrl= +#canal.instance.master.clientCertificateKeyStorePassword= + +# table meta tsdb info +canal.instance.tsdb.enable=true +#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb +#canal.instance.tsdb.dbUsername=canal +#canal.instance.tsdb.dbPassword=canal + +#canal.instance.standby.address = +#canal.instance.standby.journal.name = +#canal.instance.standby.position = +#canal.instance.standby.timestamp = +#canal.instance.standby.gtid= + +# username/password +canal.instance.dbUsername=canal +canal.instance.dbPassword=canal +canal.instance.connectionCharset = UTF-8 +# enable druid Decrypt database password +canal.instance.enableDruid=false +#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ== + +# table regex +canal.instance.filter.regex=.*\\..* +# table black regex +canal.instance.filter.black.regex=mysql\\.slave_.* +# table field filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2) +#canal.instance.filter.field=test1.t_product:id/subject/keywords,test2.t_company:id/name/contact/ch +# table field black filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2) +#canal.instance.filter.black.field=test1.t_product:subject/product_image,test2.t_company:id/name/contact/ch + +# mq config +canal.mq.topic=example +# dynamic topic route by schema or table regex +#canal.mq.dynamicTopic=mytest1.user,topic2:mytest2\\..*,.*\\..* +canal.mq.partition=0 +# hash partition config +#canal.mq.enableDynamicQueuePartition=false +#canal.mq.partitionsNum=3 +#canal.mq.dynamicTopicPartitionNum=test.*:4,mycanal:6 +#canal.mq.partitionHash=test.table:id^name,.*\\..* +################################################# \ No newline at end of file diff --git a/Others/canal/1.1.8/24.03-lts-sp2/startup.sh b/Others/canal/1.1.8/24.03-lts-sp2/startup.sh new file mode 100644 index 00000000..92b2876b --- /dev/null +++ b/Others/canal/1.1.8/24.03-lts-sp2/startup.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +current_path=`pwd` +case "`uname`" in + Linux) + bin_abs_path=$(readlink -f $(dirname $0)) + ;; + *) + bin_abs_path=`cd $(dirname $0); pwd` + ;; +esac +base=${bin_abs_path}/.. +canal_conf=$base/conf/canal.properties +canal_local_conf=$base/conf/canal_local.properties +logback_configurationFile=$base/conf/logback.xml +export LANG=en_US.UTF-8 +export BASE=$base + +if [ -f $base/bin/canal.pid ] ; then + echo "found canal.pid , Please run stop.sh first ,then startup.sh" 2>&2 + exit 1 +fi + +if [ ! -d $base/logs/canal ] ; then + mkdir -p $base/logs/canal +fi + +## set java path +if [ -z "$JAVA" ] ; then + JAVA=$(which java) +fi + +ALIBABA_JAVA="/usr/alibaba/java/bin/java" +TAOBAO_JAVA="/opt/taobao/java/bin/java" +if [ -z "$JAVA" ]; then + if [ -f $ALIBABA_JAVA ] ; then + JAVA=$ALIBABA_JAVA + elif [ -f $TAOBAO_JAVA ] ; then + JAVA=$TAOBAO_JAVA + else + echo "Cannot find a Java JDK. Please set either set JAVA or put java (>=1.5) in your PATH." 2>&2 + exit 1 + fi +fi + +case "$#" +in +0 ) + ;; +1 ) + var=$* + if [ "$var" = "local" ]; then + canal_conf=$canal_local_conf + else + if [ -f $var ] ; then + canal_conf=$var + else + echo "THE PARAMETER IS NOT CORRECT.PLEASE CHECK AGAIN." + exit + fi + fi;; +2 ) + var=$1 + if [ "$var" = "local" ]; then + canal_conf=$canal_local_conf + else + if [ -f $var ] ; then + canal_conf=$var + else + if [ "$1" = "debug" ]; then + DEBUG_PORT=$2 + DEBUG_SUSPEND="n" + JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND" + fi + fi + fi;; +* ) + echo "THE PARAMETERS MUST BE TWO OR LESS.PLEASE CHECK AGAIN." + exit;; +esac + +JavaVersion=`$JAVA -version 2>&1 |awk 'NR==1{ gsub(/"/,""); print $3 }' | awk -F '.' '{print $1}'` +str=`file -L $JAVA | grep 64-bit` +JAVA_OPTS="$JAVA_OPTS -Xss1m -XX:-UseBiasedLocking -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$base/logs" + +if [ $JavaVersion -ge 11 ] ; then + #JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$base_log/gc.log:time " + JAVA_OPTS="$JAVA_OPTS" +else + #JAVA_OPTS="$JAVA_OPTS -Xloggc:$base/logs/canal/gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime" + JAVA_OPTS="$JAVA_OPTS -XX:+AggressiveOpts -XX:+UseFastAccessorMethods -XX:+PrintAdaptiveSizePolicy -XX:+PrintTenuringDistribution" +fi + +if [ -n "$str" ]; then + if [ $JavaVersion -ge 11 ] ; then + # For G1 + JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS" + else + JAVA_OPTS="-server -Xms2g -Xmx3g -Xmn1g -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS" + fi +else + JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m $JAVA_OPTS" +fi + +JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8" +CANAL_OPTS="-DappName=otter-canal -Dlogback.configurationFile=$logback_configurationFile -Dcanal.conf=$canal_conf" + +if [ -e $canal_conf -a -e $logback_configurationFile ] +then + + for i in $base/lib/*; + do CLASSPATH=$i:"$CLASSPATH"; + done + CLASSPATH="$base/conf:$CLASSPATH"; + + echo "cd to $bin_abs_path for workaround relative path" + cd $bin_abs_path + + echo LOG CONFIGURATION : $logback_configurationFile + echo canal conf : $canal_conf + echo CLASSPATH :$CLASSPATH + $JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.deployer.CanalLauncher 1>>$base/logs/canal/canal_stdout.log 2>&1 & + echo $! > $base/bin/canal.pid + + echo "cd to $current_path for continue" + cd $current_path +else + echo "canal conf("$canal_conf") OR log configration file($logback_configurationFile) is not exist,please create then first!" +fi + + +LOG_FILE=/opt/canal/logs/example/example.log +mkdir -p $(dirname "$LOG_FILE") +touch "$LOG_FILE" +tail -f "$LOG_FILE" \ No newline at end of file diff --git a/Others/canal/README.md b/Others/canal/README.md index a2c4a5cc..f3a387e9 100644 --- a/Others/canal/README.md +++ b/Others/canal/README.md @@ -14,79 +14,38 @@ Canal is an open-source project developed by Alibaba for real-time data synchron # Supported tags and respective Dockerfile links The tag of each canal docker image is consist of the version of canal and the version of basic image. The details are as follows -| Tags | Currently | Architectures| -|------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------|--| -| [1.1.8-oe2403sp1](https://gitee.com/openeuler/openeuler-docker-images/blob/master/Others/canal/1.1.8/24.03-lts-sp1/Dockerfile) | canal 1.1.8 on openEuler 24.03-LTS-SP1 | amd64, arm64 | +| Tags | Currently | Architectures | +|--------------------------------------------------------------------------------------------------------------------------------|----------------------------------------|---------------| +| [1.1.8-oe2403sp1](https://gitee.com/openeuler/openeuler-docker-images/blob/master/Others/canal/1.1.8/24.03-lts-sp1/Dockerfile) | canal 1.1.8 on openEuler 24.03-LTS-SP1 | amd64, arm64 | +| [1.1.8-oe2403sp2](https://gitee.com/openeuler/openeuler-docker-images/blob/master/Others/canal/1.1.8/24.03-lts-sp2/Dockerfile) | canal 1.1.8 on openEuler 24.03-LTS-SP2 | amd64, arm64 | # Usage -- Add canal client dependency - Add the following dependency to your `pom.xml` to include the `Canal Java Client`. - ``` - - com.alibaba.otter - canal.client - 1.1.0 - - ``` - -- Create a standard maven project - - You can create a standard Maven project using the Maven Archetype Plugin. +- Preparation - **For older Maven versions:** - ``` - mvn archetype:create -DgroupId=com.alibaba.otter -DartifactId=canal.sample - ``` - **For Maven 3.0.5 and above(recommended):** + For a self-hosted MySQL, you need to enable binlog and configure the binlog format as `ROW`. The `my.cnf` should be configured as follows: ``` - mvn archetype:generate -DgroupId=com.alibaba.otter -DartifactId=canal.sample + [mysqld] + log-bin=mysql-bin # Enable binlog + binlog-format=ROW # Set binlog format to ROW + server_id=1 # Server ID required for MySQL replication; do not conflict with Canal's slaveId ``` - This generates a basic Maven project structure. + **Note:** For Alibaba Cloud RDS for MySQL, binlog is enabled by default, and the account already has the necessary binlog dump permissions. In this case, you can skip these steps. -- Write the `ClientSample` class - Here is a basic example of a Java client connecting to a Canal server: - ``` - package com.alibaba.otter; - - import com.alibaba.otter.canal.client.CanalConnector; - import com.alibaba.otter.canal.client.CanalConnectors; - import com.alibaba.otter.canal.protocol.Message; - - import java.net.InetSocketAddress; - - public class ClientSample { - public static void main(String[] args) { - CanalConnector connector = CanalConnectors.newSingleConnector( - new InetSocketAddress("127.0.0.1", 11111), - "example", - "", - "" - ); - try { - connector.connect(); - connector.subscribe(".*\\..*"); - connector.rollback(); - while (true) { - Message message = connector.getWithoutAck(100); - long batchId = message.getId(); - int size = message.getEntries().size(); - if (batchId != -1 && size > 0) { - System.out.printf("Received batchId: %d, entry size: %d%n", batchId, size); - } - connector.ack(batchId); - } - } finally { - connector.disconnect(); - } - } - } + You also need to grant permissions to the Canal user, so it can connect as a MySQL slave. If the account already exists, you can directly use `GRANT`: + ``` + CREATE USER 'canal' IDENTIFIED BY 'canal'; + GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'canal'@'%'; + -- GRANT ALL PRIVILEGES ON *.* TO 'canal'@'%'; + FLUSH PRIVILEGES; ``` + +- Startup - **Notes:** - * Ensure the `Canal Server` is running and correctly configured (refer to the [QuickStart](https://github.com/alibaba/canal/wiki/QuickStart)). - * The above code connects to `127.0.0.1:11111`, which is the default TCP port for the Canal Server. - * `subscribe(".*\\..*")` means subscribe to all tables in all databases. + ``` + docker run -d --name my-canal -p 30306:3306 openeuler/canal:{Tag} + ``` + After the instance `my-canal` is started, you can view its running logs using `docker logs`. # Question and answering If you have any questions or want to use some special features, please submit an issue or a pull request on [openeuler-docker-images](https://gitee.com/openeuler/openeuler-docker-images). \ No newline at end of file diff --git a/Others/canal/doc/image-info.yml b/Others/canal/doc/image-info.yml index 6285680c..769b4b9f 100644 --- a/Others/canal/doc/image-info.yml +++ b/Others/canal/doc/image-info.yml @@ -9,9 +9,10 @@ environment: | tags: | canal镜像的Tag由其版本信息和基础镜像版本信息组成，详细内容如下 - | Tag | Currently | Architectures | - |----------|-------------|------------------| - | [1.1.8-oe2403sp1](https://gitee.com/openeuler/openeuler-docker-images/blob/master/Others/canal/1.1.8/24.03-lts-sp1/Dockerfile) | canal 1.1.8 on openEuler 24.03-LTS-SP1 | amd64, arm64 | + | Tags | Currently | Architectures | + |--------------------------------------------------------------------------------------------------------------------------------|----------------------------------------|---------------| + | [1.1.8-oe2403sp1](https://gitee.com/openeuler/openeuler-docker-images/blob/master/Others/canal/1.1.8/24.03-lts-sp1/Dockerfile) | canal 1.1.8 on openEuler 24.03-LTS-SP1 | amd64, arm64 | + | [1.1.8-oe2403sp2](https://gitee.com/openeuler/openeuler-docker-images/blob/master/Others/canal/1.1.8/24.03-lts-sp2/Dockerfile) | canal 1.1.8 on openEuler 24.03-LTS-SP2 | amd64, arm64 | 注意，以下`{Tag}`的值按照需求，替换为上述表格中的tag内容。 @@ -22,73 +23,34 @@ download: | ``` usage: | - - 添加 canal 客户端依赖 - 在你的 pom.xml 中添加以下依赖项，以引入 Canal Java 客户端： - ``` - - com.alibaba.otter - canal.client - 1.1.0 - - ``` - - - 创建标准 Maven 工程 - - 你可以使用 Maven Archetype 插件来创建一个标准的 Maven 工程。 - - **旧版本 Maven:** + + - 准备 + + 对于自建 MySQL , 需要先开启 Binlog 写入功能，配置 binlog-format 为 ROW 模式，my.cnf 中配置如下： ``` - mvn archetype:create -DgroupId=com.alibaba.otter -DartifactId=canal.sample + [mysqld] + log-bin=mysql-bin # 开启 binlog + binlog-format=ROW # 选择 ROW 模式 + server_id=1 # 配置 MySQL replaction 需要定义，不要和 canal 的 slaveId 重复 ``` - **Maven 3.0.5 及以上（推荐）:** + + **注意：** 针对阿里云 RDS for MySQL , 默认打开了 binlog , 并且账号默认具有 binlog dump 权限 , 不需要任何权限或者 binlog 设置,可以直接跳过这一步 + 授权 canal 链接 MySQL 账号具有作为 MySQL slave 的权限, 如果已有账户可直接 grant + ``` - mvn archetype:generate -DgroupId=com.alibaba.otter -DartifactId=canal.sample + CREATE USER canal IDENTIFIED BY 'canal'; + GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'canal'@'%'; + -- GRANT ALL PRIVILEGES ON *.* TO 'canal'@'%' ; + FLUSH PRIVILEGES; ``` - 这将生成一个基础的 Maven 项目结构。 - - 编写 ClientSample 类 - 以下是一个 Java 客户端连接 `Canal Server` 的基本示例： - ``` - package com.alibaba.otter; + - 启动 - import com.alibaba.otter.canal.client.CanalConnector; - import com.alibaba.otter.canal.client.CanalConnectors; - import com.alibaba.otter.canal.protocol.Message; - - import java.net.InetSocketAddress; - - public class ClientSample { - public static void main(String[] args) { - CanalConnector connector = CanalConnectors.newSingleConnector( - new InetSocketAddress("127.0.0.1", 11111), - "example", - "", - "" - ); - try { - connector.connect(); - connector.subscribe(".*\\..*"); - connector.rollback(); - while (true) { - Message message = connector.getWithoutAck(100); - long batchId = message.getId(); - int size = message.getEntries().size(); - if (batchId != -1 && size > 0) { - System.out.printf("Received batchId: %d, entry size: %d%n", batchId, size); - } - connector.ack(batchId); - } - } finally { - connector.disconnect(); - } - } - } ``` - - **注意事项：** - * 请确保 `Canal Server` 已启动并正确配置(参考 [QuickStart](https://github.com/alibaba/canal/wiki/QuickStart))。 - * 上述代码连接的是 `127.0.0.1:11111`，这是 Canal Server 的默认 TCP 端口。 - * `subscribe(".*\\..*")` 表示订阅所有数据库的所有表。 + docker run -d --name my-canal -p 30306:3306 openeuler/canal:{Tag} + ``` + 当 `my-canal` 实例启动后，你可以使用 `docker logs` 查看其运行日志。 + license: Apache-2.0 license similar_packages: diff --git a/Others/canal/meta.yml b/Others/canal/meta.yml index b67b7f34..b623fa47 100644 --- a/Others/canal/meta.yml +++ b/Others/canal/meta.yml @@ -1,2 +1,4 @@ 1.1.8-oe2403sp1: path: 1.1.8/24.03-lts-sp1/Dockerfile +1.1.8-oe2403sp2: + path: 1.1.8/24.03-lts-sp2/Dockerfile \ No newline at end of file -- Gitee