开发者社区> 问答> 正文

canal-adapter v1.1.7-alpha 同步数据到es7报es-mapping

错误信息为:Load canal adapter:es7 failed .Failed to bind properties under ‘es-mapping’ to com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig$ESMapping at canal.client.adapter.es7x.es7xAdapter.init(ES7xAdapter.java:54)

adapter\conf\es7下有test.yml文件,内容如下:
dataSourceKey: defaultDS
destination: example
groupId: g1
esMapping:
_index: customer
_id: id
sql: "select id, name from test"
commitBatch: 3000

application.yml配置如下:
server:
port: 8081
spring:
jackson:
date-format: yyyy-MM-dd HH:mm:ss
time-zone: GMT+8
default-property-inclusion: non_null

canal.conf:
mode: tcp #tcp kafka rocketMQ rabbitMQ
flatMessage: true
zookeeperHosts:
syncBatchSize: 1000
retries: -1
timeout:
accessKey:
secretKey:
consumerProperties:

# canal tcp consumer
canal.tcp.server.host: 127.0.0.1:11111
canal.tcp.zookeeper.hosts:
canal.tcp.batch.size: 500
canal.tcp.username:
canal.tcp.password:
# kafka consumer
kafka.bootstrap.servers: 127.0.0.1:9092
kafka.enable.auto.commit: false
kafka.auto.commit.interval.ms: 1000
kafka.auto.offset.reset: latest
kafka.request.timeout.ms: 40000
kafka.session.timeout.ms: 30000
kafka.isolation.level: read_committed
kafka.max.poll.records: 1000
# rocketMQ consumer
rocketmq.namespace:
rocketmq.namesrv.addr: 127.0.0.1:9876
rocketmq.batch.size: 1000
rocketmq.enable.message.trace: false
rocketmq.customized.trace.topic:
rocketmq.access.channel:
rocketmq.subscribe.filter:
# rabbitMQ consumer
rabbitmq.host:
rabbitmq.virtual.host:
rabbitmq.username:
rabbitmq.password:
rabbitmq.resource.ownerId:

srcDataSources:
defaultDS:
url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true
username: root
password: 121212
canalAdapters:

  • instance: example # canal instance Name or mq topic name
    groups:
    • groupId: g1
      outerAdapters:
      • name: logger

        - name: rdb

        key: mysql1

        properties:

        jdbc.driverClassName: com.mysql.jdbc.Driver

        jdbc.url: jdbc:mysql://127.0.0.1:3306/mytest2?useUnicode=true

        jdbc.username: root

        jdbc.password: 121212

        druid.stat.enable: false

        druid.stat.slowSqlMillis: 1000

        - name: rdb

        key: oracle1

        properties:

        jdbc.driverClassName: oracle.jdbc.OracleDriver

        jdbc.url: jdbc:oracle:thin:@localhost:49161:XE

        jdbc.username: mytest

        jdbc.password: m121212

        - name: rdb

        key: postgres1

        properties:

        jdbc.driverClassName: org.postgresql.Driver

        jdbc.url: jdbc:postgresql://localhost:5432/postgres

        jdbc.username: postgres

        jdbc.password: 121212

        threads: 1

        commitSize: 3000

        - name: hbase

        properties:

        hbase.zookeeper.quorum: 127.0.0.1

        hbase.zookeeper.property.clientPort: 2181

        zookeeper.znode.parent: /hbase

      • name: es
        hosts: 127.0.0.1:9200 # 127.0.0.1:9200 for rest mode
        properties:
        mode: rest# or rest

        # security.auth: test:123456 # only used for rest mode

        cluster.name: elasticsearch

        - name: kudu

        key: kudu

        properties:

        kudu.master.address: 127.0.0.1 # ',' split multi address

        - name: phoenix

        key: phoenix

        properties:

        jdbc.driverClassName: org.apache.phoenix.jdbc.PhoenixDriver

        jdbc.url: jdbc:phoenix:127.0.0.1:2181:/hbase/db

        jdbc.username:

        jdbc.password:

deployer\conf\example\instance.properties配置如下:

#

mysql serverId , v1.0.26+ will autoGen

canal.instance.mysql.slaveId=0

enable gtid use true/false

canal.instance.gtidon=false

position info

canal.instance.master.address=127.0.0.1:3306
canal.instance.master.journal.name=
canal.instance.master.position=
canal.instance.master.timestamp=
canal.instance.master.gtid=

rds oss binlog

canal.instance.rds.accesskey=
canal.instance.rds.secretkey=
canal.instance.rds.instanceId=

table meta tsdb info

canal.instance.tsdb.enable=true

canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb

canal.instance.tsdb.dbUsername=canal

canal.instance.tsdb.dbPassword=canal

canal.instance.standby.address =

canal.instance.standby.journal.name =

canal.instance.standby.position =

canal.instance.standby.timestamp =

canal.instance.standby.gtid=

username/password

canal.instance.dbUsername=canal
canal.instance.dbPassword=canal
canal.instance.connectionCharset = UTF-8

enable druid Decrypt database password

canal.instance.enableDruid=false

canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==

table regex

canal.instance.filter.regex=mytest.test

table black regex

canal.instance.filter.black.regex=mysql\.slave_.*

table field filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)

canal.instance.filter.field=test1.t_product:id/subject/keywords,test2.t_company:id/name/contact/ch

table field black filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)

canal.instance.filter.black.field=test1.t_product:subject/product_image,test2.t_company:id/name/contact/ch

mq config

canal.mq.topic=example

dynamic topic route by schema or table regex

canal.mq.dynamicTopic=mytest1.user,topic2:mytest2\..,.\..*

canal.mq.partition=0

hash partition config

canal.mq.enableDynamicQueuePartition=false

canal.mq.partitionsNum=3

canal.mq.dynamicTopicPartitionNum=test.*:4,mycanal:6

canal.mq.partitionHash=test.table:id^name,.\..

#

multi stream for polardbx

canal.instance.multi.stream.on=false

#

展开
收起
游客krxdtkrsh6zak 2023-09-13 17:57:49 376 0
3 条回答
写回答
取消 提交回答
  • 根据您提供的错误信息,看起来是在加载 canal-adapter 的时候出现了问题。具体错误信息为:Load canal adapter:es7 failed .Failed to bind properties under ‘es-mapping’ to com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig$ESMapping。

    这个错误通常是由于配置文件中的 es-mapping 参数设置有误导致的。请检查您的 test.yml 配置文件中关于 es-mapping 的部分,确保参数设置正确。下面是一个示例的正确配置:

    yaml
    esMapping:
    _index: customer
    _id: id
    sql: "select id, name from test"
    commitBatch: 3000
    另外,请确保您的 application.yml 文件中没有对 es-mapping 参数进行配置,因为 es-mapping 是在 test.yml 中进行配置的。

    2023-09-15 11:40:05
    赞同 展开评论 打赏
  • 看起来你在使用canal-adapter v1.1.7-alpha版本同步数据到Elasticsearch 7.x时遇到了问题具体地,问题似乎与Elasticsearch的映射(mapping)有关

    首先,请确保你已经正确配置了canal-adapter和Elasticsearch确保你的配置包括正确的Elasticsearch集群地址、索引名称和其他相关设置

    然后,你应该检查Elasticsearch的映射这个错误可能是由于以下原因之一造成的:

    1. 重复的索引名:如果你试图创建具有重复名称的索引,Elasticsearch会拒绝请求并返回错误你需要检查canal-adapter的配置以确保索引名称在Elasticsearch中是唯一的
    2. 字段的映射冲突:如果你在同步数据时尝试将某些字段映射到已经存在的字段上,可能会出现冲突确保你的数据和索引映射匹配,或者在创建索引时使用dynamic: true来自动映射未预先定义的字段
    3. 不支持的映射类型:从Elasticsearch 7.0开始,将不再支持具有dynamic_templates的映射类型如果你的映射使用了这种类型,可能会导致问题你可以尝试更新映射以使用新的类型,或者在Elasticsearch 6.x版本上运行canal-adapter

    如果以上建议无法解决你的问题,你可以尝试提供更多具体的错误信息或日志,以便我能更精确地帮助你

    2023-09-13 18:11:54
    赞同 展开评论 打赏
  • 补充一下内容:
    在es创建了索引如下:
    {"mappings":{"properties":{"id":{"type":"text"},"name":{"type":"text"}}}}

    mysql数据库中test表有id和name两个字段,都是varchar类型

    2023-09-13 18:03:11
    赞同 展开评论 打赏
问答排行榜
最热
最新

相关电子书

更多
低代码开发师(初级)实战教程 立即下载
冬季实战营第三期:MySQL数据库进阶实战 立即下载
阿里巴巴DevOps 最佳实践手册 立即下载