org.apache.paimon.flink.action.cdc

Class SyncDatabaseActionBase

java.lang.Object

org.apache.paimon.flink.action.ActionBase

org.apache.paimon.flink.action.cdc.SynchronizationActionBase

org.apache.paimon.flink.action.cdc.SyncDatabaseActionBase

All Implemented Interfaces:

Action

Direct Known Subclasses:

KafkaSyncDatabaseAction, MongoDBSyncDatabaseAction, MySqlSyncDatabaseAction, PulsarSyncDatabaseAction

public abstract class SyncDatabaseActionBase
extends SynchronizationActionBase

Base Action for synchronizing into one Paimon database.

Field Summary

Fields

Modifier and Type	Field and Description
protected Map<String,String>	dbPrefix
protected Map<String,String>	dbSuffix
protected String	excludingDbs
protected String	excludingTables
protected String	includingDbs
protected String	includingTables
protected boolean	mergeShards
protected MultiTablesSinkMode	mode
protected Map<String,List<String>>	partitionKeyMultiple
protected List<String>	partitionKeys
protected List<String>	primaryKeys
protected Map<String,String>	tableMapping
protected String	tablePrefix
protected List<FileStoreTable>	tables
protected String	tableSuffix

Fields inherited from class org.apache.paimon.flink.action.cdc.SynchronizationActionBase

caseSensitive, cdcSourceConfig, database, metadataConverters, syncJobHandler, tableConfig, typeMapping

Fields inherited from class org.apache.paimon.flink.action.ActionBase

batchTEnv, catalog, catalogName, catalogOptions, env, flinkCatalog

Constructor Summary

Constructors

Constructor and Description
SyncDatabaseActionBase(String database, Map<String,String> catalogConfig, Map<String,String> cdcSourceConfig, SyncJobHandler.SourceType sourceType)

Method Summary

Modifier and Type	Method and Description
protected EventParser.Factory<RichCdcMultiplexRecord>	buildEventParserFactory()
protected void	buildSink(org.apache.flink.streaming.api.datastream.DataStream<RichCdcMultiplexRecord> input, EventParser.Factory<RichCdcMultiplexRecord> parserFactory)
SyncDatabaseActionBase	excludingDbs(String excludingDbs)
SyncDatabaseActionBase	excludingTables(String excludingTables)
SyncDatabaseActionBase	includingDbs(String includingDbs)
SyncDatabaseActionBase	includingTables(String includingTables)
SyncDatabaseActionBase	mergeShards(boolean mergeShards)
protected org.apache.flink.api.common.functions.FlatMapFunction<CdcSourceRecord,RichCdcMultiplexRecord>	recordParse()
protected abstract boolean	requirePrimaryKeys()
SyncDatabaseActionBase	withDbPrefix(Map<String,String> dbPrefix)
SyncDatabaseActionBase	withDbSuffix(Map<String,String> dbSuffix)
SyncDatabaseActionBase	withMode(MultiTablesSinkMode mode)
SyncDatabaseActionBase	withPartitionKeyMultiple(Map<String,List<String>> partitionKeyMultiple)
SyncDatabaseActionBase	withPartitionKeys(String... partitionKeys)
SyncDatabaseActionBase	withPrimaryKeys(String... primaryKeys)
SyncDatabaseActionBase	withTableMapping(Map<String,String> tableMapping)
SyncDatabaseActionBase	withTablePrefix(String tablePrefix)
SyncDatabaseActionBase	withTableSuffix(String tableSuffix)

Methods inherited from class org.apache.paimon.flink.action.cdc.SynchronizationActionBase

alterTableOptions, beforeBuildingSourceSink, build, buildSource, createCdcTimestampExtractor, run, tableConfig, validateRuntimeExecutionMode, withMetadataColumns, withTableConfig, withTypeMapping

Methods inherited from class org.apache.paimon.flink.action.ActionBase

catalogConfig, catalogLoader, compatibleCheck, execute, initFlinkCatalog, initFlinkEnv, initPaimonCatalog, toPaimonTypes, withStreamExecutionEnvironment

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

mergeShards

protected boolean mergeShards

mode

protected MultiTablesSinkMode mode

tablePrefix

protected String tablePrefix

tableSuffix

protected String tableSuffix

tableMapping

protected Map<String,String> tableMapping

dbPrefix

protected Map<String,String> dbPrefix

dbSuffix

protected Map<String,String> dbSuffix

includingTables

protected String includingTables

partitionKeys

protected List<String> partitionKeys

primaryKeys

protected List<String> primaryKeys

excludingTables

@Nullable
protected String excludingTables

includingDbs

protected String includingDbs

excludingDbs

@Nullable
protected String excludingDbs

tables

protected List<FileStoreTable> tables

partitionKeyMultiple

protected Map<String,List<String>> partitionKeyMultiple

Constructor Detail

SyncDatabaseActionBase

public SyncDatabaseActionBase(String database,
                              Map<String,String> catalogConfig,
                              Map<String,String> cdcSourceConfig,
                              SyncJobHandler.SourceType sourceType)

Method Detail

mergeShards

public SyncDatabaseActionBase mergeShards(boolean mergeShards)

withMode

public SyncDatabaseActionBase withMode(MultiTablesSinkMode mode)

withTablePrefix

public SyncDatabaseActionBase withTablePrefix(@Nullable
                                              String tablePrefix)

withTableSuffix

public SyncDatabaseActionBase withTableSuffix(@Nullable
                                              String tableSuffix)

withDbPrefix

public SyncDatabaseActionBase withDbPrefix(Map<String,String> dbPrefix)

withDbSuffix

public SyncDatabaseActionBase withDbSuffix(Map<String,String> dbSuffix)

withTableMapping

public SyncDatabaseActionBase withTableMapping(Map<String,String> tableMapping)

includingTables

public SyncDatabaseActionBase includingTables(@Nullable
                                              String includingTables)

excludingTables

public SyncDatabaseActionBase excludingTables(@Nullable
                                              String excludingTables)

includingDbs

public SyncDatabaseActionBase includingDbs(@Nullable
                                           String includingDbs)

excludingDbs

public SyncDatabaseActionBase excludingDbs(@Nullable
                                           String excludingDbs)

withPartitionKeys

public SyncDatabaseActionBase withPartitionKeys(String... partitionKeys)

withPrimaryKeys

public SyncDatabaseActionBase withPrimaryKeys(String... primaryKeys)

recordParse

protected org.apache.flink.api.common.functions.FlatMapFunction<CdcSourceRecord,RichCdcMultiplexRecord> recordParse()

Specified by:

recordParse in class SynchronizationActionBase

withPartitionKeyMultiple

public SyncDatabaseActionBase withPartitionKeyMultiple(Map<String,List<String>> partitionKeyMultiple)

buildEventParserFactory

protected EventParser.Factory<RichCdcMultiplexRecord> buildEventParserFactory()

Specified by:

buildEventParserFactory in class SynchronizationActionBase

requirePrimaryKeys

protected abstract boolean requirePrimaryKeys()

buildSink

protected void buildSink(org.apache.flink.streaming.api.datastream.DataStream<RichCdcMultiplexRecord> input,
                         EventParser.Factory<RichCdcMultiplexRecord> parserFactory)

Specified by:

buildSink in class SynchronizationActionBase