org.apache.paimon.flink.action.cdc

Class SyncDatabaseActionBase

java.lang.Object

org.apache.paimon.flink.action.ActionBase

org.apache.paimon.flink.action.cdc.SynchronizationActionBase

org.apache.paimon.flink.action.cdc.SyncDatabaseActionBase

All Implemented Interfaces:

Action

Direct Known Subclasses:

KafkaSyncDatabaseAction, MongoDBSyncDatabaseAction, MySqlSyncDatabaseAction, PulsarSyncDatabaseAction

public abstract class SyncDatabaseActionBase
extends SynchronizationActionBase

Base Action for synchronizing into one Paimon database.

Field Summary

Fields

Modifier and Type	Field and Description
protected String	excludingTables
protected String	includingTables
protected boolean	mergeShards
protected MultiTablesSinkMode	mode
protected Map<String,List<String>>	partitionKeyMultiple
protected List<String>	partitionKeys
protected List<String>	primaryKeys
protected String	tablePrefix
protected List<FileStoreTable>	tables
protected String	tableSuffix

Fields inherited from class org.apache.paimon.flink.action.cdc.SynchronizationActionBase

allowUpperCase, cdcSourceConfig, database, metadataConverters, syncJobHandler, tableConfig, typeMapping

Fields inherited from class org.apache.paimon.flink.action.ActionBase

batchTEnv, catalog, catalogName, catalogOptions, env, flinkCatalog

Constructor Summary

Constructors

Constructor and Description
SyncDatabaseActionBase(String warehouse, String database, Map<String,String> catalogConfig, Map<String,String> cdcSourceConfig, SyncJobHandler.SourceType sourceType)

Method Summary

Modifier and Type	Method and Description
protected EventParser.Factory<RichCdcMultiplexRecord>	buildEventParserFactory()
protected void	buildSink(org.apache.flink.streaming.api.datastream.DataStream<RichCdcMultiplexRecord> input, EventParser.Factory<RichCdcMultiplexRecord> parserFactory)
SyncDatabaseActionBase	excludingTables(String excludingTables)
SyncDatabaseActionBase	includingTables(String includingTables)
SyncDatabaseActionBase	mergeShards(boolean mergeShards)
protected org.apache.flink.api.common.functions.FlatMapFunction<CdcSourceRecord,RichCdcMultiplexRecord>	recordParse()
protected abstract boolean	requirePrimaryKeys()
protected void	validateCaseSensitivity()
SyncDatabaseActionBase	withMode(MultiTablesSinkMode mode)
SyncDatabaseActionBase	withPartitionKeyMultiple(Map<String,List<String>> partitionKeyMultiple)
SyncDatabaseActionBase	withPartitionKeys(String... partitionKeys)
SyncDatabaseActionBase	withPrimaryKeys(String... primaryKeys)
SyncDatabaseActionBase	withTablePrefix(String tablePrefix)
SyncDatabaseActionBase	withTableSuffix(String tableSuffix)

Methods inherited from class org.apache.paimon.flink.action.cdc.SynchronizationActionBase

alterTableOptions, beforeBuildingSourceSink, build, buildSource, createCdcTimestampExtractor, run, tableConfig, validateRuntimeExecutionMode, withMetadataColumns, withTableConfig, withTypeMapping

Methods inherited from class org.apache.paimon.flink.action.ActionBase

catalogConfig, catalogLoader, compatibleCheck, execute, initFlinkCatalog, initFlinkEnv, initPaimonCatalog, toPaimonTypes, withStreamExecutionEnvironment

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

mergeShards

protected boolean mergeShards

mode

protected MultiTablesSinkMode mode

tablePrefix

protected String tablePrefix

tableSuffix

protected String tableSuffix

includingTables

protected String includingTables

partitionKeys

protected List<String> partitionKeys

primaryKeys

protected List<String> primaryKeys

excludingTables

@Nullable
protected String excludingTables

tables

protected List<FileStoreTable> tables

partitionKeyMultiple

protected Map<String,List<String>> partitionKeyMultiple

Constructor Detail

SyncDatabaseActionBase

public SyncDatabaseActionBase(String warehouse,
                              String database,
                              Map<String,String> catalogConfig,
                              Map<String,String> cdcSourceConfig,
                              SyncJobHandler.SourceType sourceType)

Method Detail

mergeShards

public SyncDatabaseActionBase mergeShards(boolean mergeShards)

withMode

public SyncDatabaseActionBase withMode(MultiTablesSinkMode mode)

withTablePrefix

public SyncDatabaseActionBase withTablePrefix(@Nullable
                                              String tablePrefix)

withTableSuffix

public SyncDatabaseActionBase withTableSuffix(@Nullable
                                              String tableSuffix)

includingTables

public SyncDatabaseActionBase includingTables(@Nullable
                                              String includingTables)

excludingTables

public SyncDatabaseActionBase excludingTables(@Nullable
                                              String excludingTables)

withPartitionKeys

public SyncDatabaseActionBase withPartitionKeys(String... partitionKeys)

withPrimaryKeys

public SyncDatabaseActionBase withPrimaryKeys(String... primaryKeys)

validateCaseSensitivity

protected void validateCaseSensitivity()

Specified by:

validateCaseSensitivity in class SynchronizationActionBase

recordParse

protected org.apache.flink.api.common.functions.FlatMapFunction<CdcSourceRecord,RichCdcMultiplexRecord> recordParse()

Specified by:

recordParse in class SynchronizationActionBase

withPartitionKeyMultiple

public SyncDatabaseActionBase withPartitionKeyMultiple(Map<String,List<String>> partitionKeyMultiple)

buildEventParserFactory

protected EventParser.Factory<RichCdcMultiplexRecord> buildEventParserFactory()

Specified by:

buildEventParserFactory in class SynchronizationActionBase

requirePrimaryKeys

protected abstract boolean requirePrimaryKeys()

buildSink

protected void buildSink(org.apache.flink.streaming.api.datastream.DataStream<RichCdcMultiplexRecord> input,
                         EventParser.Factory<RichCdcMultiplexRecord> parserFactory)

Specified by:

buildSink in class SynchronizationActionBase