org.apache.paimon.flink.action.cdc

Class SyncTableActionBase

java.lang.Object

org.apache.paimon.flink.action.ActionBase

org.apache.paimon.flink.action.cdc.SynchronizationActionBase

org.apache.paimon.flink.action.cdc.SyncTableActionBase

All Implemented Interfaces:

Action

Direct Known Subclasses:

MessageQueueSyncTableActionBase, MongoDBSyncTableAction, MySqlSyncTableAction, PostgresSyncTableAction

public abstract class SyncTableActionBase
extends SynchronizationActionBase

Base Action for synchronizing into one Paimon table.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	SyncTableActionBase.SchemaRetrievalException Custom exception to indicate issues with schema retrieval.

Field Summary

Fields

Modifier and Type	Field and Description
protected List<String>	computedColumnArgs
protected List<ComputedColumn>	computedColumns
protected FileStoreTable	fileStoreTable
protected List<String>	partitionKeys
protected List<String>	primaryKeys
protected String	table

Fields inherited from class org.apache.paimon.flink.action.cdc.SynchronizationActionBase

allowUpperCase, cdcSourceConfig, database, metadataConverters, syncJobHandler, tableConfig, typeMapping

Fields inherited from class org.apache.paimon.flink.action.ActionBase

batchTEnv, catalog, catalogName, catalogOptions, env, flinkCatalog

Constructor Summary

Constructors

Constructor and Description
SyncTableActionBase(String warehouse, String database, String table, Map<String,String> catalogConfig, Map<String,String> cdcSourceConfig, SyncJobHandler.SourceType sourceType)

Method Summary

Modifier and Type	Method and Description
protected void	beforeBuildingSourceSink()
protected EventParser.Factory<RichCdcMultiplexRecord>	buildEventParserFactory()
protected Schema	buildPaimonSchema(Schema retrievedSchema)
protected void	buildSink(org.apache.flink.streaming.api.datastream.DataStream<RichCdcMultiplexRecord> input, EventParser.Factory<RichCdcMultiplexRecord> parserFactory)
FileStoreTable	fileStoreTable()
protected org.apache.flink.api.common.functions.FlatMapFunction<CdcSourceRecord,RichCdcMultiplexRecord>	recordParse()
protected abstract Schema	retrieveSchema()
protected void	validateCaseSensitivity()
SyncTableActionBase	withComputedColumnArgs(List<String> computedColumnArgs)
SyncTableActionBase	withPartitionKeys(List<String> partitionKeys)
SyncTableActionBase	withPartitionKeys(String... partitionKeys)
SyncTableActionBase	withPrimaryKeys(List<String> primaryKeys)
SyncTableActionBase	withPrimaryKeys(String... primaryKeys)

Methods inherited from class org.apache.paimon.flink.action.cdc.SynchronizationActionBase

alterTableOptions, build, buildSource, createCdcTimestampExtractor, run, tableConfig, withMetadataColumns, withTableConfig, withTypeMapping

Methods inherited from class org.apache.paimon.flink.action.ActionBase

catalogConfig, catalogLoader, compatibleCheck, execute, initFlinkCatalog, initFlinkEnv, initPaimonCatalog, toPaimonTypes, withStreamExecutionEnvironment

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

table

protected final String table

fileStoreTable

protected FileStoreTable fileStoreTable

partitionKeys

protected List<String> partitionKeys

primaryKeys

protected List<String> primaryKeys

computedColumnArgs

protected List<String> computedColumnArgs

computedColumns

protected List<ComputedColumn> computedColumns

Constructor Detail

SyncTableActionBase

public SyncTableActionBase(String warehouse,
                           String database,
                           String table,
                           Map<String,String> catalogConfig,
                           Map<String,String> cdcSourceConfig,
                           SyncJobHandler.SourceType sourceType)

Method Detail

withPartitionKeys

public SyncTableActionBase withPartitionKeys(String... partitionKeys)

withPartitionKeys

public SyncTableActionBase withPartitionKeys(List<String> partitionKeys)

withPrimaryKeys

public SyncTableActionBase withPrimaryKeys(String... primaryKeys)

withPrimaryKeys

public SyncTableActionBase withPrimaryKeys(List<String> primaryKeys)

withComputedColumnArgs

public SyncTableActionBase withComputedColumnArgs(List<String> computedColumnArgs)

retrieveSchema

protected abstract Schema retrieveSchema()
                                  throws Exception

Throws:

Exception

buildPaimonSchema

protected Schema buildPaimonSchema(Schema retrievedSchema)

validateCaseSensitivity

protected void validateCaseSensitivity()

Specified by:

validateCaseSensitivity in class SynchronizationActionBase

beforeBuildingSourceSink

protected void beforeBuildingSourceSink()
                                 throws Exception

Overrides:

beforeBuildingSourceSink in class SynchronizationActionBase

Throws:

Exception

recordParse

protected org.apache.flink.api.common.functions.FlatMapFunction<CdcSourceRecord,RichCdcMultiplexRecord> recordParse()

Specified by:

recordParse in class SynchronizationActionBase

buildEventParserFactory

protected EventParser.Factory<RichCdcMultiplexRecord> buildEventParserFactory()

Specified by:

buildEventParserFactory in class SynchronizationActionBase

buildSink

protected void buildSink(org.apache.flink.streaming.api.datastream.DataStream<RichCdcMultiplexRecord> input,
                         EventParser.Factory<RichCdcMultiplexRecord> parserFactory)

Specified by:

buildSink in class SynchronizationActionBase

fileStoreTable

@VisibleForTesting
public FileStoreTable fileStoreTable()