org.apache.paimon.flink.sink

Class FlinkSink<T>

java.lang.Object

org.apache.paimon.flink.sink.FlinkSink<T>

All Implemented Interfaces:

Serializable

Direct Known Subclasses:

CompactorSink, FlinkWriteSink, UnawareBucketCompactionSink

public abstract class FlinkSink<T>
extends Object
implements Serializable

Abstract sink of paimon.

See Also:

Serialized Form

Field Summary

Fields

Modifier and Type	Field and Description
protected FileStoreTable	table

Constructor Summary

Constructors

Constructor and Description
FlinkSink(FileStoreTable table, boolean ignorePreviousFiles)

Method Summary

Modifier and Type	Method and Description
static void	assertBatchAdaptiveParallelism(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment env, int sinkParallelism)
static void	assertBatchAdaptiveParallelism(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment env, int sinkParallelism, boolean writeMCacheEnabled, boolean hashDynamicMode)
static void	assertBatchAdaptiveParallelism(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment env, int sinkParallelism, String exceptionMsg)
static void	assertStreamingConfiguration(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment env)
static void	configureGlobalCommitter(org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator<?> committed, double cpuCores, MemorySize heapMemory)
protected abstract CommittableStateManager<ManifestCommittable>	createCommittableStateManager()
protected abstract Committer.Factory<Committable,ManifestCommittable>	createCommitterFactory()
protected abstract org.apache.flink.streaming.api.operators.OneInputStreamOperator<T,Committable>	createWriteOperator(StoreSinkWrite.Provider writeProvider, String commitUser)
protected org.apache.flink.streaming.api.datastream.DataStreamSink<?>	doCommit(org.apache.flink.streaming.api.datastream.DataStream<Committable> written, String commitUser)
org.apache.flink.streaming.api.datastream.DataStream<Committable>	doWrite(org.apache.flink.streaming.api.datastream.DataStream<T> input, String commitUser, Integer parallelism)
static boolean	isStreaming(org.apache.flink.streaming.api.datastream.DataStream<?> input)
static boolean	isStreaming(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment env)
org.apache.flink.streaming.api.datastream.DataStreamSink<?>	sinkFrom(org.apache.flink.streaming.api.datastream.DataStream<T> input)
org.apache.flink.streaming.api.datastream.DataStreamSink<?>	sinkFrom(org.apache.flink.streaming.api.datastream.DataStream<T> input, String initialCommitUser)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

table

protected final FileStoreTable table

Constructor Detail

FlinkSink

public FlinkSink(FileStoreTable table,
                 boolean ignorePreviousFiles)

Method Detail

sinkFrom

public org.apache.flink.streaming.api.datastream.DataStreamSink<?> sinkFrom(org.apache.flink.streaming.api.datastream.DataStream<T> input)

sinkFrom

public org.apache.flink.streaming.api.datastream.DataStreamSink<?> sinkFrom(org.apache.flink.streaming.api.datastream.DataStream<T> input,
                                                                            String initialCommitUser)

doWrite

public org.apache.flink.streaming.api.datastream.DataStream<Committable> doWrite(org.apache.flink.streaming.api.datastream.DataStream<T> input,
                                                                                 String commitUser,
                                                                                 @Nullable
                                                                                 Integer parallelism)

doCommit

protected org.apache.flink.streaming.api.datastream.DataStreamSink<?> doCommit(org.apache.flink.streaming.api.datastream.DataStream<Committable> written,
                                                                               String commitUser)

configureGlobalCommitter

public static void configureGlobalCommitter(org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator<?> committed,
                                            double cpuCores,
                                            @Nullable
                                            MemorySize heapMemory)

assertStreamingConfiguration

public static void assertStreamingConfiguration(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment env)

assertBatchAdaptiveParallelism

public static void assertBatchAdaptiveParallelism(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment env,
                                                  int sinkParallelism)

assertBatchAdaptiveParallelism

public static void assertBatchAdaptiveParallelism(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment env,
                                                  int sinkParallelism,
                                                  boolean writeMCacheEnabled,
                                                  boolean hashDynamicMode)

assertBatchAdaptiveParallelism

public static void assertBatchAdaptiveParallelism(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment env,
                                                  int sinkParallelism,
                                                  String exceptionMsg)

createWriteOperator

protected abstract org.apache.flink.streaming.api.operators.OneInputStreamOperator<T,Committable> createWriteOperator(StoreSinkWrite.Provider writeProvider,
                                                                                                                      String commitUser)

createCommitterFactory

protected abstract Committer.Factory<Committable,ManifestCommittable> createCommitterFactory()

createCommittableStateManager

protected abstract CommittableStateManager<ManifestCommittable> createCommittableStateManager()

isStreaming

public static boolean isStreaming(org.apache.flink.streaming.api.datastream.DataStream<?> input)

isStreaming

public static boolean isStreaming(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment env)