T
- type of record to write.public abstract class AbstractFileStoreWrite<T> extends Object implements FileStoreWrite<T>
FileStoreWrite
implementation.Modifier and Type | Class and Description |
---|---|
static class |
AbstractFileStoreWrite.WriterContainer<T>
RecordWriter with the snapshot id it is created upon and the identifier of its last
modified commit. |
FileStoreWrite.State<T>
Modifier and Type | Field and Description |
---|---|
protected CompactionMetrics |
compactionMetrics |
protected IOManager |
ioManager |
protected boolean |
isStreamingMode |
protected SnapshotManager |
snapshotManager |
protected String |
tableName |
protected Map<BinaryRow,Map<Integer,AbstractFileStoreWrite.WriterContainer<T>>> |
writers |
Modifier | Constructor and Description |
---|---|
protected |
AbstractFileStoreWrite(SnapshotManager snapshotManager,
FileStoreScan scan,
IndexMaintainer.Factory<T> indexFactory,
DeletionVectorsMaintainer.Factory dvMaintainerFactory,
String tableName,
int totalBuckets,
RowType partitionType,
int writerNumberMax,
boolean legacyPartitionName) |
Modifier and Type | Method and Description |
---|---|
List<FileStoreWrite.State<T>> |
checkpoint()
Extract state of the current operation instance.
|
void |
close()
Close the writer.
|
void |
compact(BinaryRow partition,
int bucket,
boolean fullCompaction)
Compact data stored in given partition and bucket.
|
protected static <T> java.util.function.Function<AbstractFileStoreWrite.WriterContainer<T>,Boolean> |
createConflictAwareWriterCleanChecker(String commitUser,
SnapshotManager snapshotManager) |
protected static <T> java.util.function.Function<AbstractFileStoreWrite.WriterContainer<T>,Boolean> |
createNoConflictAwareWriterCleanChecker() |
protected abstract RecordWriter<T> |
createWriter(Long snapshotId,
BinaryRow partition,
int bucket,
List<DataFileMeta> restoreFiles,
long restoredMaxSeqNumber,
CommitIncrement restoreIncrement,
ExecutorService compactExecutor,
DeletionVectorsMaintainer deletionVectorsMaintainer) |
protected abstract java.util.function.Function<AbstractFileStoreWrite.WriterContainer<T>,Boolean> |
createWriterCleanChecker() |
AbstractFileStoreWrite.WriterContainer<T> |
createWriterContainer(BinaryRow partition,
int bucket,
boolean ignorePreviousFiles) |
protected void |
forceBufferSpill() |
Map<BinaryRow,List<Integer>> |
getActiveBuckets() |
ExecutorService |
getCompactExecutor() |
protected AbstractFileStoreWrite.WriterContainer<T> |
getWriterWrapper(BinaryRow partition,
int bucket) |
void |
notifyNewFiles(long snapshotId,
BinaryRow partition,
int bucket,
List<DataFileMeta> files)
Notify that some new files are created at given snapshot in given bucket.
|
protected void |
notifyNewWriter(RecordWriter<T> writer) |
List<CommitMessage> |
prepareCommit(boolean waitCompaction,
long commitIdentifier)
Prepare commit in the write.
|
void |
restore(List<FileStoreWrite.State<T>> states)
Restore state of a previous operation instance into the current operation instance.
|
void |
withCompactExecutor(ExecutorService compactExecutor) |
void |
withExecutionMode(boolean isStreamingMode)
We detect whether it is in batch mode, if so, we do some optimization.
|
void |
withIgnorePreviousFiles(boolean ignorePreviousFiles)
Set whether the write operation should ignore previously stored files.
|
void |
withInsertOnly(boolean insertOnly)
This method is called when the insert only status of the records changes.
|
FileStoreWrite<T> |
withIOManager(IOManager ioManager) |
FileStoreWrite<T> |
withMemoryPoolFactory(MemoryPoolFactory memoryPoolFactory)
With memory pool factory for the current file store write.
|
FileStoreWrite<T> |
withMetricRegistry(MetricRegistry metricRegistry)
With metrics to measure compaction.
|
void |
write(BinaryRow partition,
int bucket,
T data)
Write the data to the store according to the partition and bucket.
|
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
withMemoryPool
protected final SnapshotManager snapshotManager
protected final Map<BinaryRow,Map<Integer,AbstractFileStoreWrite.WriterContainer<T>>> writers
protected boolean isStreamingMode
protected CompactionMetrics compactionMetrics
protected final String tableName
protected AbstractFileStoreWrite(SnapshotManager snapshotManager, FileStoreScan scan, @Nullable IndexMaintainer.Factory<T> indexFactory, @Nullable DeletionVectorsMaintainer.Factory dvMaintainerFactory, String tableName, int totalBuckets, RowType partitionType, int writerNumberMax, boolean legacyPartitionName)
public FileStoreWrite<T> withIOManager(IOManager ioManager)
withIOManager
in interface FileStoreWrite<T>
public FileStoreWrite<T> withMemoryPoolFactory(MemoryPoolFactory memoryPoolFactory)
FileStoreWrite
withMemoryPoolFactory
in interface FileStoreWrite<T>
memoryPoolFactory
- the given memory pool factory.public void withIgnorePreviousFiles(boolean ignorePreviousFiles)
FileStoreWrite
withIgnorePreviousFiles
in interface FileStoreWrite<T>
ignorePreviousFiles
- whether the write operation should ignore previously stored files.public void withCompactExecutor(ExecutorService compactExecutor)
withCompactExecutor
in interface FileStoreWrite<T>
public void withInsertOnly(boolean insertOnly)
FileStoreWrite
withInsertOnly
in interface FileStoreWrite<T>
insertOnly
- If true, all the following records would be of RowKind.INSERT
, and no two records would have the same primary
key.public void write(BinaryRow partition, int bucket, T data) throws Exception
FileStoreWrite
write
in interface FileStoreWrite<T>
partition
- the partition of the databucket
- the bucket id of the datadata
- the given dataException
- the thrown exception when writing the recordpublic void compact(BinaryRow partition, int bucket, boolean fullCompaction) throws Exception
FileStoreWrite
compact
in interface FileStoreWrite<T>
partition
- the partition to compactbucket
- the bucket to compactfullCompaction
- whether to trigger full compaction or just normal compactionException
- the thrown exception when compacting the recordspublic void notifyNewFiles(long snapshotId, BinaryRow partition, int bucket, List<DataFileMeta> files)
FileStoreWrite
Most probably, these files are created by another job. Currently this method is only used by the dedicated compact job to see files created by writer jobs.
notifyNewFiles
in interface FileStoreWrite<T>
snapshotId
- the snapshot id where new files are createdpartition
- the partition where new files are createdbucket
- the bucket where new files are createdfiles
- the new files themselvespublic List<CommitMessage> prepareCommit(boolean waitCompaction, long commitIdentifier) throws Exception
FileStoreWrite
prepareCommit
in interface FileStoreWrite<T>
waitCompaction
- if this method need to wait for current compaction to completecommitIdentifier
- identifier of the commit being preparedException
- the thrown exceptionprotected abstract java.util.function.Function<AbstractFileStoreWrite.WriterContainer<T>,Boolean> createWriterCleanChecker()
protected static <T> java.util.function.Function<AbstractFileStoreWrite.WriterContainer<T>,Boolean> createConflictAwareWriterCleanChecker(String commitUser, SnapshotManager snapshotManager)
protected static <T> java.util.function.Function<AbstractFileStoreWrite.WriterContainer<T>,Boolean> createNoConflictAwareWriterCleanChecker()
public void close() throws Exception
FileStoreWrite
close
in interface FileStoreWrite<T>
Exception
- the thrown exceptionpublic List<FileStoreWrite.State<T>> checkpoint()
Restorable
checkpoint
in interface Restorable<List<FileStoreWrite.State<T>>>
public void restore(List<FileStoreWrite.State<T>> states)
Restorable
restore
in interface Restorable<List<FileStoreWrite.State<T>>>
protected AbstractFileStoreWrite.WriterContainer<T> getWriterWrapper(BinaryRow partition, int bucket)
@VisibleForTesting public AbstractFileStoreWrite.WriterContainer<T> createWriterContainer(BinaryRow partition, int bucket, boolean ignorePreviousFiles)
public void withExecutionMode(boolean isStreamingMode)
FileStoreWrite
withExecutionMode
in interface FileStoreWrite<T>
isStreamingMode
- whether in streaming modepublic FileStoreWrite<T> withMetricRegistry(MetricRegistry metricRegistry)
FileStoreWrite
withMetricRegistry
in interface FileStoreWrite<T>
@VisibleForTesting public ExecutorService getCompactExecutor()
protected void notifyNewWriter(RecordWriter<T> writer)
protected abstract RecordWriter<T> createWriter(@Nullable Long snapshotId, BinaryRow partition, int bucket, List<DataFileMeta> restoreFiles, long restoredMaxSeqNumber, @Nullable CommitIncrement restoreIncrement, ExecutorService compactExecutor, @Nullable DeletionVectorsMaintainer deletionVectorsMaintainer)
Copyright © 2023–2024 The Apache Software Foundation. All rights reserved.