public abstract class FileDeletionBase<T extends Snapshot> extends Object
Modifier and Type | Field and Description |
---|---|
protected boolean |
changelogDecoupled |
protected Map<BinaryRow,Set<Integer>> |
deletionBuckets |
protected FileIO |
fileIO |
protected IndexFileHandler |
indexFileHandler |
protected ManifestFile |
manifestFile |
protected ManifestList |
manifestList |
protected FileStorePathFactory |
pathFactory |
protected StatsFileHandler |
statsFileHandler |
Constructor and Description |
---|
FileDeletionBase(FileIO fileIO,
FileStorePathFactory pathFactory,
ManifestFile manifestFile,
ManifestList manifestList,
IndexFileHandler indexFileHandler,
StatsFileHandler statsFileHandler,
boolean cleanEmptyDirectories,
int deleteFileThreadNum) |
Modifier and Type | Method and Description |
---|---|
protected void |
addMergedDataFiles(Map<BinaryRow,Map<Integer,Set<String>>> dataFiles,
Snapshot snapshot)
NOTE: This method is used for building data file skipping set.
|
void |
cleanEmptyDirectories()
Try to delete data directories that may be empty after data file deletion.
|
void |
cleanUnusedDataFiles(String manifestList,
java.util.function.Predicate<ManifestEntry> skipper) |
abstract void |
cleanUnusedDataFiles(T snapshot,
java.util.function.Predicate<ManifestEntry> skipper)
Clean data files that will not be used anymore in the snapshot.
|
void |
cleanUnusedIndexManifests(Snapshot snapshot,
Set<String> skippingSet) |
void |
cleanUnusedManifestList(String manifestName,
Set<String> skippingSet) |
protected void |
cleanUnusedManifests(Snapshot snapshot,
Set<String> skippingSet,
boolean deleteDataManifestLists,
boolean deleteChangelog) |
abstract void |
cleanUnusedManifests(T snapshot,
Set<String> skippingSet)
Clean metadata files that will not be used anymore of a snapshot, including data manifests,
index manifests and manifest lists.
|
void |
cleanUnusedStatisticsManifests(Snapshot snapshot,
Set<String> skippingSet) |
protected boolean |
containsDataFile(Map<BinaryRow,Map<Integer,Set<String>>> dataFiles,
ManifestEntry testee) |
java.util.function.Predicate<ManifestEntry> |
createDataFileSkipperForTags(List<Snapshot> taggedSnapshots,
long expiringSnapshotId) |
void |
deleteAddedDataFiles(String manifestListName)
Delete added file in the manifest list files.
|
protected <F> void |
deleteFiles(Collection<F> files,
java.util.function.Consumer<F> deletion) |
protected void |
doCleanUnusedDataFile(Map<Path,Pair<ManifestEntry,List<Path>>> dataFileToDelete,
java.util.function.Predicate<ManifestEntry> skipper) |
protected void |
getDataFileToDelete(Map<Path,Pair<ManifestEntry,List<Path>>> dataFileToDelete,
List<ManifestEntry> dataFileEntries) |
Set<String> |
manifestSkippingSet(List<Snapshot> skippingSnapshots) |
Set<String> |
manifestSkippingSet(Snapshot skippingSnapshot)
Changelogs were not checked.
|
protected List<String> |
readManifestFileNames(List<ManifestFileMeta> manifestFileMetas) |
protected Collection<ManifestEntry> |
readMergedDataFiles(Snapshot snapshot) |
protected void |
recordDeletionBuckets(ManifestEntry entry) |
void |
setChangelogDecoupled(boolean changelogDecoupled) |
protected List<String> |
tryReadDataManifests(Snapshot snapshot) |
protected List<ManifestFileMeta> |
tryReadManifestList(String manifestListName)
It is possible that a job was killed during expiration and some manifest files have been
deleted, so if the clean methods need to get manifests of a snapshot to be cleaned, we should
try to read manifests and return empty list if failed instead of calling
ManifestList.readDataManifests(org.apache.paimon.Snapshot) directly. |
protected final FileIO fileIO
protected final FileStorePathFactory pathFactory
protected final ManifestFile manifestFile
protected final ManifestList manifestList
protected final IndexFileHandler indexFileHandler
protected final StatsFileHandler statsFileHandler
protected boolean changelogDecoupled
public FileDeletionBase(FileIO fileIO, FileStorePathFactory pathFactory, ManifestFile manifestFile, ManifestList manifestList, IndexFileHandler indexFileHandler, StatsFileHandler statsFileHandler, boolean cleanEmptyDirectories, int deleteFileThreadNum)
public abstract void cleanUnusedDataFiles(T snapshot, java.util.function.Predicate<ManifestEntry> skipper)
snapshot
- Snapshot
that will be cleanedskipper
- if the test result of a data file is true, it will be skipped when deleting;
else it will be deletedpublic abstract void cleanUnusedManifests(T snapshot, Set<String> skippingSet)
snapshot
- Snapshot
that will be cleanedskippingSet
- manifests that should not be deletedpublic void setChangelogDecoupled(boolean changelogDecoupled)
public void cleanEmptyDirectories()
protected void recordDeletionBuckets(ManifestEntry entry)
public void cleanUnusedDataFiles(String manifestList, java.util.function.Predicate<ManifestEntry> skipper)
protected void doCleanUnusedDataFile(Map<Path,Pair<ManifestEntry,List<Path>>> dataFileToDelete, java.util.function.Predicate<ManifestEntry> skipper)
protected void getDataFileToDelete(Map<Path,Pair<ManifestEntry,List<Path>>> dataFileToDelete, List<ManifestEntry> dataFileEntries)
public void deleteAddedDataFiles(String manifestListName)
manifestListName
- name of manifest listpublic void cleanUnusedStatisticsManifests(Snapshot snapshot, Set<String> skippingSet)
public void cleanUnusedIndexManifests(Snapshot snapshot, Set<String> skippingSet)
public void cleanUnusedManifestList(String manifestName, Set<String> skippingSet)
protected void cleanUnusedManifests(Snapshot snapshot, Set<String> skippingSet, boolean deleteDataManifestLists, boolean deleteChangelog)
public java.util.function.Predicate<ManifestEntry> createDataFileSkipperForTags(List<Snapshot> taggedSnapshots, long expiringSnapshotId) throws Exception
Exception
protected List<ManifestFileMeta> tryReadManifestList(String manifestListName)
ManifestList.readDataManifests(org.apache.paimon.Snapshot)
directly.protected List<String> readManifestFileNames(List<ManifestFileMeta> manifestFileMetas)
protected void addMergedDataFiles(Map<BinaryRow,Map<Integer,Set<String>>> dataFiles, Snapshot snapshot) throws IOException
IOException
protected Collection<ManifestEntry> readMergedDataFiles(Snapshot snapshot) throws IOException
IOException
protected boolean containsDataFile(Map<BinaryRow,Map<Integer,Set<String>>> dataFiles, ManifestEntry testee)
public Set<String> manifestSkippingSet(Snapshot skippingSnapshot)
protected <F> void deleteFiles(Collection<F> files, java.util.function.Consumer<F> deletion)
Copyright © 2023–2024 The Apache Software Foundation. All rights reserved.