public class WriterImpl extends Object implements org.apache.orc.impl.WriterInternal, org.apache.orc.MemoryManager.Callback
This class is unsynchronized like most Stream objects, so from the creation of an OrcFile and all access to a single instance has to be from a single thread.
There are no known cases where these happen between different threads today.
Caveat: the MemoryManager is created during WriterOptions create, that has to be confined to a single thread as well.
NOTE: The file was copied and modified to support zstd-jni. This feature is only supported in ORC 2.0, but 2.0 only supports JDK17. We need to support JDK8.
Constructor and Description |
---|
WriterImpl(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path,
OrcFile.WriterOptions opts) |
Modifier and Type | Method and Description |
---|---|
void |
addRowBatch(org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch batch) |
void |
addUserMetadata(String name,
ByteBuffer value) |
void |
appendStripe(byte[] stripe,
int offset,
int length,
org.apache.orc.StripeInformation stripeInfo,
org.apache.orc.OrcProto.StripeStatistics stripeStatistics) |
void |
appendStripe(byte[] stripe,
int offset,
int length,
org.apache.orc.StripeInformation stripeInfo,
org.apache.orc.StripeStatistics[] stripeStatistics) |
void |
appendUserMetadata(List<org.apache.orc.OrcProto.UserMetadataItem> userMetadata) |
boolean |
checkMemory(double newScale) |
void |
close() |
static org.apache.orc.CompressionCodec |
createCodec(CompressionKind kind) |
long |
estimateMemory() |
org.apache.orc.CompressionCodec |
getCompressionCodec() |
static int |
getEstimatedBufferSize(long stripeSize,
int numColumns,
int bs) |
long |
getNumberOfRows()
Row count gets updated when flushing the stripes.
|
long |
getRawDataSize()
Raw data size will be compute when writing the file footer.
|
org.apache.orc.TypeDescription |
getSchema() |
org.apache.orc.ColumnStatistics[] |
getStatistics() |
List<org.apache.orc.StripeInformation> |
getStripes() |
void |
increaseCompressionSize(int newSize) |
long |
writeIntermediateFooter() |
public WriterImpl(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, OrcFile.WriterOptions opts) throws IOException
IOException
public static int getEstimatedBufferSize(long stripeSize, int numColumns, int bs)
public void increaseCompressionSize(int newSize)
increaseCompressionSize
in interface org.apache.orc.impl.WriterInternal
public static org.apache.orc.CompressionCodec createCodec(CompressionKind kind)
public boolean checkMemory(double newScale) throws IOException
checkMemory
in interface org.apache.orc.MemoryManager.Callback
IOException
public org.apache.orc.TypeDescription getSchema()
getSchema
in interface org.apache.orc.Writer
public void addUserMetadata(String name, ByteBuffer value)
addUserMetadata
in interface org.apache.orc.Writer
public void addRowBatch(org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch batch) throws IOException
addRowBatch
in interface org.apache.orc.Writer
IOException
public void close() throws IOException
close
in interface Closeable
close
in interface AutoCloseable
close
in interface org.apache.orc.Writer
IOException
public long getRawDataSize()
getRawDataSize
in interface org.apache.orc.Writer
public long getNumberOfRows()
getNumberOfRows
in interface org.apache.orc.Writer
public long writeIntermediateFooter() throws IOException
writeIntermediateFooter
in interface org.apache.orc.Writer
IOException
public void appendStripe(byte[] stripe, int offset, int length, org.apache.orc.StripeInformation stripeInfo, org.apache.orc.OrcProto.StripeStatistics stripeStatistics) throws IOException
appendStripe
in interface org.apache.orc.Writer
IOException
public void appendStripe(byte[] stripe, int offset, int length, org.apache.orc.StripeInformation stripeInfo, org.apache.orc.StripeStatistics[] stripeStatistics) throws IOException
IOException
public void appendUserMetadata(List<org.apache.orc.OrcProto.UserMetadataItem> userMetadata)
appendUserMetadata
in interface org.apache.orc.Writer
public org.apache.orc.ColumnStatistics[] getStatistics()
getStatistics
in interface org.apache.orc.Writer
public List<org.apache.orc.StripeInformation> getStripes() throws IOException
IOException
public org.apache.orc.CompressionCodec getCompressionCodec()
public long estimateMemory()
Copyright © 2023–2024 The Apache Software Foundation. All rights reserved.