org.apache.paimon.spark

Class SparkCatalog

java.lang.Object

org.apache.paimon.spark.catalog.SparkBaseCatalog

org.apache.paimon.spark.SparkCatalog

All Implemented Interfaces:

ProcedureCatalog, SupportFunction, WithPaimonCatalog, org.apache.spark.sql.connector.catalog.CatalogPlugin, org.apache.spark.sql.connector.catalog.FunctionCatalog, org.apache.spark.sql.connector.catalog.SupportsNamespaces, org.apache.spark.sql.connector.catalog.TableCatalog

public class SparkCatalog
extends SparkBaseCatalog
implements SupportFunction

Spark TableCatalog for paimon.

Field Summary

Fields

Modifier and Type	Field and Description
protected Catalog	catalog

Fields inherited from class org.apache.paimon.spark.catalog.SparkBaseCatalog

catalogName

Fields inherited from interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

PROP_COMMENT, PROP_LOCATION, PROP_OWNER

Fields inherited from interface org.apache.spark.sql.connector.catalog.TableCatalog

OPTION_PREFIX, PROP_COMMENT, PROP_EXTERNAL, PROP_IS_MANAGED_LOCATION, PROP_LOCATION, PROP_OWNER, PROP_PROVIDER

Constructor Summary

Constructors

Constructor and Description
SparkCatalog()

Method Summary

Modifier and Type	Method and Description
void	alterNamespace(String[] namespace, org.apache.spark.sql.connector.catalog.NamespaceChange... changes)
org.apache.spark.sql.connector.catalog.Table	alterTable(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.connector.catalog.TableChange... changes)
protected List<String>	convertPartitionTransforms(org.apache.spark.sql.connector.expressions.Transform[] transforms)
void	createNamespace(String[] namespace, Map<String,String> metadata)
org.apache.paimon.spark.SparkTable	createTable(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] partitions, Map<String,String> properties)
String[]	defaultNamespace()
boolean	dropNamespace(String[] namespace) Drop a namespace from the catalog, recursively dropping all objects within the namespace.
boolean	dropNamespace(String[] namespace, boolean cascade) Drop a namespace from the catalog with cascade mode, recursively dropping all objects within the namespace if cascade is true.
boolean	dropTable(org.apache.spark.sql.connector.catalog.Identifier ident)
void	initialize(String name, org.apache.spark.sql.util.CaseInsensitiveStringMap options)
void	invalidateTable(org.apache.spark.sql.connector.catalog.Identifier ident)
String[][]	listNamespaces()
String[][]	listNamespaces(String[] namespace)
org.apache.spark.sql.connector.catalog.Identifier[]	listTables(String[] namespace)
Map<String,String>	loadNamespaceMetadata(String[] namespace)
protected org.apache.paimon.spark.SparkTable	loadSparkTable(org.apache.spark.sql.connector.catalog.Identifier ident, Map<String,String> extraOptions)
org.apache.paimon.spark.SparkTable	loadTable(org.apache.spark.sql.connector.catalog.Identifier ident)
org.apache.paimon.spark.SparkTable	loadTable(org.apache.spark.sql.connector.catalog.Identifier ident, long timestamp) Do not annotate with @override here to maintain compatibility with Spark 3.2-.
org.apache.paimon.spark.SparkTable	loadTable(org.apache.spark.sql.connector.catalog.Identifier ident, String version) Do not annotate with @override here to maintain compatibility with Spark 3.2-.
Catalog	paimonCatalog()
void	renameTable(org.apache.spark.sql.connector.catalog.Identifier oldIdent, org.apache.spark.sql.connector.catalog.Identifier newIdent)
boolean	tableExists(org.apache.spark.sql.connector.catalog.Identifier ident)
protected Identifier	toIdentifier(org.apache.spark.sql.connector.catalog.Identifier ident)

Methods inherited from class org.apache.paimon.spark.catalog.SparkBaseCatalog

loadProcedure, name, usePaimon

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.paimon.spark.catalog.SupportFunction

isFunctionNamespace, isSystemNamespace, listFunctions, loadFunction

Methods inherited from interface org.apache.spark.sql.connector.catalog.FunctionCatalog

functionExists

Methods inherited from interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

namespaceExists

Methods inherited from interface org.apache.spark.sql.connector.catalog.CatalogPlugin

name

Methods inherited from interface org.apache.spark.sql.connector.catalog.TableCatalog

capabilities, createTable, purgeTable, useNullableQuerySchema

Field Detail

catalog

protected Catalog catalog

Constructor Detail

SparkCatalog

public SparkCatalog()

Method Detail

initialize

public void initialize(String name,
                       org.apache.spark.sql.util.CaseInsensitiveStringMap options)

Specified by:

initialize in interface org.apache.spark.sql.connector.catalog.CatalogPlugin

paimonCatalog

public Catalog paimonCatalog()

Specified by:

paimonCatalog in interface WithPaimonCatalog

defaultNamespace

public String[] defaultNamespace()

Specified by:

defaultNamespace in interface org.apache.spark.sql.connector.catalog.CatalogPlugin

createNamespace

public void createNamespace(String[] namespace,
                            Map<String,String> metadata)
                     throws org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException

Specified by:

createNamespace in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

Throws:

org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException

listNamespaces

public String[][] listNamespaces()

Specified by:

listNamespaces in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

listNamespaces

public String[][] listNamespaces(String[] namespace)
                          throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

Specified by:

listNamespaces in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

loadNamespaceMetadata

public Map<String,String> loadNamespaceMetadata(String[] namespace)
                                         throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

Specified by:

loadNamespaceMetadata in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

dropNamespace

public boolean dropNamespace(String[] namespace)
                      throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

Drop a namespace from the catalog, recursively dropping all objects within the namespace. This interface implementation only supports the Spark 3.0, 3.1 and 3.2.

If the catalog implementation does not support this operation, it may throw UnsupportedOperationException.

Parameters:

namespace - a multi-part namespace

Returns:

true if the namespace was dropped

Throws:

UnsupportedOperationException - If drop is not a supported operation

org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

dropNamespace

public boolean dropNamespace(String[] namespace,
                             boolean cascade)
                      throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

Drop a namespace from the catalog with cascade mode, recursively dropping all objects within the namespace if cascade is true. This interface implementation supports the Spark 3.3+.

If the catalog implementation does not support this operation, it may throw UnsupportedOperationException.

Specified by:

dropNamespace in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

Parameters:

namespace - a multi-part namespace

cascade - When true, deletes all objects under the namespace

Returns:

true if the namespace was dropped

Throws:

UnsupportedOperationException - If drop is not a supported operation

org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

listTables

public org.apache.spark.sql.connector.catalog.Identifier[] listTables(String[] namespace)
                                                               throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

Specified by:

listTables in interface org.apache.spark.sql.connector.catalog.TableCatalog

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

invalidateTable

public void invalidateTable(org.apache.spark.sql.connector.catalog.Identifier ident)

Specified by:

invalidateTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

loadTable

public org.apache.paimon.spark.SparkTable loadTable(org.apache.spark.sql.connector.catalog.Identifier ident)
                                             throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException

Specified by:

loadTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchTableException

loadTable

public org.apache.paimon.spark.SparkTable loadTable(org.apache.spark.sql.connector.catalog.Identifier ident,
                                                    String version)
                                             throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException

Do not annotate with @override here to maintain compatibility with Spark 3.2-.

Specified by:

loadTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchTableException

loadTable

public org.apache.paimon.spark.SparkTable loadTable(org.apache.spark.sql.connector.catalog.Identifier ident,
                                                    long timestamp)
                                             throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException

Do not annotate with @override here to maintain compatibility with Spark 3.2-.

NOTE: Time unit of timestamp here is microsecond (see TableCatalog.loadTable(Identifier, long)). But in SQL you should use seconds.

Specified by:

loadTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchTableException

tableExists

public boolean tableExists(org.apache.spark.sql.connector.catalog.Identifier ident)

Specified by:

tableExists in interface org.apache.spark.sql.connector.catalog.TableCatalog

alterTable

public org.apache.spark.sql.connector.catalog.Table alterTable(org.apache.spark.sql.connector.catalog.Identifier ident,
                                                               org.apache.spark.sql.connector.catalog.TableChange... changes)
                                                        throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException

Specified by:

alterTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchTableException

createTable

public org.apache.paimon.spark.SparkTable createTable(org.apache.spark.sql.connector.catalog.Identifier ident,
                                                      org.apache.spark.sql.types.StructType schema,
                                                      org.apache.spark.sql.connector.expressions.Transform[] partitions,
                                                      Map<String,String> properties)
                                               throws org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException,
                                                      org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

Specified by:

createTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

Throws:

org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException

org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

dropTable

public boolean dropTable(org.apache.spark.sql.connector.catalog.Identifier ident)

Specified by:

dropTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

renameTable

public void renameTable(org.apache.spark.sql.connector.catalog.Identifier oldIdent,
                        org.apache.spark.sql.connector.catalog.Identifier newIdent)
                 throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException,
                        org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException

Specified by:

renameTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchTableException

org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException

toIdentifier

protected Identifier toIdentifier(org.apache.spark.sql.connector.catalog.Identifier ident)
                           throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchTableException

loadSparkTable

protected org.apache.paimon.spark.SparkTable loadSparkTable(org.apache.spark.sql.connector.catalog.Identifier ident,
                                                            Map<String,String> extraOptions)
                                                     throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchTableException

convertPartitionTransforms

protected List<String> convertPartitionTransforms(org.apache.spark.sql.connector.expressions.Transform[] transforms)

alterNamespace

public void alterNamespace(String[] namespace,
                           org.apache.spark.sql.connector.catalog.NamespaceChange... changes)

Specified by:

alterNamespace in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces