org.apache.paimon.spark

Class SparkGenericCatalog

java.lang.Object

org.apache.paimon.spark.catalog.SparkBaseCatalog

org.apache.paimon.spark.SparkGenericCatalog

All Implemented Interfaces:

ProcedureCatalog, WithPaimonCatalog, org.apache.spark.sql.connector.catalog.CatalogExtension, org.apache.spark.sql.connector.catalog.CatalogPlugin, org.apache.spark.sql.connector.catalog.FunctionCatalog, org.apache.spark.sql.connector.catalog.SupportsNamespaces, org.apache.spark.sql.connector.catalog.TableCatalog

public class SparkGenericCatalog
extends SparkBaseCatalog
implements org.apache.spark.sql.connector.catalog.CatalogExtension

A Spark catalog that can also load non-Paimon tables.

Most of the content of this class is referenced from Iceberg's SparkSessionCatalog.

Field Summary

Fields inherited from class org.apache.paimon.spark.catalog.SparkBaseCatalog

catalogName

Fields inherited from interface org.apache.spark.sql.connector.catalog.TableCatalog

OPTION_PREFIX, PROP_COMMENT, PROP_EXTERNAL, PROP_IS_MANAGED_LOCATION, PROP_LOCATION, PROP_OWNER, PROP_PROVIDER

Fields inherited from interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

PROP_COMMENT, PROP_LOCATION, PROP_OWNER

Constructor Summary

Constructors

Constructor and Description
SparkGenericCatalog()

Method Summary

Modifier and Type	Method and Description
void	alterNamespace(String[] namespace, org.apache.spark.sql.connector.catalog.NamespaceChange... changes)
org.apache.spark.sql.connector.catalog.Table	alterTable(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.connector.catalog.TableChange... changes)
void	createNamespace(String[] namespace, Map<String,String> metadata)
org.apache.spark.sql.connector.catalog.Table	createTable(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] partitions, Map<String,String> properties)
String[]	defaultNamespace()
boolean	dropNamespace(String[] namespace, boolean cascade)
boolean	dropTable(org.apache.spark.sql.connector.catalog.Identifier ident)
void	initialize(String name, org.apache.spark.sql.util.CaseInsensitiveStringMap options)
void	invalidateTable(org.apache.spark.sql.connector.catalog.Identifier ident)
org.apache.spark.sql.connector.catalog.Identifier[]	listFunctions(String[] namespace)
String[][]	listNamespaces()
String[][]	listNamespaces(String[] namespace)
org.apache.spark.sql.connector.catalog.Identifier[]	listTables(String[] namespace)
org.apache.spark.sql.connector.catalog.functions.UnboundFunction	loadFunction(org.apache.spark.sql.connector.catalog.Identifier ident)
Map<String,String>	loadNamespaceMetadata(String[] namespace)
org.apache.spark.sql.connector.catalog.Table	loadTable(org.apache.spark.sql.connector.catalog.Identifier ident)
org.apache.spark.sql.connector.catalog.Table	loadTable(org.apache.spark.sql.connector.catalog.Identifier ident, long timestamp)
org.apache.spark.sql.connector.catalog.Table	loadTable(org.apache.spark.sql.connector.catalog.Identifier ident, String version)
boolean	namespaceExists(String[] namespace)
Catalog	paimonCatalog()
boolean	purgeTable(org.apache.spark.sql.connector.catalog.Identifier ident)
void	renameTable(org.apache.spark.sql.connector.catalog.Identifier from, org.apache.spark.sql.connector.catalog.Identifier to)
void	setDelegateCatalog(org.apache.spark.sql.connector.catalog.CatalogPlugin delegate)

Methods inherited from class org.apache.paimon.spark.catalog.SparkBaseCatalog

loadProcedure, name, usePaimon

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.spark.sql.connector.catalog.TableCatalog

capabilities, createTable, loadTable, tableExists, useNullableQuerySchema

Methods inherited from interface org.apache.spark.sql.connector.catalog.FunctionCatalog

functionExists

Methods inherited from interface org.apache.spark.sql.connector.catalog.CatalogPlugin

name

Constructor Detail

SparkGenericCatalog

public SparkGenericCatalog()

Method Detail

paimonCatalog

public Catalog paimonCatalog()

Specified by:

paimonCatalog in interface WithPaimonCatalog

defaultNamespace

public String[] defaultNamespace()

Specified by:

defaultNamespace in interface org.apache.spark.sql.connector.catalog.CatalogPlugin

listNamespaces

public String[][] listNamespaces()
                          throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

Specified by:

listNamespaces in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

listNamespaces

public String[][] listNamespaces(String[] namespace)
                          throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

Specified by:

listNamespaces in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

namespaceExists

public boolean namespaceExists(String[] namespace)

Specified by:

namespaceExists in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

loadNamespaceMetadata

public Map<String,String> loadNamespaceMetadata(String[] namespace)
                                         throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

Specified by:

loadNamespaceMetadata in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

createNamespace

public void createNamespace(String[] namespace,
                            Map<String,String> metadata)
                     throws org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException

Specified by:

createNamespace in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

Throws:

org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException

alterNamespace

public void alterNamespace(String[] namespace,
                           org.apache.spark.sql.connector.catalog.NamespaceChange... changes)
                    throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

Specified by:

alterNamespace in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

dropNamespace

public boolean dropNamespace(String[] namespace,
                             boolean cascade)
                      throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException,
                             org.apache.spark.sql.catalyst.analysis.NonEmptyNamespaceException

Specified by:

dropNamespace in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

org.apache.spark.sql.catalyst.analysis.NonEmptyNamespaceException

listTables

public org.apache.spark.sql.connector.catalog.Identifier[] listTables(String[] namespace)
                                                               throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

Specified by:

listTables in interface org.apache.spark.sql.connector.catalog.TableCatalog

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

loadTable

public org.apache.spark.sql.connector.catalog.Table loadTable(org.apache.spark.sql.connector.catalog.Identifier ident)
                                                       throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException

Specified by:

loadTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchTableException

loadTable

public org.apache.spark.sql.connector.catalog.Table loadTable(org.apache.spark.sql.connector.catalog.Identifier ident,
                                                              String version)
                                                       throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException

Specified by:

loadTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchTableException

loadTable

public org.apache.spark.sql.connector.catalog.Table loadTable(org.apache.spark.sql.connector.catalog.Identifier ident,
                                                              long timestamp)
                                                       throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException

Specified by:

loadTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchTableException

invalidateTable

public void invalidateTable(org.apache.spark.sql.connector.catalog.Identifier ident)

Specified by:

invalidateTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

createTable

public org.apache.spark.sql.connector.catalog.Table createTable(org.apache.spark.sql.connector.catalog.Identifier ident,
                                                                org.apache.spark.sql.types.StructType schema,
                                                                org.apache.spark.sql.connector.expressions.Transform[] partitions,
                                                                Map<String,String> properties)
                                                         throws org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException,
                                                                org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

Specified by:

createTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

Throws:

org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException

org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

alterTable

public org.apache.spark.sql.connector.catalog.Table alterTable(org.apache.spark.sql.connector.catalog.Identifier ident,
                                                               org.apache.spark.sql.connector.catalog.TableChange... changes)
                                                        throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException

Specified by:

alterTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchTableException

dropTable

public boolean dropTable(org.apache.spark.sql.connector.catalog.Identifier ident)

Specified by:

dropTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

purgeTable

public boolean purgeTable(org.apache.spark.sql.connector.catalog.Identifier ident)

Specified by:

purgeTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

renameTable

public void renameTable(org.apache.spark.sql.connector.catalog.Identifier from,
                        org.apache.spark.sql.connector.catalog.Identifier to)
                 throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException,
                        org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException

Specified by:

renameTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchTableException

org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException

initialize

public final void initialize(String name,
                             org.apache.spark.sql.util.CaseInsensitiveStringMap options)

Specified by:

initialize in interface org.apache.spark.sql.connector.catalog.CatalogPlugin

setDelegateCatalog

public void setDelegateCatalog(org.apache.spark.sql.connector.catalog.CatalogPlugin delegate)

Specified by:

setDelegateCatalog in interface org.apache.spark.sql.connector.catalog.CatalogExtension

listFunctions

public org.apache.spark.sql.connector.catalog.Identifier[] listFunctions(String[] namespace)
                                                                  throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

Specified by:

listFunctions in interface org.apache.spark.sql.connector.catalog.FunctionCatalog

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

loadFunction

public org.apache.spark.sql.connector.catalog.functions.UnboundFunction loadFunction(org.apache.spark.sql.connector.catalog.Identifier ident)
                                                                              throws org.apache.spark.sql.catalyst.analysis.NoSuchFunctionException

Specified by:

loadFunction in interface org.apache.spark.sql.connector.catalog.FunctionCatalog

Throws:

org.apache.spark.sql.catalyst.analysis.NoSuchFunctionException