ParquetMetadataHandle

Instance Constructors

new ParquetMetadataHandle(session: SparkSession, tableIdentifier: String)

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clean(): Unit

Cleans the MetadataHandle instance (implementation specific)
Cleans the MetadataHandle instance (implementation specific)

Definition Classes
MetadataHandle
def clearStats(): Unit

Clears the latest stats
Clears the latest stats

Definition Classes
MetadataHandle
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def dropAllMetadata(): Unit

Drops all of the metadata associated with the given index
Drops all of the metadata associated with the given index

Definition Classes
ParquetMetadataHandle → MetadataHandle
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def finalizeMetadataUpload(): Unit

Finalize metadata creation in the metadatastore (implementation specific)
Finalize metadata creation in the metadatastore (implementation specific)

Definition Classes
ParquetMetadataHandle → MetadataHandle
def getAllIndexedFiles(): Future[Set[String]]

Returns a set of all indexed files (async)
Returns a set of all indexed files (async)

Definition Classes
ParquetMetadataHandle → MetadataHandle
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getDeletionChunkSize(): Int

returns
Maximum number of objects to delete in one chunk

Definition Classes
ParquetMetadataHandle → MetadataHandle
def getIndexStatus(): IndexStatusResult

Returns index statistics
Returns index statistics

Definition Classes
ParquetMetadataHandle → MetadataHandle
def getIndexes(): Seq[Index]

returns the sequence of indexes that exist in the metadatastore for the tableIdentifier
returns the sequence of indexes that exist in the metadatastore for the tableIdentifier

Definition Classes
ParquetMetadataHandle → MetadataHandle
def getMdVersionStatus(): MetadataVersionStatus

returns the version status of the metadata.
returns the version status of the metadata. we do not have a strict requirement for the metadatastore to use metadata from a version different than its current version for filtering / refresh but we do expect it to be able to tell the version status, and whether or not it can be upgraded to comply with the current version.

Definition Classes
ParquetMetadataHandle → MetadataHandle
def getNumberOfIndexedObjects(): Long

Gets the number of indexed objects
Gets the number of indexed objects
returns
the number of indexed objects
def getRequiredObjects(query: Any): Future[Set[String]]

Returns the required file ids for the given query (async)
Returns the required file ids for the given query (async)
query
the query to be used in order to get the relevant files (this query is of type Any and it is the responsibility of the metadatastore implementation to cast it to as instance which matches the translation for this MetaDataStore)
returns
the set of fileids required for this query

Definition Classes
ParquetMetadataHandle → MetadataHandle
def getStats(): Option[QueryIndexStatsResult]

Definition Classes
MetadataHandle
def getUploadChunkSize(): Int

returns
Maximum number of objects to index in one chunk

Definition Classes
ParquetMetadataHandle → MetadataHandle
def hashCode(): Int

Definition Classes
AnyRef → Any
def indexExists(): Boolean

Returns true if an index exists for the tableIdentifier
Returns true if an index exists for the tableIdentifier
returns
true if the an index exists for the tableIdentifier

Definition Classes
ParquetMetadataHandle → MetadataHandle
def initMetadataUpload(indexes: Seq[Index]): Unit

Initialize metadata for a dataset in the metadatastore (implementation specific)
Initialize metadata for a dataset in the metadatastore (implementation specific)
indexes
a sequence of metadata indexes

Definition Classes
MetadataHandle
def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

Attributes
protected
Definition Classes
Logging
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes
protected
Definition Classes
Logging
def isEncryptionSupported(): Boolean

Returns whether or not this MetadataHandle supports encryption
Returns whether or not this MetadataHandle supports encryption
returns
true if the MetadataHandle supports encryption

Definition Classes
ParquetMetadataHandle → MetadataHandle
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isMetadataUpgradePossible(): Boolean

returns whether or not the metadata can be upgraded to comply with the current version
returns whether or not the metadata can be upgraded to comply with the current version
returns
true if the metadata can be upgraded, false otherwise

Definition Classes
ParquetMetadataHandle → MetadataHandle
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def refresh(): Unit

Refreshes the MetadataHandle by re-syncing with the metadatastore (implementation specific)
Refreshes the MetadataHandle by re-syncing with the metadatastore (implementation specific)

Definition Classes
ParquetMetadataHandle → MetadataHandle
def removeMetaDataForFiles(files: Seq[String]): Unit

Removes the metadata for a sequence of files.
Removes the metadata for a sequence of files. This method may assume that the Metadta version status is MetadataVersionStatus.CURRENT
files
a sequence of files for which the metadata will be removed

Definition Classes
ParquetMetadataHandle → MetadataHandle
val session: SparkSession
def setMDPath(md: String, type: String): ParquetMetadataHandle
def setParams(params: Map[String, String]): Unit

Set metadataStore specific parameters such as user, password, host Note: the implementation should not count on the Map not changing during the run therefore, it needs to save its parameters locally
Set metadataStore specific parameters such as user, password, host Note: the implementation should not count on the Map not changing during the run therefore, it needs to save its parameters locally
params
a map of parameters (each metadata store expects certain input)

Definition Classes
ParquetMetadataHandle → MetadataHandle
final def setParams(params: Map[String, String]): Unit

Definition Classes
MetadataHandle
var stats: Option[QueryIndexStatsResult]

Attributes
protected
Definition Classes
MetadataHandle
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
def updateQueryStats(currentFilterStatistics: QueryIndexStatsResult): Unit

Updates the MetadataHandle stats with the given query stats called from the dataSkippingFileIndex rule in the catalyst to update the stats during the query tun time
Updates the MetadataHandle stats with the given query stats called from the dataSkippingFileIndex rule in the catalyst to update the stats during the query tun time
currentFilterStatistics
the updated stats

Definition Classes
MetadataHandle
def upgradeMetadata(indexes: Seq[Index]): Unit

Upgrades the metadata to comply with the current version
Upgrades the metadata to comply with the current version
indexes
- the indexes stored in the metadataStore.

Definition Classes
ParquetMetadataHandle → MetadataHandle
def uploadMetadata(metaData: RDD[Row], indexes: Seq[Index], isRefresh: Boolean): Unit

Uploads the metadata to the metadatastore This method may assume that the metadata version status is MetadataVersionStatus.CURRENT
Uploads the metadata to the metadatastore This method may assume that the metadata version status is MetadataVersionStatus.CURRENT
metaData
RDD that contains for each file a list of abstract metaData types to be uploaded
indexes
a sequence of indexes that created the metadata
isRefresh
indicates whether the operation is a refresh operation

Definition Classes
ParquetMetadataHandle → MetadataHandle
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package parquet

class ParquetMetadataHandle extends MetadataHandle with Logging with Serializable

Instance Constructors

new ParquetMetadataHandle(session: SparkSession, tableIdentifier: String)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clean(): Unit

def clearStats(): Unit

def clone(): AnyRef

def dropAllMetadata(): Unit

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

def finalizeMetadataUpload(): Unit

def getAllIndexedFiles(): Future[Set[String]]

final def getClass(): Class[_]

def getDeletionChunkSize(): Int

def getIndexStatus(): IndexStatusResult

def getIndexes(): Seq[Index]

def getMdVersionStatus(): MetadataVersionStatus

def getNumberOfIndexedObjects(): Long

def getRequiredObjects(query: Any): Future[Set[String]]

def getStats(): Option[QueryIndexStatsResult]

def getUploadChunkSize(): Int

def hashCode(): Int

def indexExists(): Boolean

def initMetadataUpload(indexes: Seq[Index]): Unit

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

def isEncryptionSupported(): Boolean

final def isInstanceOf[T0]: Boolean

def isMetadataUpgradePossible(): Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def refresh(): Unit

def removeMetaDataForFiles(files: Seq[String]): Unit

val session: SparkSession

def setMDPath(md: String, type: String): ParquetMetadataHandle

def setParams(params: Map[String, String]): Unit

final def setParams(params: Map[String, String]): Unit

var stats: Option[QueryIndexStatsResult]

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

def updateQueryStats(currentFilterStatistics: QueryIndexStatsResult): Unit

def upgradeMetadata(indexes: Seq[Index]): Unit

def uploadMetadata(metaData: RDD[Row], indexes: Seq[Index], isRefresh: Boolean): Unit

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Serializable

Inherited from Serializable

Inherited from Logging

Inherited from MetadataHandle

Inherited from AnyRef

Inherited from Any

Ungrouped