DataSkippingFileFilter

Instance Constructors

new DataSkippingFileFilter(tid: String, metadataStoreManager: MetadataStoreManager, sparkSession: SparkSession, params: Map[String, String] = Map.empty[String, String])

tid
the table identifier for which the DataSkippingFileFilter is built
metadataStoreManager
the MetadataStoreManager to be used in order to get io.xskipper.metadatastore.MetadataHandle for the identifier
params
a map of parameters to be set by the DataSkippingFileFilter on the io.xskipper.metadatastore.MetadataHandle

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clearCurrentFilterStatistics(): Unit

Clears the current filter statistics.
Clears the current filter statistics. Should be called at the end each call to handleStatistics which updates the accumulated stats

Attributes
protected
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
val currentFilterStatistics: HashMap[String, Long]

Attributes
protected
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def handleStatistics(): Unit

Update the IndexMeta associated with the table
def hashCode(): Int

Definition Classes
AnyRef → Any
def init(dataFilters: Seq[Expression], partitionFilters: Seq[Expression], metadataFilterFactories: Seq[MetadataFilterFactory], clauseTranslators: Seq[ClauseTranslator]): Unit

Filters the partition directory by removing unnecessary objects from each partition directory
Filters the partition directory by removing unnecessary objects from each partition directory
dataFilters
query predicates for actual data columns (not partitions)
partitionFilters
the partition predicates from the query
metadataFilterFactories
a sequence of MetadataFilterFactory to generate filters according to the index on the dataset
clauseTranslators
a sequence of ClauseTranslators to be applied on the clauses
returns
a sequence of PartitionDirectory after filtering the unnecessary objects using the metadata
def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

Attributes
protected
Definition Classes
Logging
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes
protected
Definition Classes
Logging
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isRequired(fs: FileStatus): Boolean

Returns true if the current file is required for the given query by checking if it is present in the required files or not indexed
Returns true if the current file is required for the given query by checking if it is present in the required files or not indexed
fs
the file status to check
returns
true if the file is required, false otherwise
def isSkipabble(): Boolean

returns
true if the current query is relevant for skipping. i.e - it has indexed files and a metadata query can be generated
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
lazy val metadataHandler: MetadataHandle

Attributes
protected
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
def updateStats(fs: FileStatus, shouldScan: Boolean): Unit

Update the stats for a given fileStatus
Update the stats for a given fileStatus
fs
the fileStatus to be updated
shouldScan
whether this file should be scanned or not

Attributes
protected
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package search

class DataSkippingFileFilter extends Logging

Instance Constructors

new DataSkippingFileFilter(tid: String, metadataStoreManager: MetadataStoreManager, sparkSession: SparkSession, params: Map[String, String] = Map.empty[String, String])

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clearCurrentFilterStatistics(): Unit

def clone(): AnyRef

val currentFilterStatistics: HashMap[String, Long]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def handleStatistics(): Unit

def hashCode(): Int

def init(dataFilters: Seq[Expression], partitionFilters: Seq[Expression], metadataFilterFactories: Seq[MetadataFilterFactory], clauseTranslators: Seq[ClauseTranslator]): Unit

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

final def isInstanceOf[T0]: Boolean

def isRequired(fs: FileStatus): Boolean

def isSkipabble(): Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

lazy val metadataHandler: MetadataHandle

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

def updateStats(fs: FileStatus, shouldScan: Boolean): Unit

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Ungrouped