ParquetUtils

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

5. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

6. final def eq(arg0: AnyRef): Boolean

   

   Definition Classes

   AnyRef

7. def equals(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

8. def finalize(): Unit

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( classOf[java.lang.Throwable] )

9. final def getClass(): Class[_]

   

   Definition Classes

   AnyRef → Any

10. def getColumnName(idx: Index, version: Long = ...): String

    

    returns the column name for the specified index and version

    returns the column name for the specified index and version

    idx

    the index for which the column name needs to created

    version

    version number, the metadata spec of which will determine the column name

11. def getColumnNameForCols(cols: Seq[String], idxName: String, version: Long = ...): String

    

12. def getIndexSchema(index: Index, translators: Seq[ParquetMetaDataTranslator]): Option[DataType]

    

    Given an index and schema translator tries searching for the first available translation.

    Given an index and schema translator tries searching for the first available translation. to a native DataFrame schema. if no translation is found return None

    index

    the index to translate

    translators

    the list of available translators

    returns

    the DataType associated with the translation

13. def getMdVersionStatus(version: Long): MetadataVersionStatus

    

14. def getMdVersionStatusFromDf(df: DataFrame): MetadataVersionStatus

    

15. def getVersion(schema: StructType): Long

    

    retrieves the version number from a metadata DataFrame Schema, returns 0 if the version is not explicitly defined (files without version number are implicitly declared version 0).

    retrieves the version number from a metadata DataFrame Schema, returns 0 if the version is not explicitly defined (files without version number are implicitly declared version 0). the function assumes the obj_name column exists in the schema.

    schema

    - the schema of the metadata df

16. def getVersion(df: DataFrame): Long

    

    retrieves the version number from a Metadata DataFrame, returns 0 if the version is not explicitly defined (files without version number are implicitly declared version 0).

    retrieves the version number from a Metadata DataFrame, returns 0 if the version is not explicitly defined (files without version number are implicitly declared version 0). the function assumes the obj_name column exists in the schema.

17. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

18. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

19. def initializeLogIfNecessary(isInterpreter: Boolean): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

20. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

21. def isPmeAvailable(): Boolean

    

    checks if Parquet Modular Encryption (PME) is available the check is performed by verifying that org.apache.parquet.crypto.AesCipher is available (will be available if and only if PME is loaded)

    checks if Parquet Modular Encryption (PME) is available the check is performed by verifying that org.apache.parquet.crypto.AesCipher is available (will be available if and only if PME is loaded)

    returns

    true if PME is loaded, else false

22. def isTraceEnabled(): Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

23. def log: Logger

    

    Attributes

    protected

    Definition Classes

    Logging

24. def logDebug(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

25. def logDebug(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

26. def logError(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

27. def logError(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

28. def logInfo(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

29. def logInfo(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

30. def logName: String

    

    Attributes

    protected

    Definition Classes

    Logging

31. def logTrace(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

32. def logTrace(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

33. def logWarning(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

34. def logWarning(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

35. def mdFileToDF(session: SparkSession, mdPath: String): DataFrame

    

36. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

37. final def notify(): Unit

    

    Definition Classes

    AnyRef

38. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

39. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

40. def toString(): String

    

    Definition Classes

    AnyRef → Any

41. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

42. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

43. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )