Utils

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

5. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

6. def createCustomIndex(idxClsName: String, cols: Seq[String], keyMetadata: Option[String], params: Map[String, String]): Index

   

7. final def eq(arg0: AnyRef): Boolean

   

   Definition Classes

   AnyRef

8. def equals(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

9. def finalize(): Unit

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( classOf[java.lang.Throwable] )

10. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

11. def getClassInstance[T](clsName: String): Option[T]

    

    Creates a scala class instance with empty constructor (using reflection)

    Creates a scala class instance with empty constructor (using reflection)

    clsName

    the full name of the object

    returns

    a class instance if succeeded , None otherwise

12. def getFileId(fs: FileStatus): String

    

    Given a FileStatus return the name associated with this path

13. def getName(field: Expression): String

    

    Returns the expression string (full column name), flattened, without any quoting or backticks added.

    Returns the expression string (full column name), flattened, without any quoting or backticks added. NOTE: because backticks are removed, these is a possibility of collision for example, expression that select columns "a.b" and "a.b will return the same name here!

    field

    the expression to get name for

14. def getObjectInstance[T](clsName: String): Option[T]

    

    Creates a scala object instance using the object name (using reflection)

    Creates a scala object instance using the object name (using reflection)

    clsName

    the full name of the object

    returns

    the object if succeeded , None otherwise

15. def getPartitionColumns(df: DataFrame): Set[String]

    

    Extracts the partition columns from a DataFrame

    Extracts the partition columns from a DataFrame

    df

    the DataFrame for which the partition columns are to be extracted

16. def getPathDisplayName(uri: String): String

    

    Custom logic to be used to rename the paths which are displayed in the xskipper output DataFrame-s

    Custom logic to be used to rename the paths which are displayed in the xskipper output DataFrame-s

    uri

    the path to be displayed

17. def getSchemaFields(field: StructField, prefix: String = ""): Seq[(String, (String, StructField))]

    

    Returns schema fields recursively

    Returns schema fields recursively

    field

    schema root field

    returns

    (field name in lower case, (field name, field))

    Exceptions thrown

    XskipperException if schema field name contains a dot

18. def getTable(spark: SparkSession, tableIdentifier: String): DataFrame

    

    Returns the DataFrame associated with the given table identifier

    Returns the DataFrame associated with the given table identifier

    spark

    SparkSession instance

    tableIdentifier

    the table identifier requested

    returns

    the DataFrame associated with the given table identifier

19. def getTableIdentifier(uri: String): String

    

    Given a URI/table identifier returns the table identifier that will be used in xskipper

    Given a URI/table identifier returns the table identifier that will be used in xskipper

    uri

    the uri to be parsed

    returns

    The corresponding Table identifier returns it as it is

20. def getTableIdentifierDisplayName(tid: String): String

    

    Custom logic to be used to rename the tid which are displayed in the xskipper output DataFrame-s

    Custom logic to be used to rename the tid which are displayed in the xskipper output DataFrame-s

    tid

    the path to be displayed

21. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

22. var identifier: Identifier

    

23. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

24. def initializeLogIfNecessary(isInterpreter: Boolean): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

25. def isColumnNameValid(colName: String): Boolean

    

    Determines whether or not the column name is valid for indexing.

    Determines whether or not the column name is valid for indexing. For now, a name is legal if it's a legal spark column name + no vertex along the column's schema tree path contains a "." for example, "a.b.c" is illegal (the path is [a.b, c]), but "a.b.c" ([a,b,c]) is legal.

    colName

    the column name to check

    returns

    true if the column name is valid, false otherwise

26. def isExpressionValidForSelection(expr: Expression): Boolean

    

    Checks if an expression is a valid single-column selection a valid single-column selection is an expression that selects a single column that is either flat (Attribute) or a nested column that is not a part of an array.

27. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

28. def isSchemaValid(schema: StructType): Boolean

    

    Checks if the given schema is valid The schema is considered valid if it doens't contain column names with .

    Checks if the given schema is valid The schema is considered valid if it doens't contain column names with .

    schema

    the schema to check

    returns

    true if the schema is valid, false otherwise

29. def isTraceEnabled(): Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

30. def log: Logger

    

    Attributes

    protected

    Definition Classes

    Logging

31. def logDebug(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

32. def logDebug(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

33. def logError(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

34. def logError(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

35. def logInfo(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

36. def logInfo(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

37. def logName: String

    

    Attributes

    protected

    Definition Classes

    Logging

38. def logTrace(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

39. def logTrace(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

40. def logWarning(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

41. def logWarning(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

42. def memoryStringToMb(str: String): Int

    

    Convert a Java memory parameter passed to -Xmx (such as 300m or 1g) to a number of mebibytes.

43. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

44. final def notify(): Unit

    

    Definition Classes

    AnyRef

45. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

46. def stringToPath(path: String): Path

    

    Receives a string and tries to convert it to org.apache.hadoop.fs.Path

    Receives a string and tries to convert it to org.apache.hadoop.fs.Path

    path

    the path to convert

    returns

    org.apache.hadoop.fs.Path if conversion succeeded or exception else

47. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

48. def toString(): String

    

    Definition Classes

    AnyRef → Any

49. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

50. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

51. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )