MetadataProcessor

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. def analyzeAndUploadMetadata(format: String, options: Map[String, String], indexes: Seq[Index], fileIds: Seq[(String, String)], schema: Option[StructType], isRefresh: Boolean = false): Unit

   

   Collects the indexes and them to the metadatastore using the MetadataHandle

   Collects the indexes and them to the metadatastore using the MetadataHandle

   format

   the format to be used when reading each object

   options

   the options to be used when reading each object Note: all objects are assumed to have the same options and format.

   indexes

   a sequence of indexes that will be applied on the indexed dataset

   fileIds

   a sequence of (String, String) where the first string is the file name and the second is the fileID

   schema

   (optional) the expected schema (since we are reading object by object the schema can be provided according to the full dataframe)

   isRefresh

   indicates whether the operation is a refresh

5. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

6. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

7. def collectNewFiles(files: Seq[FileStatus], isRefresh: Boolean): (Seq[(String, String)], Seq[String])

   

   Collects the list of files that needs to be indexed A file needs to be indexed if: 1.

   Collects the list of files that needs to be indexed A file needs to be indexed if: 1. It is a new file that was not indexed before 2. It is an indexed file which changed since it was indexed

   files

   The list of files the will be compared against the existing indexed files

   isRefresh

   indicates whether this is a refresh operation or not, in case this is not a refresh operation assuming no indexed files exits

   returns

   Sequence of (String, String) where the first string is the file name and the second is the fileID for all of new/modified files, Sequence of files to be removed from the metadatastore (since they were updated)

8. final def eq(arg0: AnyRef): Boolean

   

   Definition Classes

   AnyRef

9. def equals(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

10. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

11. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

12. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

13. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

14. def initializeLogIfNecessary(isInterpreter: Boolean): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

15. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

16. def isTraceEnabled(): Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

17. def log: Logger

    

    Attributes

    protected

    Definition Classes

    Logging

18. def logDebug(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

19. def logDebug(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

20. def logError(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

21. def logError(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

22. def logInfo(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

23. def logInfo(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

24. def logName: String

    

    Attributes

    protected

    Definition Classes

    Logging

25. def logTrace(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

26. def logTrace(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

27. def logWarning(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

28. def logWarning(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

29. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

30. final def notify(): Unit

    

    Definition Classes

    AnyRef

31. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

32. def prepareForRefresh(indexes: Seq[Index]): Unit

    

33. def removeMetadataForFiles(files: Seq[String]): Int

    

    Removes the metadata for a given list of files

    Removes the metadata for a given list of files

    files

    the list of files to remove metadata for

34. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

35. val tableIdentifier: String

    

36. def toString(): String

    

    Definition Classes

    AnyRef → Any

37. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

38. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

39. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )