Advertisement
NLinker

Spark FunctionResourceType

Dec 5th, 2017
185
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Scala 1.53 KB | None | 0 0
  1. package org.apache.spark.sql.catalyst.catalog
  2.  
  3. import org.apache.spark.sql.AnalysisException
  4.  
  5. /** A trait that represents the type of a resourced needed by a function. */
  6. abstract class FunctionResourceType(val resourceType: String)
  7.  
  8. object JarResource extends FunctionResourceType("jar")
  9.  
  10. object FileResource extends FunctionResourceType("file")
  11.  
  12. // We do not allow users to specify an archive because it is YARN specific.
  13. // When loading resources, we will throw an exception and ask users to
  14. // use --archive with spark submit.
  15. object ArchiveResource extends FunctionResourceType("archive")
  16.  
  17. object FunctionResourceType {
  18.   def fromString(resourceType: String): FunctionResourceType = {
  19.     resourceType.toLowerCase match {
  20.       case "jar" => JarResource
  21.       case "file" => FileResource
  22.       case "archive" => ArchiveResource
  23.       case other =>
  24.         throw new AnalysisException(s"Resource Type '$resourceType' is not supported.")
  25.     }
  26.   }
  27. }
  28.  
  29. case class FunctionResource(resourceType: FunctionResourceType, uri: String)
  30.  
  31. /**
  32.  * A simple trait representing a class that can be used to load resources used by
  33.  * a function. Because only a SQLContext can load resources, we create this trait
  34.  * to avoid of explicitly passing SQLContext around.
  35.  */
  36. trait FunctionResourceLoader {
  37.   def loadResource(resource: FunctionResource): Unit
  38. }
  39.  
  40. object DummyFunctionResourceLoader extends FunctionResourceLoader {
  41.   override def loadResource(resource: FunctionResource): Unit = {
  42.     throw new UnsupportedOperationException
  43.   }
  44. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement