Amazon Glue Scala 数据源特性

程序包：com.amazonaws.services.glue

用于生成 DynamicFrame 的高级接口。

trait DataSource {

  def getDynamicFrame : DynamicFrame 

  def getDynamicFrame( minPartitions : Int,
                       targetPartitions : Int
                     ) : DynamicFrame 
  def getDataFrame : DataFrame
					 
  /** @param num: the number of records for sampling.
    * @param options: optional parameters to control sampling behavior. Current available parameter for Amazon S3 sources in options:
    *  1. maxSamplePartitions: the maximum number of partitions the sampling will read. 
    *  2. maxSampleFilesPerPartition: the maximum number of files the sampling will read in one partition.
    */
  def getSampleDynamicFrame(num:Int, options: JsonOptions = JsonOptions.empty): DynamicFrame 

  def glueContext : GlueContext

  def setFormat( format : String,
                 options : String
               ) : Unit 

  def setFormat( format : String,
                 options : JsonOptions
               ) : Unit

  def supportsFormat( format : String ) : Boolean

  def withFormat( format : String,
                  options : JsonOptions = JsonOptions.empty
                ) : DataSource 
}