Amazon Glue Scala DataSource trait
Package: com.amazonaws.services.glue
A high-level interface for producing a DynamicFrame
.
trait DataSource {
def getDynamicFrame : DynamicFrame
def getDynamicFrame( minPartitions : Int,
targetPartitions : Int
) : DynamicFrame
def getDataFrame : DataFrame
/** @param num: the number of records for sampling.
* @param options: optional parameters to control sampling behavior. Current available parameter for Amazon S3 sources in options:
* 1. maxSamplePartitions: the maximum number of partitions the sampling will read.
* 2. maxSampleFilesPerPartition: the maximum number of files the sampling will read in one partition.
*/
def getSampleDynamicFrame(num:Int, options: JsonOptions = JsonOptions.empty): DynamicFrame
def glueContext : GlueContext
def setFormat( format : String,
options : String
) : Unit
def setFormat( format : String,
options : JsonOptions
) : Unit
def supportsFormat( format : String ) : Boolean
def withFormat( format : String,
options : JsonOptions = JsonOptions.empty
) : DataSource
}