Spark使用OSS select优化数据查询（公测）

spark.sparkContext.hadoopConfiguration.set("fs.s3a.access.key", "")
spark.sparkContext.hadoopConfiguration.set("fs.s3a.secret.key", "")
spark.sparkContext.hadoopConfiguration.set("fs.s3a.endpoint", "")
spark.sparkContext.hadoopConfiguration.set("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
val schema = StructType(
 List(StructField("name", StringType, false),
 StructField("age", IntegerType, false)))

val df = spark.read.format("minioSelectCSV")//minioSelectJSON
  .schema(schema)
  .options(Map("quote" -> "\'", "header" -> "true", "delimiter" -> ","))
  .load("s3://bukcet/object")
df.select("*").show()
sql:

spark.sql("CREATE TEMPORARY VIEW MyView (age INT, name STRING) USING minioSelectCSV OPTIONS (path \"s3://bucket/object\")")
spark.sql("select * from MyView where age > 10").show()