package parquet
Type Members
- class ParquetDataSourceV2 extends FileDataSourceV2
- case class ParquetPartitionReaderFactory(sqlConf: SQLConf, broadcastedConf: Broadcast[SerializableConfiguration], dataSchema: StructType, readDataSchema: StructType, partitionSchema: StructType, filters: Array[Filter], parquetOptions: ParquetOptions) extends FilePartitionReaderFactory with Logging with Product with Serializable
A factory used to create Parquet readers.
A factory used to create Parquet readers.
- sqlConf
SQL configuration.
- broadcastedConf
Broadcast serializable Hadoop Configuration.
- dataSchema
Schema of Parquet files.
- readDataSchema
Required schema of Parquet files.
- partitionSchema
Schema of partitions.
- filters
Filters to be pushed down in the batch scan.
- parquetOptions
The options of Parquet datasource that are set for the read.
- case class ParquetScan(sparkSession: SparkSession, hadoopConf: Configuration, fileIndex: PartitioningAwareFileIndex, dataSchema: StructType, readDataSchema: StructType, readPartitionSchema: StructType, pushedFilters: Array[Filter], options: CaseInsensitiveStringMap, partitionFilters: Seq[Expression] = Seq.empty, dataFilters: Seq[Expression] = Seq.empty) extends FileScan with Product with Serializable
- case class ParquetScanBuilder(sparkSession: SparkSession, fileIndex: PartitioningAwareFileIndex, schema: StructType, dataSchema: StructType, options: CaseInsensitiveStringMap) extends FileScanBuilder with SupportsPushDownFilters with Product with Serializable
- case class ParquetTable(name: String, sparkSession: SparkSession, options: CaseInsensitiveStringMap, paths: Seq[String], userSpecifiedSchema: Option[StructType], fallbackFileFormat: Class[_ <: FileFormat]) extends FileTable with Product with Serializable
- case class ParquetWrite(paths: Seq[String], formatName: String, supportsDataType: (DataType) => Boolean, info: LogicalWriteInfo) extends FileWrite with Logging with Product with Serializable