Class PslWrite (0.4.5) 

public class PslWrite implements WriteBuilder, SupportsStreamingUpdateAsAppend, BatchWrite, StreamingWrite

Pub/Sub Lite class for writing.

Note that SupportsStreamingUpdateAsAppend is the same hack that kafka uses to opt-in to writing aggregates without requiring windowing.

Inheritance

java.lang.Object > PslWrite

Implements

org.apache.spark.sql.connector.write.WriteBuilder, org.apache.spark.sql.internal.connector.SupportsStreamingUpdateAsAppend, org.apache.spark.sql.connector.write.BatchWrite, org.apache.spark.sql.connector.write.streaming.StreamingWrite

Constructors

PslWrite(StructType inputSchema, PslWriteDataSourceOptions writeOptions)

public PslWrite(StructType inputSchema, PslWriteDataSourceOptions writeOptions)
Parameters
Name Description
inputSchema org.apache.spark.sql.types.StructType
writeOptions PslWriteDataSourceOptions

Methods

abort(long epochId, WriterCommitMessage[] messages)

public void abort(long epochId, WriterCommitMessage[] messages)
Parameters
Name Description
epochId long
messages org.apache.spark.sql.connector.write.WriterCommitMessage[]

abort(WriterCommitMessage[] messages)

public void abort(WriterCommitMessage[] messages)
Parameter
Name Description
messages org.apache.spark.sql.connector.write.WriterCommitMessage[]

buildForBatch()

public BatchWrite buildForBatch()
Returns
Type Description
org.apache.spark.sql.connector.write.BatchWrite

buildForStreaming()

public StreamingWrite buildForStreaming()
Returns
Type Description
org.apache.spark.sql.connector.write.streaming.StreamingWrite

commit(long epochId, WriterCommitMessage[] messages)

public void commit(long epochId, WriterCommitMessage[] messages)
Parameters
Name Description
epochId long
messages org.apache.spark.sql.connector.write.WriterCommitMessage[]

commit(WriterCommitMessage[] messages)

public void commit(WriterCommitMessage[] messages)
Parameter
Name Description
messages org.apache.spark.sql.connector.write.WriterCommitMessage[]

createBatchWriterFactory(PhysicalWriteInfo info)

public DataWriterFactory createBatchWriterFactory(PhysicalWriteInfo info)
Parameter
Name Description
info org.apache.spark.sql.connector.write.PhysicalWriteInfo
Returns
Type Description
org.apache.spark.sql.connector.write.DataWriterFactory

createStreamingWriterFactory(PhysicalWriteInfo info)

public StreamingDataWriterFactory createStreamingWriterFactory(PhysicalWriteInfo info)
Parameter
Name Description
info org.apache.spark.sql.connector.write.PhysicalWriteInfo
Returns
Type Description
org.apache.spark.sql.connector.write.streaming.StreamingDataWriterFactory