com.splicemachine.derby.stream.spark

Class BulkDataSetWriter

java.lang.Object

com.splicemachine.derby.stream.spark.BulkDataSetWriter

Direct Known Subclasses:

BulkDeleteDataSetWriter, BulkInsertDataSetWriter, BulkLoadIndexDataSetWriter

public class BulkDataSetWriter
extends java.lang.Object

Created by jyuan on 5/31/17.

Field Summary

Fields

Modifier and Type	Field and Description
protected DataSet	dataSet
protected long	heapConglom
protected static org.apache.log4j.Logger	LOG
protected OperationContext	operationContext
protected TxnView	txn

Constructor Summary

Constructors

Constructor and Description
BulkDataSetWriter()
BulkDataSetWriter(DataSet dataset, OperationContext operationContext, long heapConglom, TxnView txn)

Method Summary

Modifier and Type	Method and Description
protected void	bulkLoad(java.util.List<BulkImportPartition> bulkImportPartitions, java.lang.String bulkImportDirectory, java.lang.String prefix)
protected void	dumpCutPoints(java.util.List<scala.Tuple2<java.lang.Long,byte[][]>> cutPointsList, java.lang.String bulkImportDirectory) Output cut points to files
protected void	getAllConglomerates(java.util.List<java.lang.Long> allCongloms, java.util.ArrayList<com.splicemachine.ddl.DDLMessage.TentativeIndex> tentativeIndexList)
static java.util.List<BulkImportPartition>	getBulkImportPartitions(java.util.List<java.lang.Long> congloms, java.lang.String bulkImportDirectory) Get actual partition boundaries for each table and index
protected java.util.List<scala.Tuple2<java.lang.Long,byte[][]>>	getCutPoints(double sampleFraction, java.util.List<scala.Tuple2<java.lang.Long,scala.Tuple2<java.lang.Double,ColumnStatisticsImpl>>> statistics) Calculate cut points according to statistics.
static org.apache.hadoop.fs.Path	getRandomFilename(org.apache.hadoop.fs.Path dir) Generate a file name
protected java.util.Map<java.lang.Long,scala.Tuple2<java.lang.Double,ColumnStatisticsImpl>>	mergeResults(java.util.List<scala.Tuple2<java.lang.Long,scala.Tuple2<java.lang.Double,ColumnStatisticsImpl>>> tuples) Merge statistics from each RDD partition
protected void	partitionUsingRDDSortUsingDataFrame(java.util.List<BulkImportPartition> bulkImportPartitions, DataSet rowAndIndexes, HFileGenerationFunction hfileGenerationFunction)
protected void	splitTables(java.util.List<scala.Tuple2<java.lang.Long,byte[][]>> cutPointsList) Split a table using cut points

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

dataSet

protected DataSet dataSet

operationContext

protected OperationContext operationContext

heapConglom

protected long heapConglom

txn

protected TxnView txn

LOG

protected static final org.apache.log4j.Logger LOG

Constructor Detail

BulkDataSetWriter

public BulkDataSetWriter()

BulkDataSetWriter

public BulkDataSetWriter(DataSet dataset,
                         OperationContext operationContext,
                         long heapConglom,
                         TxnView txn)

Method Detail

getBulkImportPartitions

public static java.util.List<BulkImportPartition> getBulkImportPartitions(java.util.List<java.lang.Long> congloms,
                                                                          java.lang.String bulkImportDirectory)
                                                                   throws StandardException

Get actual partition boundaries for each table and index

Parameters:

congloms -

Returns:

Throws:

StandardException

getRandomFilename

public static org.apache.hadoop.fs.Path getRandomFilename(org.apache.hadoop.fs.Path dir)
                                                   throws java.io.IOException

Generate a file name

Parameters:

dir -

Returns:

Throws:

java.io.IOException

partitionUsingRDDSortUsingDataFrame

protected void partitionUsingRDDSortUsingDataFrame(java.util.List<BulkImportPartition> bulkImportPartitions,
                                                   DataSet rowAndIndexes,
                                                   HFileGenerationFunction hfileGenerationFunction)

bulkLoad

protected void bulkLoad(java.util.List<BulkImportPartition> bulkImportPartitions,
                        java.lang.String bulkImportDirectory,
                        java.lang.String prefix)
                 throws StandardException

Parameters:

bulkImportPartitions -

Throws:

StandardException

getAllConglomerates

protected void getAllConglomerates(java.util.List<java.lang.Long> allCongloms,
                                   java.util.ArrayList<com.splicemachine.ddl.DDLMessage.TentativeIndex> tentativeIndexList)
                            throws StandardException

Throws:

StandardException

getCutPoints

protected java.util.List<scala.Tuple2<java.lang.Long,byte[][]>> getCutPoints(double sampleFraction,
                                                                             java.util.List<scala.Tuple2<java.lang.Long,scala.Tuple2<java.lang.Double,ColumnStatisticsImpl>>> statistics)
                                                                      throws StandardException

Calculate cut points according to statistics. Number of cut points is decided by max region size.

Parameters:

statistics -

Returns:

Throws:

StandardException

splitTables

protected void splitTables(java.util.List<scala.Tuple2<java.lang.Long,byte[][]>> cutPointsList)
                    throws StandardException

Split a table using cut points

Parameters:

cutPointsList -

Throws:

StandardException

dumpCutPoints

protected void dumpCutPoints(java.util.List<scala.Tuple2<java.lang.Long,byte[][]>> cutPointsList,
                             java.lang.String bulkImportDirectory)
                      throws StandardException

Output cut points to files

Parameters:

cutPointsList -

Throws:

java.io.IOException

StandardException

mergeResults

protected java.util.Map<java.lang.Long,scala.Tuple2<java.lang.Double,ColumnStatisticsImpl>> mergeResults(java.util.List<scala.Tuple2<java.lang.Long,scala.Tuple2<java.lang.Double,ColumnStatisticsImpl>>> tuples)
                                                                                                  throws StandardException

Merge statistics from each RDD partition

Parameters:

tuples -

Returns:

Throws:

StandardException