org.apache.hadoop.hive.ql.plan

Class FileSinkDesc

java.lang.Object

org.apache.hadoop.hive.ql.plan.AbstractOperatorDesc

org.apache.hadoop.hive.ql.plan.FileSinkDesc

All Implemented Interfaces:

Serializable, Cloneable, IStatsGatherDesc, OperatorDesc

public class FileSinkDesc
extends AbstractOperatorDesc
implements IStatsGatherDesc

FileSinkDesc.

See Also:

Serialized Form

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	FileSinkDesc.DPSortState
class	FileSinkDesc.FileSinkOperatorExplainVectorization

Field Summary

Fields inherited from class org.apache.hadoop.hive.ql.plan.AbstractOperatorDesc

colExprMap, memAvailable, memNeeded, opProps, opTraits, runtimeStatsTmpDir, statistics, vectorDesc, vectorMode

Constructor Summary

Constructors

Constructor and Description
FileSinkDesc()
FileSinkDesc(org.apache.hadoop.fs.Path dirName, TableDesc tableInfo, boolean compressed)
FileSinkDesc(org.apache.hadoop.fs.Path dirName, TableDesc tableInfo, boolean compressed, int destTableId, boolean multiFileSpray, boolean canBeMerged, int numFiles, int totalFiles, ArrayList<ExprNodeDesc> partitionCols, DynamicPartitionCtx dpCtx, org.apache.hadoop.fs.Path destPath, Long mmWriteId, boolean isMmCtas, boolean isInsertOverwrite)

Method Summary

Modifier and Type	Method and Description
boolean	canBeMerged()
Object	clone()
String	getCompressCodec()
boolean	getCompressed()
String	getCompressType()
org.apache.hadoop.fs.Path	getDestPath()
int	getDestTableId()
org.apache.hadoop.fs.Path	getDirName()
String	getDirNameString()
FileSinkDesc.DPSortState	getDpSortState()
String	getDpSortStateString()
DynamicPartitionCtx	getDynPartCtx()
FileSinkDesc.FileSinkOperatorExplainVectorization	getFileSinkVectorization()
org.apache.hadoop.fs.Path	getFinalDirName()
boolean	getInsertOverwrite()
ListBucketingCtx	getLbCtx()
List<FileSinkDesc>	getLinkedFileSinkDesc()
org.apache.hadoop.fs.Path	getMergeInputDirName() getFinalDirName that takes into account MM, but not DP, LB or buckets.
int	getNumFiles()
org.apache.hadoop.fs.Path	getParentDir()
ArrayList<ExprNodeDesc>	getPartitionCols()
int	getStatementId() See AcidOutputFormat.Options.statementId(int)
String	getStaticSpec()
String	getStatsAggPrefix() Construct the key prefix used as (intermediate) statistics publishing and aggregation.
Table	getTable()
TableDesc	getTableInfo()
long	getTableWriteId()
String	getTmpStatsDir()
int	getTotalFiles()
AcidUtils.Operation	getWriteType()
String	getWriteTypeString()
boolean	isFullAcidTable()
boolean	isGatherStats()
boolean	isHiveServerQuery()
boolean	isLinkedFileSink()
boolean	isMaterialization()
boolean	isMerge()
boolean	isMmCtas()
boolean	isMmTable()
boolean	isMultiFileSpray()
boolean	isRemovedReduceSinkBucketSort()
boolean	isSame(OperatorDesc other) The default implementation delegates to Object.equals(Object).
boolean	isStatsReliable()
boolean	isTemporary()
boolean	isUsingBatchingSerDe()
void	setCanBeMerged(boolean canBeMerged)
void	setCompressCodec(String intermediateCompressorCodec)
void	setCompressed(boolean compressed)
void	setCompressType(String intermediateCompressType)
void	setDestTableId(int destTableId)
void	setDirName(org.apache.hadoop.fs.Path dirName)
void	setDpSortState(FileSinkDesc.DPSortState dpSortState)
void	setDynPartCtx(DynamicPartitionCtx dpc)
void	setGatherStats(boolean gatherStats)
void	setHiveServerQuery(boolean isHiveServerQuery)
void	setInsertOverwrite(boolean isInsertOverwrite)
void	setIsMerge(boolean b)
void	setIsUsingBatchingSerDe(boolean isUsingBatchingSerDe)
void	setLbCtx(ListBucketingCtx lbCtx)
void	setLinkedFileSink(boolean linkedFileSink)
void	setLinkedFileSinkDesc(List<FileSinkDesc> linkedFileSinkDesc)
void	setMaterialization(boolean materialization)
void	setMmWriteId(Long mmWriteId)
void	setMultiFileSpray(boolean multiFileSpray)
void	setNumFiles(int numFiles)
void	setPartitionCols(ArrayList<ExprNodeDesc> partitionCols)
void	setRemovedReduceSinkBucketSort(boolean removedReduceSinkBucketSort)
void	setStatementId(int id)
void	setStaticSpec(String staticSpec)
void	setStatsAggPrefix(String k) Set the stats aggregation key.
void	setStatsReliable(boolean statsReliable)
void	setStatsTmpDir(String statsCollectionTempDir)
void	setTable(Table table)
void	setTableInfo(TableDesc tableInfo)
void	setTableWriteId(long id)
void	setTemporary(boolean temporary)
void	setTotalFiles(int totalFiles)
void	setWriteType(AcidUtils.Operation type)

Methods inherited from class org.apache.hadoop.hive.ql.plan.AbstractOperatorDesc

fillSignature, getColumnExprMap, getColumnExprMapForExplain, getMaxMemoryAvailable, getMemoryNeeded, getOpProps, getRuntimeStatsTmpDir, getStatistics, getTraits, getUserLevelStatistics, getVectorDesc, getVectorMode, setColumnExprMap, setMaxMemoryAvailable, setMemoryNeeded, setOpProps, setRuntimeStatsTmpDir, setStatistics, setTraits, setVectorDesc, setVectorMode

Methods inherited from class java.lang.Object

equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

FileSinkDesc

public FileSinkDesc()

FileSinkDesc

public FileSinkDesc(org.apache.hadoop.fs.Path dirName,
                    TableDesc tableInfo,
                    boolean compressed,
                    int destTableId,
                    boolean multiFileSpray,
                    boolean canBeMerged,
                    int numFiles,
                    int totalFiles,
                    ArrayList<ExprNodeDesc> partitionCols,
                    DynamicPartitionCtx dpCtx,
                    org.apache.hadoop.fs.Path destPath,
                    Long mmWriteId,
                    boolean isMmCtas,
                    boolean isInsertOverwrite)

Parameters:

destPath - - the final destination for data

FileSinkDesc

public FileSinkDesc(org.apache.hadoop.fs.Path dirName,
                    TableDesc tableInfo,
                    boolean compressed)

Method Detail

clone

public Object clone()
             throws CloneNotSupportedException

Specified by:

clone in interface OperatorDesc

Overrides:

clone in class AbstractOperatorDesc

Throws:

CloneNotSupportedException

isHiveServerQuery

public boolean isHiveServerQuery()

setHiveServerQuery

public void setHiveServerQuery(boolean isHiveServerQuery)

isUsingBatchingSerDe

public boolean isUsingBatchingSerDe()

setIsUsingBatchingSerDe

public void setIsUsingBatchingSerDe(boolean isUsingBatchingSerDe)

getDirName

public org.apache.hadoop.fs.Path getDirName()

getDirNameString

public String getDirNameString()

setDirName

public void setDirName(org.apache.hadoop.fs.Path dirName)

getFinalDirName

public org.apache.hadoop.fs.Path getFinalDirName()

getMergeInputDirName

public org.apache.hadoop.fs.Path getMergeInputDirName()

getFinalDirName that takes into account MM, but not DP, LB or buckets.

getTableInfo

public TableDesc getTableInfo()

setTableInfo

public void setTableInfo(TableDesc tableInfo)

getCompressed

public boolean getCompressed()

setCompressed

public void setCompressed(boolean compressed)

getDestTableId

public int getDestTableId()

setDestTableId

public void setDestTableId(int destTableId)

getCompressCodec

public String getCompressCodec()

setCompressCodec

public void setCompressCodec(String intermediateCompressorCodec)

getCompressType

public String getCompressType()

setCompressType

public void setCompressType(String intermediateCompressType)

isMultiFileSpray

public boolean isMultiFileSpray()

Returns:

the multiFileSpray

setMultiFileSpray

public void setMultiFileSpray(boolean multiFileSpray)

Parameters:

multiFileSpray - the multiFileSpray to set

isTemporary

public boolean isTemporary()

Returns:

destination is temporary

setTemporary

public void setTemporary(boolean temporary)

isMmTable

public boolean isMmTable()

isFullAcidTable

public boolean isFullAcidTable()

isMaterialization

public boolean isMaterialization()

setMaterialization

public void setMaterialization(boolean materialization)

canBeMerged

public boolean canBeMerged()

setCanBeMerged

public void setCanBeMerged(boolean canBeMerged)

getTotalFiles

public int getTotalFiles()

Returns:

the totalFiles

setTotalFiles

public void setTotalFiles(int totalFiles)

Parameters:

totalFiles - the totalFiles to set

getPartitionCols

public ArrayList<ExprNodeDesc> getPartitionCols()

Returns:

the partitionCols

setPartitionCols

public void setPartitionCols(ArrayList<ExprNodeDesc> partitionCols)

Parameters:

partitionCols - the partitionCols to set

getNumFiles

public int getNumFiles()

Returns:

the numFiles

setNumFiles

public void setNumFiles(int numFiles)

Parameters:

numFiles - the numFiles to set

setDynPartCtx

public void setDynPartCtx(DynamicPartitionCtx dpc)

getDynPartCtx

public DynamicPartitionCtx getDynPartCtx()

setStaticSpec

public void setStaticSpec(String staticSpec)

getStaticSpec

public String getStaticSpec()

setGatherStats

public void setGatherStats(boolean gatherStats)

isGatherStats

public boolean isGatherStats()

Specified by:

isGatherStats in interface IStatsGatherDesc

getStatsAggPrefix

public String getStatsAggPrefix()

Construct the key prefix used as (intermediate) statistics publishing and aggregation. During stats publishing phase, this key prefix will be appended with the optional dynamic partition spec and the task ID. The whole key uniquely identifies the output of a task for this job. In the stats aggregation phase, all rows with the same prefix plus dynamic partition specs (obtained at run-time after MR job finishes) will be serving as the prefix: all rows with the same prefix (output of all tasks for this job) will be aggregated.

Specified by:

getStatsAggPrefix in interface IStatsGatherDesc

Returns:

key prefix used for stats publishing and aggregation.

setStatsAggPrefix

public void setStatsAggPrefix(String k)

Set the stats aggregation key. If the input string is not terminated by Path.SEPARATOR aggregation key will add one to make it as a directory name.

Parameters:

k - input directory name.

isLinkedFileSink

public boolean isLinkedFileSink()

setLinkedFileSink

public void setLinkedFileSink(boolean linkedFileSink)

getParentDir

public org.apache.hadoop.fs.Path getParentDir()

isStatsReliable

public boolean isStatsReliable()

setStatsReliable

public void setStatsReliable(boolean statsReliable)

getLbCtx

public ListBucketingCtx getLbCtx()

Returns:

the lbCtx

setLbCtx

public void setLbCtx(ListBucketingCtx lbCtx)

Parameters:

lbCtx - the lbCtx to set

getLinkedFileSinkDesc

public List<FileSinkDesc> getLinkedFileSinkDesc()

setLinkedFileSinkDesc

public void setLinkedFileSinkDesc(List<FileSinkDesc> linkedFileSinkDesc)

isRemovedReduceSinkBucketSort

public boolean isRemovedReduceSinkBucketSort()

setRemovedReduceSinkBucketSort

public void setRemovedReduceSinkBucketSort(boolean removedReduceSinkBucketSort)

getDpSortState

public FileSinkDesc.DPSortState getDpSortState()

getDpSortStateString

public String getDpSortStateString()

setDpSortState

public void setDpSortState(FileSinkDesc.DPSortState dpSortState)

setWriteType

public void setWriteType(AcidUtils.Operation type)

getWriteType

public AcidUtils.Operation getWriteType()

getWriteTypeString

public String getWriteTypeString()

setTableWriteId

public void setTableWriteId(long id)

getTableWriteId

public long getTableWriteId()

setStatementId

public void setStatementId(int id)

getStatementId

public int getStatementId()

See AcidOutputFormat.Options.statementId(int)

getDestPath

public org.apache.hadoop.fs.Path getDestPath()

getTable

public Table getTable()

setTable

public void setTable(Table table)

getTmpStatsDir

public String getTmpStatsDir()

Specified by:

getTmpStatsDir in interface IStatsGatherDesc

setStatsTmpDir

public void setStatsTmpDir(String statsCollectionTempDir)

setMmWriteId

public void setMmWriteId(Long mmWriteId)

setIsMerge

public void setIsMerge(boolean b)

isMerge

public boolean isMerge()

isMmCtas

public boolean isMmCtas()

getFileSinkVectorization

public FileSinkDesc.FileSinkOperatorExplainVectorization getFileSinkVectorization()

setInsertOverwrite

public void setInsertOverwrite(boolean isInsertOverwrite)

getInsertOverwrite

public boolean getInsertOverwrite()

isSame

public boolean isSame(OperatorDesc other)

Description copied from class: AbstractOperatorDesc

The default implementation delegates to Object.equals(Object). Intended to be overridden by sub classes.

Specified by:

isSame in interface OperatorDesc

Overrides:

isSame in class AbstractOperatorDesc