org.apache.hive.hcatalog.common

Class HCatUtil

java.lang.Object

org.apache.hive.hcatalog.common.HCatUtil

public class HCatUtil
extends Object

Constructor Summary

Constructors

Constructor and Description
HCatUtil()

Method Summary

Modifier and Type	Method and Description
static void	assertNotNull(Object t, String msg, org.slf4j.Logger logger)
static boolean	checkJobContextIfRunningFromBackend(org.apache.hadoop.mapreduce.JobContext j)
static void	closeHiveClientQuietly(IMetaStoreClient client)
static void	configureOutputStorageHandler(HiveStorageHandler storageHandler, org.apache.hadoop.conf.Configuration conf, OutputJobInfo outputJobInfo)
static void	copyConf(org.apache.hadoop.conf.Configuration src, org.apache.hadoop.conf.Configuration dest) Replace the contents of dest with the contents of src
static void	copyJobPropertiesToJobConf(Map<String,String> jobProperties, org.apache.hadoop.mapred.JobConf jobConf)
static byte[]	decodeBytes(String str)
static Object	deserialize(String str)
static String	encodeBytes(byte[] bytes)
static HCatSchema	extractSchema(Partition partition)
static HCatSchema	extractSchema(Table table)
static org.apache.hadoop.security.token.Token<? extends org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier>	extractThriftToken(String tokenStrForm, String tokenSignature)
static Pair<String,String>	getDbAndTableName(String tableName)
static List<org.apache.hadoop.hive.metastore.api.FieldSchema>	getFieldSchemaList(List<HCatFieldSchema> hcatFields)
static List<HCatFieldSchema>	getHCatFieldSchemaList(org.apache.hadoop.hive.metastore.api.FieldSchema... fields)
static List<HCatFieldSchema>	getHCatFieldSchemaList(List<org.apache.hadoop.hive.metastore.api.FieldSchema> fields)
static Map<String,String>	getHCatKeyHiveConf(org.apache.hadoop.mapred.JobConf conf)
static HiveMetaStoreClient	getHiveClient(HiveConf hiveConf) Deprecated.
static HiveConf	getHiveConf(org.apache.hadoop.conf.Configuration conf)
static IMetaStoreClient	getHiveMetastoreClient(HiveConf hiveConf) Get or create a hive client depending on whether it exits in cache or not
static Map<String,String>	getInputJobProperties(HiveStorageHandler storageHandler, InputJobInfo inputJobInfo)
static org.apache.hadoop.mapred.JobConf	getJobConfFromContext(org.apache.hadoop.mapreduce.JobContext jobContext)
static org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier>	getJobTrackerDelegationToken(org.apache.hadoop.conf.Configuration conf, String userName)
static HCatSchema	getPartitionColumns(Table table) return the partition columns from a table instance
static HiveStorageHandler	getStorageHandler(org.apache.hadoop.conf.Configuration conf, PartInfo partitionInfo)
static HiveStorageHandler	getStorageHandler(org.apache.hadoop.conf.Configuration conf, StorerInfo storerInfo) Create an instance of a storage handler defined in storerInfo.
static HiveStorageHandler	getStorageHandler(org.apache.hadoop.conf.Configuration conf, String storageHandler, String serDe, String inputFormat, String outputFormat) Create an instance of a storage handler.
static Table	getTable(IMetaStoreClient client, String dbName, String tableName)
static HCatSchema	getTableSchemaWithPtnCols(Table table)
static boolean	isHadoop23()
static String	makePathASafeFileName(String filePath) Used by various tests to make sure the path is safe for Windows
static String	serialize(Serializable obj)
static HiveConf	storePropertiesToHiveConf(Properties properties, HiveConf hiveConf)
static boolean	validateExecuteBitPresentIfReadOrWrite(org.apache.hadoop.fs.permission.FsAction perms) Ensure that read or write permissions are not granted without also granting execute permissions.
static boolean	validateMorePermissive(org.apache.hadoop.fs.permission.FsAction first, org.apache.hadoop.fs.permission.FsAction second) Test if the first FsAction is more permissive than the second.
static List<org.apache.hadoop.hive.metastore.api.FieldSchema>	validatePartitionSchema(Table table, HCatSchema partitionSchema) Validate partition schema, checks if the column types match between the partition and the existing table schema.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

HCatUtil

public HCatUtil()

Method Detail

checkJobContextIfRunningFromBackend

public static boolean checkJobContextIfRunningFromBackend(org.apache.hadoop.mapreduce.JobContext j)

serialize

public static String serialize(Serializable obj)
                        throws IOException

Throws:

IOException

deserialize

public static Object deserialize(String str)
                          throws IOException

Throws:

IOException

encodeBytes

public static String encodeBytes(byte[] bytes)

decodeBytes

public static byte[] decodeBytes(String str)

getHCatFieldSchemaList

public static List<HCatFieldSchema> getHCatFieldSchemaList(org.apache.hadoop.hive.metastore.api.FieldSchema... fields)
                                                    throws HCatException

Throws:

HCatException

getHCatFieldSchemaList

public static List<HCatFieldSchema> getHCatFieldSchemaList(List<org.apache.hadoop.hive.metastore.api.FieldSchema> fields)
                                                    throws HCatException

Throws:

HCatException

extractSchema

public static HCatSchema extractSchema(Table table)
                                throws HCatException

Throws:

HCatException

extractSchema

public static HCatSchema extractSchema(Partition partition)
                                throws HCatException

Throws:

HCatException

getFieldSchemaList

public static List<org.apache.hadoop.hive.metastore.api.FieldSchema> getFieldSchemaList(List<HCatFieldSchema> hcatFields)

getTable

public static Table getTable(IMetaStoreClient client,
                             String dbName,
                             String tableName)
                      throws org.apache.hadoop.hive.metastore.api.NoSuchObjectException,
                             org.apache.thrift.TException,
                             org.apache.hadoop.hive.metastore.api.MetaException

Throws:

org.apache.hadoop.hive.metastore.api.NoSuchObjectException

org.apache.thrift.TException

org.apache.hadoop.hive.metastore.api.MetaException

getTableSchemaWithPtnCols

public static HCatSchema getTableSchemaWithPtnCols(Table table)
                                            throws IOException

Throws:

IOException

getPartitionColumns

public static HCatSchema getPartitionColumns(Table table)
                                      throws IOException

return the partition columns from a table instance

Parameters:

table - the instance to extract partition columns from

Returns:

HCatSchema instance which contains the partition columns

Throws:

IOException

validatePartitionSchema

public static List<org.apache.hadoop.hive.metastore.api.FieldSchema> validatePartitionSchema(Table table,
                                                                                             HCatSchema partitionSchema)
                                                                                      throws IOException

Validate partition schema, checks if the column types match between the partition and the existing table schema. Returns the list of columns present in the partition but not in the table.

Parameters:

table - the table

partitionSchema - the partition schema

Returns:

the list of newly added fields

Throws:

IOException - Signals that an I/O exception has occurred.

validateMorePermissive

public static boolean validateMorePermissive(org.apache.hadoop.fs.permission.FsAction first,
                                             org.apache.hadoop.fs.permission.FsAction second)

Test if the first FsAction is more permissive than the second. This is useful in cases where we want to ensure that a file owner has more permissions than the group they belong to, for eg. More completely(but potentially more cryptically) owner-r >= group-r >= world-r : bitwise and-masked with 0444 => 444 >= 440 >= 400 >= 000 owner-w >= group-w >= world-w : bitwise and-masked with &0222 => 222 >= 220 >= 200 >= 000 owner-x >= group-x >= world-x : bitwise and-masked with &0111 => 111 >= 110 >= 100 >= 000

Returns:

true if first FsAction is more permissive than the second, false if not.

validateExecuteBitPresentIfReadOrWrite

public static boolean validateExecuteBitPresentIfReadOrWrite(org.apache.hadoop.fs.permission.FsAction perms)

Ensure that read or write permissions are not granted without also granting execute permissions. Essentially, r-- , rw- and -w- are invalid, r-x, -wx, rwx, ---, --x are valid

Parameters:

perms - The FsAction to verify

Returns:

true if the presence of read or write permission is accompanied by execute permissions

getJobTrackerDelegationToken

public static org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier> getJobTrackerDelegationToken(org.apache.hadoop.conf.Configuration conf,
                                                                                                                                                                   String userName)
                                                                                                                                                            throws Exception

Throws:

Exception

extractThriftToken

public static org.apache.hadoop.security.token.Token<? extends org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier> extractThriftToken(String tokenStrForm,
                                                                                                                                                                 String tokenSignature)
                                                                                                                                                          throws org.apache.hadoop.hive.metastore.api.MetaException,
                                                                                                                                                                 org.apache.thrift.TException,
                                                                                                                                                                 IOException

Throws:

org.apache.hadoop.hive.metastore.api.MetaException

org.apache.thrift.TException

IOException

getStorageHandler

public static HiveStorageHandler getStorageHandler(org.apache.hadoop.conf.Configuration conf,
                                                   StorerInfo storerInfo)
                                            throws IOException

Create an instance of a storage handler defined in storerInfo. If one cannot be found then FosterStorageHandler is used to encapsulate the InputFormat, OutputFormat and SerDe. This StorageHandler assumes the other supplied storage artifacts are for a file-based storage system.

Parameters:

conf - job's configuration will be used to configure the Configurable StorageHandler

storerInfo - StorerInfo to definining the StorageHandler and InputFormat, OutputFormat and SerDe

Returns:

storageHandler instance

Throws:

IOException

getStorageHandler

public static HiveStorageHandler getStorageHandler(org.apache.hadoop.conf.Configuration conf,
                                                   PartInfo partitionInfo)
                                            throws IOException

Throws:

IOException

getStorageHandler

public static HiveStorageHandler getStorageHandler(org.apache.hadoop.conf.Configuration conf,
                                                   String storageHandler,
                                                   String serDe,
                                                   String inputFormat,
                                                   String outputFormat)
                                            throws IOException

Create an instance of a storage handler. If storageHandler == null, then surrrogate StorageHandler is used to encapsulate the InputFormat, OutputFormat and SerDe. This StorageHandler assumes the other supplied storage artifacts are for a file-based storage system.

Parameters:

conf - job's configuration will be used to configure the Configurable StorageHandler

storageHandler - fully qualified class name of the desired StorageHandle instance

serDe - fully qualified class name of the desired SerDe instance

inputFormat - fully qualified class name of the desired InputFormat instance

outputFormat - fully qualified class name of the desired outputFormat instance

Returns:

storageHandler instance

Throws:

IOException

getDbAndTableName

public static Pair<String,String> getDbAndTableName(String tableName)
                                             throws IOException

Throws:

IOException

getInputJobProperties

public static Map<String,String> getInputJobProperties(HiveStorageHandler storageHandler,
                                                       InputJobInfo inputJobInfo)

configureOutputStorageHandler

@InterfaceAudience.Private
 @InterfaceStability.Evolving
public static void configureOutputStorageHandler(HiveStorageHandler storageHandler,
                                                                                                          org.apache.hadoop.conf.Configuration conf,
                                                                                                          OutputJobInfo outputJobInfo)

copyConf

public static void copyConf(org.apache.hadoop.conf.Configuration src,
                            org.apache.hadoop.conf.Configuration dest)

Replace the contents of dest with the contents of src

Parameters:

src -

dest -

getHiveMetastoreClient

public static IMetaStoreClient getHiveMetastoreClient(HiveConf hiveConf)
                                               throws org.apache.hadoop.hive.metastore.api.MetaException,
                                                      IOException

Get or create a hive client depending on whether it exits in cache or not

Parameters:

hiveConf - The hive configuration

Returns:

the client

Throws:

org.apache.hadoop.hive.metastore.api.MetaException - When HiveMetaStoreClient couldn't be created

IOException

getHiveClient

@Deprecated
public static HiveMetaStoreClient getHiveClient(HiveConf hiveConf)
                                                     throws org.apache.hadoop.hive.metastore.api.MetaException,
                                                            IOException

Deprecated.

Get or create a hive client depending on whether it exits in cache or not.

Parameters:

hiveConf - The hive configuration

Returns:

the client

Throws:

org.apache.hadoop.hive.metastore.api.MetaException - When HiveMetaStoreClient couldn't be created

IOException

closeHiveClientQuietly

public static void closeHiveClientQuietly(IMetaStoreClient client)

getHiveConf

public static HiveConf getHiveConf(org.apache.hadoop.conf.Configuration conf)
                            throws IOException

Throws:

IOException

storePropertiesToHiveConf

public static HiveConf storePropertiesToHiveConf(Properties properties,
                                                 HiveConf hiveConf)
                                          throws IOException

Throws:

IOException

getJobConfFromContext

public static org.apache.hadoop.mapred.JobConf getJobConfFromContext(org.apache.hadoop.mapreduce.JobContext jobContext)

getHCatKeyHiveConf

public static Map<String,String> getHCatKeyHiveConf(org.apache.hadoop.mapred.JobConf conf)

copyJobPropertiesToJobConf

public static void copyJobPropertiesToJobConf(Map<String,String> jobProperties,
                                              org.apache.hadoop.mapred.JobConf jobConf)

isHadoop23

public static boolean isHadoop23()

makePathASafeFileName

public static String makePathASafeFileName(String filePath)

Used by various tests to make sure the path is safe for Windows

assertNotNull

public static void assertNotNull(Object t,
                                 String msg,
                                 org.slf4j.Logger logger)