org.apache.hadoop.hive.druid

Class DruidStorageHandlerUtils

java.lang.Object

org.apache.hadoop.hive.druid.DruidStorageHandlerUtils

public final class DruidStorageHandlerUtils
extends Object

Utils class for Druid storage handler.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static interface	DruidStorageHandlerUtils.DataPusher Simple interface for retry operations

Field Summary

Fields

Modifier and Type	Field and Description
static com.google.common.collect.Interner<io.druid.timeline.DataSegment>	DATA_SEGMENT_INTERNER Generic Interner implementation used to read segments object from metadata storage
static org.joda.time.Interval	DEFAULT_INTERVAL
static String	DEFAULT_TIMESTAMP_COLUMN
static String	DESCRIPTOR_JSON
static String	EVENT_TIMESTAMP_COLUMN
static io.druid.segment.IndexIO	INDEX_IO Used by druid to perform IO on indexes
static io.druid.segment.IndexMergerV9	INDEX_MERGER_V9 Used by druid to merge indexes
static String	INDEX_ZIP
static com.fasterxml.jackson.databind.ObjectMapper	JSON_MAPPER Mapper to use to serialize/deserialize Druid objects (JSON)
static com.fasterxml.jackson.databind.ObjectMapper	SMILE_MAPPER Mapper to use to serialize/deserialize Druid objects (SMILE)

Constructor Summary

Constructors

Constructor and Description
DruidStorageHandlerUtils()

Method Summary

Modifier and Type	Method and Description
static void	addDependencyJars(org.apache.hadoop.conf.Configuration conf, Class<?>... classes)
static String	createScanAllQuery(String dataSourceName)
static io.druid.segment.loading.DataSegmentPusher	createSegmentPusherForDirectory(String segmentDirectory, org.apache.hadoop.conf.Configuration configuration)
static com.metamx.http.client.Request	createSmileRequest(String address, io.druid.query.Query query) Method that creates a request for Druid query using SMILE format.
static boolean	disableDataSource(io.druid.metadata.SQLMetadataConnector connector, io.druid.metadata.MetadataStorageTablesConfig metadataStorageTablesConfig, String dataSource)
static void	disableDataSourceWithHandle(org.skife.jdbi.v2.Handle handle, io.druid.metadata.MetadataStorageTablesConfig metadataStorageTablesConfig, String dataSource)
static Collection<String>	getAllDataSourceNames(io.druid.metadata.SQLMetadataConnector connector, io.druid.metadata.MetadataStorageTablesConfig metadataStorageTablesConfig)
static List<io.druid.timeline.DataSegment>	getCreatedSegments(org.apache.hadoop.fs.Path taskDir, org.apache.hadoop.conf.Configuration conf)
static List<io.druid.timeline.DataSegment>	getDataSegmentList(io.druid.metadata.SQLMetadataConnector connector, io.druid.metadata.MetadataStorageTablesConfig metadataStorageTablesConfig, String dataSource)
static io.druid.java.util.common.Pair<List<io.druid.data.input.impl.DimensionSchema>,io.druid.query.aggregation.AggregatorFactory[]>	getDimensionsAndAggregates(org.apache.hadoop.conf.Configuration jc, List<String> columnNames, List<TypeInfo> columnTypes)
static io.druid.segment.indexing.granularity.GranularitySpec	getGranularitySpec(org.apache.hadoop.conf.Configuration configuration, Properties tableProperties)
static io.druid.segment.IndexSpec	getIndexSpec(org.apache.hadoop.conf.Configuration jc)
static org.apache.hadoop.fs.Path	getPath(io.druid.timeline.DataSegment dataSegment)
static String	getURL(com.metamx.http.client.HttpClient client, URL url)
static org.apache.hadoop.fs.Path	makeSegmentDescriptorOutputPath(io.druid.timeline.DataSegment pushedSegment, org.apache.hadoop.fs.Path segmentsDescriptorDir)
static List<io.druid.timeline.DataSegment>	publishSegmentsAndCommit(io.druid.metadata.SQLMetadataConnector connector, io.druid.metadata.MetadataStorageTablesConfig metadataStorageTablesConfig, String dataSource, List<io.druid.timeline.DataSegment> segments, boolean overwrite, org.apache.hadoop.conf.Configuration conf, io.druid.segment.loading.DataSegmentPusher dataSegmentPusher) First computes the segments timeline to accommodate new segments for insert into case Then moves segments to druid deep storage with updated metadata/version ALL IS DONE IN ONE TRANSACTION
static io.druid.timeline.DataSegment	publishSegmentWithShardSpec(io.druid.timeline.DataSegment segment, io.druid.timeline.partition.ShardSpec shardSpec, String version, org.apache.hadoop.fs.FileSystem fs, io.druid.segment.loading.DataSegmentPusher dataSegmentPusher)
static InputStream	submitRequest(com.metamx.http.client.HttpClient client, com.metamx.http.client.Request request) Method that submits a request to an Http address and retrieves the result.
static void	writeSegmentDescriptor(org.apache.hadoop.fs.FileSystem outputFS, io.druid.timeline.DataSegment segment, org.apache.hadoop.fs.Path descriptorPath) This function will write to filesystem serialized from of segment descriptor if an existing file exists it will try to replace it.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

DEFAULT_TIMESTAMP_COLUMN

public static final String DEFAULT_TIMESTAMP_COLUMN

See Also:

Constant Field Values

EVENT_TIMESTAMP_COLUMN

public static final String EVENT_TIMESTAMP_COLUMN

See Also:

Constant Field Values

INDEX_ZIP

public static final String INDEX_ZIP

See Also:

Constant Field Values

DESCRIPTOR_JSON

public static final String DESCRIPTOR_JSON

See Also:

Constant Field Values

DEFAULT_INTERVAL

public static final org.joda.time.Interval DEFAULT_INTERVAL

JSON_MAPPER

public static final com.fasterxml.jackson.databind.ObjectMapper JSON_MAPPER

Mapper to use to serialize/deserialize Druid objects (JSON)

SMILE_MAPPER

public static final com.fasterxml.jackson.databind.ObjectMapper SMILE_MAPPER

Mapper to use to serialize/deserialize Druid objects (SMILE)

INDEX_IO

public static final io.druid.segment.IndexIO INDEX_IO

Used by druid to perform IO on indexes

INDEX_MERGER_V9

public static final io.druid.segment.IndexMergerV9 INDEX_MERGER_V9

Used by druid to merge indexes

DATA_SEGMENT_INTERNER

public static final com.google.common.collect.Interner<io.druid.timeline.DataSegment> DATA_SEGMENT_INTERNER

Generic Interner implementation used to read segments object from metadata storage

Constructor Detail

DruidStorageHandlerUtils

public DruidStorageHandlerUtils()

Method Detail

createSmileRequest

public static com.metamx.http.client.Request createSmileRequest(String address,
                                                                io.druid.query.Query query)
                                                         throws IOException

Method that creates a request for Druid query using SMILE format.

Parameters:

address -

query -

Returns:

Throws:

IOException

submitRequest

public static InputStream submitRequest(com.metamx.http.client.HttpClient client,
                                        com.metamx.http.client.Request request)
                                 throws IOException

Method that submits a request to an Http address and retrieves the result. The caller is responsible for closing the stream once it finishes consuming it.

Parameters:

client -

request -

Returns:

Throws:

IOException

getURL

public static String getURL(com.metamx.http.client.HttpClient client,
                            URL url)
                     throws IOException

Throws:

IOException

getCreatedSegments

public static List<io.druid.timeline.DataSegment> getCreatedSegments(org.apache.hadoop.fs.Path taskDir,
                                                                     org.apache.hadoop.conf.Configuration conf)
                                                              throws IOException

Parameters:

taskDir - path to the directory containing the segments descriptor info the descriptor path will be .../workingPath/task_id/DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME/*.json

conf - hadoop conf to get the file system

Returns:

List of DataSegments

Throws:

IOException - can be for the case we did not produce data.

writeSegmentDescriptor

public static void writeSegmentDescriptor(org.apache.hadoop.fs.FileSystem outputFS,
                                          io.druid.timeline.DataSegment segment,
                                          org.apache.hadoop.fs.Path descriptorPath)
                                   throws IOException

This function will write to filesystem serialized from of segment descriptor if an existing file exists it will try to replace it.

Parameters:

outputFS - filesystem

segment - DataSegment object

descriptorPath - path

Throws:

IOException

getAllDataSourceNames

public static Collection<String> getAllDataSourceNames(io.druid.metadata.SQLMetadataConnector connector,
                                                       io.druid.metadata.MetadataStorageTablesConfig metadataStorageTablesConfig)

Parameters:

connector - SQL metadata connector to the metadata storage

metadataStorageTablesConfig - Table config

Returns:

all the active data sources in the metadata storage

disableDataSource

public static boolean disableDataSource(io.druid.metadata.SQLMetadataConnector connector,
                                        io.druid.metadata.MetadataStorageTablesConfig metadataStorageTablesConfig,
                                        String dataSource)

Parameters:

connector - SQL connector to metadata

metadataStorageTablesConfig - Tables configuration

dataSource - Name of data source

Returns:

true if the data source was successfully disabled false otherwise

publishSegmentsAndCommit

public static List<io.druid.timeline.DataSegment> publishSegmentsAndCommit(io.druid.metadata.SQLMetadataConnector connector,
                                                                           io.druid.metadata.MetadataStorageTablesConfig metadataStorageTablesConfig,
                                                                           String dataSource,
                                                                           List<io.druid.timeline.DataSegment> segments,
                                                                           boolean overwrite,
                                                                           org.apache.hadoop.conf.Configuration conf,
                                                                           io.druid.segment.loading.DataSegmentPusher dataSegmentPusher)
                                                                    throws org.skife.jdbi.v2.exceptions.CallbackFailedException

First computes the segments timeline to accommodate new segments for insert into case Then moves segments to druid deep storage with updated metadata/version ALL IS DONE IN ONE TRANSACTION

Parameters:

connector - DBI connector to commit

metadataStorageTablesConfig - Druid metadata tables definitions

dataSource - Druid datasource name

segments - List of segments to move and commit to metadata

overwrite - if it is an insert overwrite

conf - Configuration

dataSegmentPusher - segment pusher

Returns:

List of successfully published Druid segments. This list has the updated versions and metadata about segments after move and timeline sorting

Throws:

org.skife.jdbi.v2.exceptions.CallbackFailedException

disableDataSourceWithHandle

public static void disableDataSourceWithHandle(org.skife.jdbi.v2.Handle handle,
                                               io.druid.metadata.MetadataStorageTablesConfig metadataStorageTablesConfig,
                                               String dataSource)

getDataSegmentList

public static List<io.druid.timeline.DataSegment> getDataSegmentList(io.druid.metadata.SQLMetadataConnector connector,
                                                                     io.druid.metadata.MetadataStorageTablesConfig metadataStorageTablesConfig,
                                                                     String dataSource)

Parameters:

connector - SQL connector to metadata

metadataStorageTablesConfig - Tables configuration

dataSource - Name of data source

Returns:

List of all data segments part of the given data source

makeSegmentDescriptorOutputPath

public static org.apache.hadoop.fs.Path makeSegmentDescriptorOutputPath(io.druid.timeline.DataSegment pushedSegment,
                                                                        org.apache.hadoop.fs.Path segmentsDescriptorDir)

Parameters:

pushedSegment -

segmentsDescriptorDir -

Returns:

a sanitize file name

createScanAllQuery

public static String createScanAllQuery(String dataSourceName)
                                 throws com.fasterxml.jackson.core.JsonProcessingException

Throws:

com.fasterxml.jackson.core.JsonProcessingException

addDependencyJars

public static void addDependencyJars(org.apache.hadoop.conf.Configuration conf,
                                     Class<?>... classes)
                              throws IOException

Throws:

IOException

createSegmentPusherForDirectory

public static io.druid.segment.loading.DataSegmentPusher createSegmentPusherForDirectory(String segmentDirectory,
                                                                                         org.apache.hadoop.conf.Configuration configuration)
                                                                                  throws IOException

Throws:

IOException

publishSegmentWithShardSpec

public static io.druid.timeline.DataSegment publishSegmentWithShardSpec(io.druid.timeline.DataSegment segment,
                                                                        io.druid.timeline.partition.ShardSpec shardSpec,
                                                                        String version,
                                                                        org.apache.hadoop.fs.FileSystem fs,
                                                                        io.druid.segment.loading.DataSegmentPusher dataSegmentPusher)
                                                                 throws IOException

Throws:

IOException

getPath

public static org.apache.hadoop.fs.Path getPath(io.druid.timeline.DataSegment dataSegment)

getGranularitySpec

public static io.druid.segment.indexing.granularity.GranularitySpec getGranularitySpec(org.apache.hadoop.conf.Configuration configuration,
                                                                                       Properties tableProperties)

getIndexSpec

public static io.druid.segment.IndexSpec getIndexSpec(org.apache.hadoop.conf.Configuration jc)

getDimensionsAndAggregates

public static io.druid.java.util.common.Pair<List<io.druid.data.input.impl.DimensionSchema>,io.druid.query.aggregation.AggregatorFactory[]> getDimensionsAndAggregates(org.apache.hadoop.conf.Configuration jc,
                                                                                                                                                                       List<String> columnNames,
                                                                                                                                                                       List<TypeInfo> columnTypes)